diff options
Diffstat (limited to 'third_party/nix/src/libstore/store-api.cc')
| -rw-r--r-- | third_party/nix/src/libstore/store-api.cc | 1381 |
1 files changed, 646 insertions, 735 deletions
diff --git a/third_party/nix/src/libstore/store-api.cc b/third_party/nix/src/libstore/store-api.cc index 5f63c53b56..d8ee689044 100644 --- a/third_party/nix/src/libstore/store-api.cc +++ b/third_party/nix/src/libstore/store-api.cc @@ -1,117 +1,98 @@ +#include "store-api.hh" +#include <future> #include "crypto.hh" +#include "derivations.hh" #include "globals.hh" -#include "store-api.hh" -#include "util.hh" +#include "json.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "json.hh" -#include "derivations.hh" - -#include <future> - +#include "util.hh" namespace nix { - -bool Store::isInStore(const Path & path) const -{ - return isInDir(path, storeDir); -} - - -bool Store::isStorePath(const Path & path) const -{ - return isInStore(path) - && path.size() >= storeDir.size() + 1 + storePathHashLen - && path.find('/', storeDir.size() + 1) == Path::npos; +bool Store::isInStore(const Path& path) const { + return isInDir(path, storeDir); } - -void Store::assertStorePath(const Path & path) const -{ - if (!isStorePath(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); +bool Store::isStorePath(const Path& path) const { + return isInStore(path) && + path.size() >= storeDir.size() + 1 + storePathHashLen && + path.find('/', storeDir.size() + 1) == Path::npos; } - -Path Store::toStorePath(const Path & path) const -{ - if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); - Path::size_type slash = path.find('/', storeDir.size() + 1); - if (slash == Path::npos) - return path; - else - return Path(path, 0, slash); +void Store::assertStorePath(const Path& path) const { + if (!isStorePath(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); } - -Path Store::followLinksToStore(const Path & _path) const -{ - Path path = absPath(_path); - while (!isInStore(path)) { - if (!isLink(path)) break; - string target = readLink(path); - path = absPath(target, dirOf(path)); - } - if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); +Path Store::toStorePath(const Path& path) const { + if (!isInStore(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); + Path::size_type slash = path.find('/', storeDir.size() + 1); + if (slash == Path::npos) return path; + else + return Path(path, 0, slash); +} + +Path Store::followLinksToStore(const Path& _path) const { + Path path = absPath(_path); + while (!isInStore(path)) { + if (!isLink(path)) break; + string target = readLink(path); + path = absPath(target, dirOf(path)); + } + if (!isInStore(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); + return path; +} + +Path Store::followLinksToStorePath(const Path& path) const { + return toStorePath(followLinksToStore(path)); +} + +string storePathToName(const Path& path) { + auto base = baseNameOf(path); + assert(base.size() == storePathHashLen || + (base.size() > storePathHashLen && base[storePathHashLen] == '-')); + return base.size() == storePathHashLen ? "" + : string(base, storePathHashLen + 1); +} + +string storePathToHash(const Path& path) { + auto base = baseNameOf(path); + assert(base.size() >= storePathHashLen); + return string(base, 0, storePathHashLen); +} + +void checkStoreName(const string& name) { + string validChars = "+-._?="; + + auto baseError = + format( + "The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % + validChars % name; + + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (string(name, 0, 1) == ".") + throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); + for (auto& i : name) + if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || + (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { + throw Error(baseError % (format("the '%1%' character is invalid") % i)); + } } - -Path Store::followLinksToStorePath(const Path & path) const -{ - return toStorePath(followLinksToStore(path)); -} - - -string storePathToName(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); - return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); -} - - -string storePathToHash(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() >= storePathHashLen); - return string(base, 0, storePathHashLen); -} - - -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - - auto baseError = format("The path name '%2%' is invalid: %3%. " - "Path names are alphanumeric and can include the symbols %1% " - "and must not begin with a period. " - "Note: If '%2%' is a source file and you cannot rename it on " - "disk, builtins.path { name = ... } can be used to give it an " - "alternative name.") % validChars % name; - - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (string(name, 0, 1) == ".") - throw Error(baseError % "it is illegal to start the name with a period"); - /* Disallow names longer than 211 characters. ext4’s max is 256, - but we need extra space for the hash and .chroot extensions. */ - if (name.length() > 211) - throw Error(baseError % "name must be less than 212 characters"); - for (auto & i : name) - if (!((i >= 'A' && i <= 'Z') || - (i >= 'a' && i <= 'z') || - (i >= '0' && i <= '9') || - validChars.find(i) != string::npos)) - { - throw Error(baseError % (format("the '%1%' character is invalid") % i)); - } -} - - /* Store paths have the following form: <store>/<h>-<name> @@ -182,802 +163,732 @@ void checkStoreName(const string & name) "source:". */ +Path Store::makeStorePath(const string& type, const Hash& hash, + const string& name) const { + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; -Path Store::makeStorePath(const string & type, - const Hash & hash, const string & name) const -{ - /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; + checkStoreName(name); - checkStoreName(name); - - return storeDir + "/" - + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) - + "-" + name; + return storeDir + "/" + + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) + + "-" + name; } - -Path Store::makeOutputPath(const string & id, - const Hash & hash, const string & name) const -{ - return makeStorePath("output:" + id, hash, - name + (id == "out" ? "" : "-" + id)); +Path Store::makeOutputPath(const string& id, const Hash& hash, + const string& name) const { + return makeStorePath("output:" + id, hash, + name + (id == "out" ? "" : "-" + id)); } - -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const -{ - return hash.type == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), - name); +Path Store::makeFixedOutputPath(bool recursive, const Hash& hash, + const string& name) const { + return hash.type == htSHA256 && recursive + ? makeStorePath("source", hash, name) + : makeStorePath( + "output:out", + hashString(htSHA256, + "fixed:out:" + (recursive ? (string) "r:" : "") + + hash.to_string(Base16) + ":"), + name); } - -Path Store::makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - string type = "text"; - for (auto & i : references) { - type += ":"; - type += i; - } - return makeStorePath(type, hash, name); -} - - -std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, - const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const -{ - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - Path dstPath = makeFixedOutputPath(recursive, h, name); - return std::pair<Path, Hash>(dstPath, h); +Path Store::makeTextPath(const string& name, const Hash& hash, + const PathSet& references) const { + assert(hash.type == htSHA256); + /* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ + string type = "text"; + for (auto& i : references) { + type += ":"; + type += i; + } + return makeStorePath(type, hash, name); } - -Path Store::computeStorePathForText(const string & name, const string & s, - const PathSet & references) const -{ - return makeTextPath(name, hashString(htSHA256, s), references); +std::pair<Path, Hash> Store::computeStorePathForPath(const string& name, + const Path& srcPath, + bool recursive, + HashType hashAlgo, + PathFilter& filter) const { + Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + Path dstPath = makeFixedOutputPath(recursive, h, name); + return std::pair<Path, Hash>(dstPath, h); } - -Store::Store(const Params & params) - : Config(params) - , state({(size_t) pathInfoCacheSize}) -{ +Path Store::computeStorePathForText(const string& name, const string& s, + const PathSet& references) const { + return makeTextPath(name, hashString(htSHA256, s), references); } +Store::Store(const Params& params) + : Config(params), state({(size_t)pathInfoCacheSize}) {} -std::string Store::getUri() -{ - return ""; -} - +std::string Store::getUri() { return ""; } -bool Store::isValidPath(const Path & storePath) -{ - assertStorePath(storePath); +bool Store::isValidPath(const Path& storePath) { + assertStorePath(storePath); - auto hashPart = storePathToHash(storePath); + auto hashPart = storePathToHash(storePath); - { - auto state_(state.lock()); - auto res = state_->pathInfoCache.get(hashPart); - if (res) { - stats.narInfoReadAverted++; - return *res != 0; - } + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + return *res != 0; } - - if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); - if (res.first != NarInfoDiskCache::oUnknown) { - stats.narInfoReadAverted++; - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, - res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); - return res.first == NarInfoDiskCache::oValid; - } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert( + hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + return res.first == NarInfoDiskCache::oValid; } + } - bool valid = isValidPathUncached(storePath); + bool valid = isValidPathUncached(storePath); - if (diskCache && !valid) - // FIXME: handle valid = true case. - diskCache->upsertNarInfo(getUri(), hashPart, 0); + if (diskCache && !valid) + // FIXME: handle valid = true case. + diskCache->upsertNarInfo(getUri(), hashPart, 0); - return valid; + return valid; } - /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const Path & path) -{ - try { - queryPathInfo(path); - return true; - } catch (InvalidPath &) { - return false; - } +bool Store::isValidPathUncached(const Path& path) { + try { + queryPathInfo(path); + return true; + } catch (InvalidPath&) { + return false; + } } +ref<const ValidPathInfo> Store::queryPathInfo(const Path& storePath) { + std::promise<ref<ValidPathInfo>> promise; -ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) -{ - std::promise<ref<ValidPathInfo>> promise; - - queryPathInfo(storePath, - {[&](std::future<ref<ValidPathInfo>> result) { - try { - promise.set_value(result.get()); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }}); + queryPathInfo(storePath, {[&](std::future<ref<ValidPathInfo>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); - return promise.get_future().get(); + return promise.get_future().get(); } +void Store::queryPathInfo(const Path& storePath, + Callback<ref<ValidPathInfo>> callback) noexcept { + std::string hashPart; -void Store::queryPathInfo(const Path & storePath, - Callback<ref<ValidPathInfo>> callback) noexcept -{ - std::string hashPart; + try { + assertStorePath(storePath); - try { - assertStorePath(storePath); + hashPart = storePathToHash(storePath); - hashPart = storePathToHash(storePath); + { + auto res = state.lock()->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + if (!*res) + throw InvalidPath(format("path '%s' is not valid") % storePath); + return callback(ref<ValidPathInfo>(*res)); + } + } + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; { - auto res = state.lock()->pathInfoCache.get(hashPart); - if (res) { - stats.narInfoReadAverted++; - if (!*res) - throw InvalidPath(format("path '%s' is not valid") % storePath); - return callback(ref<ValidPathInfo>(*res)); - } - } - - if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); - if (res.first != NarInfoDiskCache::oUnknown) { - stats.narInfoReadAverted++; - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, - res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); - if (res.first == NarInfoDiskCache::oInvalid || - (res.second->path != storePath && storePathToName(storePath) != "")) - throw InvalidPath(format("path '%s' is not valid") % storePath); - } - return callback(ref<ValidPathInfo>(res.second)); - } + auto state_(state.lock()); + state_->pathInfoCache.upsert( + hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + if (res.first == NarInfoDiskCache::oInvalid || + (res.second->path != storePath && + storePathToName(storePath) != "")) + throw InvalidPath(format("path '%s' is not valid") % storePath); } + return callback(ref<ValidPathInfo>(res.second)); + } + } - } catch (...) { return callback.rethrow(); } - - auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - - queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) { - - try { - auto info = fut.get(); - - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, info); - - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, info); - } - - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { - stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); - } - - (*callbackPtr)(ref<ValidPathInfo>(info)); - } catch (...) { callbackPtr->rethrow(); } - }}); -} + } catch (...) { + return callback.rethrow(); + } + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); -PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) -{ - struct State - { - size_t left; - PathSet valid; - std::exception_ptr exc; - }; - - Sync<State> state_(State{paths.size(), PathSet()}); + queryPathInfoUncached( + storePath, {[this, storePath, hashPart, callbackPtr]( + std::future<std::shared_ptr<ValidPathInfo>> fut) { + try { + auto info = fut.get(); - std::condition_variable wakeup; - ThreadPool pool; + if (diskCache) diskCache->upsertNarInfo(getUri(), hashPart, info); - auto doQuery = [&](const Path & path ) { - checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { - auto state(state_.lock()); - try { - auto info = fut.get(); - state->valid.insert(path); - } catch (InvalidPath &) { - } catch (...) { - state->exc = std::current_exception(); - } - assert(state->left); - if (!--state->left) - wakeup.notify_one(); + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } + + if (!info || + (info->path != storePath && storePathToName(storePath) != "")) { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } + + (*callbackPtr)(ref<ValidPathInfo>(info)); + } catch (...) { + callbackPtr->rethrow(); + } + }}); +} + +PathSet Store::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + struct State { + size_t left; + PathSet valid; + std::exception_ptr exc; + }; + + Sync<State> state_(State{paths.size(), PathSet()}); + + std::condition_variable wakeup; + ThreadPool pool; + + auto doQuery = [&](const Path& path) { + checkInterrupt(); + queryPathInfo( + path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); + state->valid.insert(path); + } catch (InvalidPath&) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (!--state->left) wakeup.notify_one(); }}); - }; + }; - for (auto & path : paths) - pool.enqueue(std::bind(doQuery, path)); + for (auto& path : paths) pool.enqueue(std::bind(doQuery, path)); - pool.process(); + pool.process(); - while (true) { - auto state(state_.lock()); - if (!state->left) { - if (state->exc) std::rethrow_exception(state->exc); - return state->valid; - } - state.wait(wakeup); + while (true) { + auto state(state_.lock()); + if (!state->left) { + if (state->exc) std::rethrow_exception(state->exc); + return state->valid; } + state.wait(wakeup); + } } - /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string Store::makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash) -{ - string s = ""; +string Store::makeValidityRegistration(const PathSet& paths, bool showDerivers, + bool showHash) { + string s = ""; - for (auto & i : paths) { - s += i + "\n"; + for (auto& i : paths) { + s += i + "\n"; - auto info = queryPathInfo(i); + auto info = queryPathInfo(i); - if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; - s += (format("%1%\n") % info->narSize).str(); - } - - Path deriver = showDerivers ? info->deriver : ""; - s += deriver + "\n"; - - s += (format("%1%\n") % info->references.size()).str(); - - for (auto & j : info->references) - s += j + "\n"; + if (showHash) { + s += info->narHash.to_string(Base16, false) + "\n"; + s += (format("%1%\n") % info->narSize).str(); } - return s; -} - + Path deriver = showDerivers ? info->deriver : ""; + s += deriver + "\n"; -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) -{ - auto jsonList = jsonOut.list(); + s += (format("%1%\n") % info->references.size()).str(); - for (auto storePath : storePaths) { - auto jsonPath = jsonList.object(); - jsonPath.attr("path", storePath); + for (auto& j : info->references) s += j + "\n"; + } - try { - auto info = queryPathInfo(storePath); - storePath = info->path; - - jsonPath - .attr("narHash", info->narHash.to_string()) - .attr("narSize", info->narSize); + return s; +} - { - auto jsonRefs = jsonPath.list("references"); - for (auto & ref : info->references) - jsonRefs.elem(ref); - } +void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths, + bool includeImpureInfo, bool showClosureSize, + AllowInvalidFlag allowInvalid) { + auto jsonList = jsonOut.list(); - if (info->ca != "") - jsonPath.attr("ca", info->ca); + for (auto storePath : storePaths) { + auto jsonPath = jsonList.object(); + jsonPath.attr("path", storePath); - std::pair<uint64_t, uint64_t> closureSizes; + try { + auto info = queryPathInfo(storePath); + storePath = info->path; - if (showClosureSize) { - closureSizes = getClosureSize(storePath); - jsonPath.attr("closureSize", closureSizes.first); - } + jsonPath.attr("narHash", info->narHash.to_string()) + .attr("narSize", info->narSize); - if (includeImpureInfo) { + { + auto jsonRefs = jsonPath.list("references"); + for (auto& ref : info->references) jsonRefs.elem(ref); + } - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); + if (info->ca != "") jsonPath.attr("ca", info->ca); - if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + std::pair<uint64_t, uint64_t> closureSizes; - if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + if (showClosureSize) { + closureSizes = getClosureSize(storePath); + jsonPath.attr("closureSize", closureSizes.first); + } - if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); - for (auto & sig : info->sigs) - jsonSigs.elem(sig); - } + if (includeImpureInfo) { + if (info->deriver != "") jsonPath.attr("deriver", info->deriver); - auto narInfo = std::dynamic_pointer_cast<const NarInfo>( - std::shared_ptr<const ValidPathInfo>(info)); + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath.attr("url", narInfo->url); - if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); - if (narInfo->fileSize) - jsonPath.attr("downloadSize", narInfo->fileSize); - if (showClosureSize) - jsonPath.attr("closureDownloadSize", closureSizes.second); - } - } + if (info->ultimate) jsonPath.attr("ultimate", info->ultimate); - } catch (InvalidPath &) { - jsonPath.attr("valid", false); + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto& sig : info->sigs) jsonSigs.elem(sig); } - } -} - -std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - PathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} + if (narInfo) { + if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); + if (narInfo->fileHash) + jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + if (narInfo->fileSize) + jsonPath.attr("downloadSize", narInfo->fileSize); + if (showClosureSize) + jsonPath.attr("closureDownloadSize", closureSizes.second); + } + } -const Store::Stats & Store::getStats() -{ - { - auto state_(state.lock()); - stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } catch (InvalidPath&) { + jsonPath.attr("valid", false); } - return stats; + } } - -void Store::buildPaths(const PathSet & paths, BuildMode buildMode) -{ - for (auto & path : paths) - if (isDerivation(path)) - unsupported("buildPaths"); - - if (queryValidPaths(paths).size() != paths.size()) - unsupported("buildPaths"); +std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path& storePath) { + uint64_t totalNarSize = 0, totalDownloadSize = 0; + PathSet closure; + computeFSClosure(storePath, closure, false, false); + for (auto& p : closure) { + auto info = queryPathInfo(p); + totalNarSize += info->narSize; + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + if (narInfo) totalDownloadSize += narInfo->fileSize; + } + return {totalNarSize, totalDownloadSize}; } +const Store::Stats& Store::getStats() { + { + auto state_(state.lock()); + stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } + return stats; +} -void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) -{ - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); - - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); - - auto info = srcStore->queryPathInfo(storePath); - - uint64_t total = 0; - - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } +void Store::buildPaths(const PathSet& paths, BuildMode buildMode) { + for (auto& path : paths) + if (isDerivation(path)) unsupported("buildPaths"); - if (info->ultimate) { - auto info2 = make_ref<ValidPathInfo>(*info); - info2->ultimate = false; - info = info2; - } + if (queryValidPaths(paths).size() != paths.size()) unsupported("buildPaths"); +} - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); +void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, + const Path& storePath, RepairFlag repair, + CheckSigsFlag checkSigs) { + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, + srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore->queryPathInfo(storePath); + + uint64_t total = 0; + + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref<ValidPathInfo>(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource( + [&](Sink& sink) { + LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); }); srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); + }, + [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", + storePath, srcStore->getUri()); + }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); } +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const PathSet& storePaths, RepairFlag repair, + CheckSigsFlag checkSigs, SubstituteFlag substitute) { + PathSet valid = dstStore->queryValidPaths(storePaths, substitute); -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) -{ - PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + PathSet missing; + for (auto& path : storePaths) + if (!valid.count(path)) missing.insert(path); - PathSet missing; - for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); + if (missing.empty()) return; - if (missing.empty()) return; + Activity act(*logger, lvlInfo, actCopyPaths, + fmt("copying %d paths", missing.size())); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + std::atomic<size_t> nrDone{0}; + std::atomic<size_t> nrFailed{0}; + std::atomic<uint64_t> bytesExpected{0}; + std::atomic<uint64_t> nrRunning{0}; - std::atomic<size_t> nrDone{0}; - std::atomic<size_t> nrFailed{0}; - std::atomic<uint64_t> bytesExpected{0}; - std::atomic<uint64_t> nrRunning{0}; + auto showProgress = [&]() { + act.progress(nrDone, missing.size(), nrRunning, nrFailed); + }; - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; + ThreadPool pool; - ThreadPool pool; + processGraph<Path>( + pool, PathSet(missing.begin(), missing.end()), - processGraph<Path>(pool, - PathSet(missing.begin(), missing.end()), + [&](const Path& storePath) { + if (dstStore->isValidPath(storePath)) { + nrDone++; + showProgress(); + return PathSet(); + } - [&](const Path & storePath) { - if (dstStore->isValidPath(storePath)) { - nrDone++; - showProgress(); - return PathSet(); - } + auto info = srcStore->queryPathInfo(storePath); - auto info = srcStore->queryPathInfo(storePath); - - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); - - return info->references; - }, - - [&](const Path & storePath) { - checkInterrupt(); - - if (!dstStore->isValidPath(storePath)) { - MaintainCount<decltype(nrRunning)> mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); - showProgress(); - return; - } - } + bytesExpected += info->narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info->references; + }, + + [&](const Path& storePath) { + checkInterrupt(); - nrDone++; + if (!dstStore->isValidPath(storePath)) { + MaintainCount<decltype(nrRunning)> mc(nrRunning); + showProgress(); + try { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } catch (Error& e) { + nrFailed++; + if (!settings.keepGoing) throw e; + logger->log(lvlError, + format("could not copy %s: %s") % storePath % e.what()); showProgress(); - }); -} + return; + } + } + nrDone++; + showProgress(); + }); +} void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, - SubstituteFlag substitute) -{ - PathSet closure; - srcStore->computeFSClosure({storePaths}, closure); - copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); -} - - -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) -{ - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } - if (hashGiven) { - string s; - getline(str, s); - info.narHash = Hash(s, htSHA256); - getline(str, s); - if (!string2Int(s, info.narSize)) throw Error("number expected"); - } - getline(str, info.deriver); - string s; int n; - getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(str, s); - info.references.insert(s); - } - if (!str || str.eof()) throw Error("missing input"); + const PathSet& storePaths, RepairFlag repair, + CheckSigsFlag checkSigs, SubstituteFlag substitute) { + PathSet closure; + srcStore->computeFSClosure({storePaths}, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + +ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven) { + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { + info.path = ""; return info; -} - - -string showPaths(const PathSet & paths) -{ + } + if (hashGiven) { string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + getline(str, s); + info.narHash = Hash(s, htSHA256); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); + } + getline(str, info.deriver); + string s; + int n; + getline(str, s); + if (!string2Int(s, n)) throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) throw Error("missing input"); + return info; } - -std::string ValidPathInfo::fingerprint() const -{ - if (narSize == 0 || !narHash) - throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known") - % path); - return - "1;" + path + ";" - + narHash.to_string(Base32) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", references); +string showPaths(const PathSet& paths) { + string s; + for (auto& i : paths) { + if (s.size() != 0) s += ", "; + s += "'" + i + "'"; + } + return s; } - -void ValidPathInfo::sign(const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint())); +std::string ValidPathInfo::fingerprint() const { + if (narSize == 0 || !narHash) + throw Error(format("cannot calculate fingerprint of path '%s' because its " + "size/hash is not known") % + path); + return "1;" + path + ";" + narHash.to_string(Base32) + ";" + + std::to_string(narSize) + ";" + concatStringsSep(",", references); } - -bool ValidPathInfo::isContentAddressed(const Store & store) const -{ - auto warn = [&]() { - printError(format("warning: path '%s' claims to be content-addressed but isn't") % path); - }; - - if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); - if (store.makeTextPath(storePathToName(path), hash, references) == path) - return true; - else - warn(); - } - - else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - Hash hash(std::string(ca, recursive ? 8 : 6)); - if (references.empty() && - store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) - return true; - else - warn(); - } - - return false; +void ValidPathInfo::sign(const SecretKey& secretKey) { + sigs.insert(secretKey.signDetached(fingerprint())); } +bool ValidPathInfo::isContentAddressed(const Store& store) const { + auto warn = [&]() { + printError( + format("warning: path '%s' claims to be content-addressed but isn't") % + path); + }; -size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const -{ - if (isContentAddressed(store)) return maxSigs; + if (hasPrefix(ca, "text:")) { + Hash hash(std::string(ca, 5)); + if (store.makeTextPath(storePathToName(path), hash, references) == path) + return true; + else + warn(); + } + + else if (hasPrefix(ca, "fixed:")) { + bool recursive = ca.compare(6, 2, "r:") == 0; + Hash hash(std::string(ca, recursive ? 8 : 6)); + if (references.empty() && + store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == + path) + return true; + else + warn(); + } - size_t good = 0; - for (auto & sig : sigs) - if (checkSignature(publicKeys, sig)) - good++; - return good; + return false; } +size_t ValidPathInfo::checkSignatures(const Store& store, + const PublicKeys& publicKeys) const { + if (isContentAddressed(store)) return maxSigs; -bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const -{ - return verifyDetached(fingerprint(), sig, publicKeys); + size_t good = 0; + for (auto& sig : sigs) + if (checkSignature(publicKeys, sig)) good++; + return good; } - -Strings ValidPathInfo::shortRefs() const -{ - Strings refs; - for (auto & r : references) - refs.push_back(baseNameOf(r)); - return refs; +bool ValidPathInfo::checkSignature(const PublicKeys& publicKeys, + const std::string& sig) const { + return verifyDetached(fingerprint(), sig, publicKeys); } - -std::string makeFixedOutputCA(bool recursive, const Hash & hash) -{ - return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); +Strings ValidPathInfo::shortRefs() const { + Strings refs; + for (auto& r : references) refs.push_back(baseNameOf(r)); + return refs; } - -void Store::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor); +std::string makeFixedOutputCA(bool recursive, const Hash& hash) { + return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); } -void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - StringSource source(*nar); - addToStore(info, source, repair, checkSigs, accessor); +void Store::addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, + accessor); } +void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + StringSource source(*nar); + addToStore(info, source, repair, checkSigs, accessor); } +} // namespace nix #include "local-store.hh" #include "remote-store.hh" - namespace nix { - -RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; +RegisterStoreImplementation::Implementations* + RegisterStoreImplementation::implementations = 0; /* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) -{ - auto uri(uri_); - Store::Params params; - auto q = uri.find('?'); - if (q != std::string::npos) { - for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; +std::pair<std::string, Store::Params> splitUriAndParams( + const std::string& uri_) { + auto uri(uri_); + Store::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { + auto e = s.find('='); + if (e != std::string::npos) { + auto value = s.substr(e + 1); + std::string decoded; + for (size_t i = 0; i < value.size();) { + if (value[i] == '%') { + if (i + 2 >= value.size()) + throw Error("invalid URI parameter '%s'", value); + try { + decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw Error("invalid URI parameter '%s'", value); } + } else + decoded += value[i++]; } - uri = uri_.substr(0, q); + params[s.substr(0, e)] = decoded; + } } - return {uri, params}; -} - -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) -{ - auto [uri, uriParams] = splitUriAndParams(uri_); - auto params = extraParams; - params.insert(uriParams.begin(), uriParams.end()); - - for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri, params); - if (store) { - store->warnUnknownSettings(); - return ref<Store>(store); - } + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +ref<Store> openStore(const std::string& uri_, + const Store::Params& extraParams) { + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); + + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri, params); + if (store) { + store->warnUnknownSettings(); + return ref<Store>(store); } + } - throw Error("don't know how to open Nix store '%s'", uri); -} - - -StoreType getStoreType(const std::string & uri, const std::string & stateDir) -{ - if (uri == "daemon") { - return tDaemon; - } else if (uri == "local" || hasPrefix(uri, "/")) { - return tLocal; - } else if (uri == "" || uri == "auto") { - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return tLocal; - else if (pathExists(settings.nixDaemonSocketFile)) - return tDaemon; - else - return tLocal; - } else { - return tOther; - } + throw Error("don't know how to open Nix store '%s'", uri); } - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { - case tDaemon: - return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); - case tLocal: { - Store::Params params2 = params; - if (hasPrefix(uri, "/")) - params2["root"] = uri; - return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); - } - default: - return nullptr; +StoreType getStoreType(const std::string& uri, const std::string& stateDir) { + if (uri == "daemon") { + return tDaemon; + } else if (uri == "local" || hasPrefix(uri, "/")) { + return tLocal; + } else if (uri == "" || uri == "auto") { + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return tLocal; + else if (pathExists(settings.nixDaemonSocketFile)) + return tDaemon; + else + return tLocal; + } else { + return tOther; + } +} + +static RegisterStoreImplementation regStore([](const std::string& uri, + const Store::Params& params) + -> std::shared_ptr<Store> { + switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + case tDaemon: + return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); + case tLocal: { + Store::Params params2 = params; + if (hasPrefix(uri, "/")) params2["root"] = uri; + return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); } + default: + return nullptr; + } }); +std::list<ref<Store>> getDefaultSubstituters() { + static auto stores([]() { + std::list<ref<Store>> stores; -std::list<ref<Store>> getDefaultSubstituters() -{ - static auto stores([]() { - std::list<ref<Store>> stores; - - StringSet done; + StringSet done; - auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); - try { - stores.push_back(openStore(uri)); - } catch (Error & e) { - printError("warning: %s", e.what()); - } - }; + auto addStore = [&](const std::string& uri) { + if (done.count(uri)) return; + done.insert(uri); + try { + stores.push_back(openStore(uri)); + } catch (Error& e) { + printError("warning: %s", e.what()); + } + }; - for (auto uri : settings.substituters.get()) - addStore(uri); + for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.extraSubstituters.get()) - addStore(uri); + for (auto uri : settings.extraSubstituters.get()) addStore(uri); - stores.sort([](ref<Store> & a, ref<Store> & b) { - return a->getPriority() < b->getPriority(); - }); - - return stores; - } ()); + stores.sort([](ref<Store>& a, ref<Store>& b) { + return a->getPriority() < b->getPriority(); + }); return stores; -} - + }()); + return stores; } + +} // namespace nix |