diff options
Diffstat (limited to 'src')
32 files changed, 240 insertions, 135 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index abf3669b5b35..279ae62f69cc 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -38,6 +38,12 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } +static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} + static int _main(int argc, char * * argv) { { @@ -97,9 +103,10 @@ static int _main(int argc, char * * argv) source >> drvPath; auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling - && ( neededSystem == settings.thisSystem - || settings.extraPlatforms.get().count(neededSystem) > 0); + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2a194d0e08c9..d8e10d9f20e1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -130,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v) } +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + string showType(const Value & v) { switch (v.type) { @@ -144,8 +154,10 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tFloat: return "a float"; } @@ -1799,6 +1811,7 @@ void EvalState::printStats() gc.attr("totalBytes", totalBytes); } #endif + if (countCalls) { { auto obj = topObj.object("primops"); @@ -1834,6 +1847,11 @@ void EvalState::printStats() } } } + + if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string & s) { list.elem(s); }); + } } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9fe3878916d5..a314e01e0a71 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -81,7 +81,7 @@ public: /* The allowed filesystem paths in restricted or pure evaluation mode. */ - std::experimental::optional<PathSet> allowedPaths; + std::optional<PathSet> allowedPaths; Value vEmptySet; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d38ed2df3b18..21a4d7917fce 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v, } -std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v, +std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index daaa635fe1b1..d7860fc6a4bc 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos; /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::experimental::optional<DrvInfo> getDerivation(EvalState & state, +std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 79f3e2f4506e..80f7a492b1a1 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index cbd576d7d126..7870393076b8 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,7 +1,7 @@ %glr-parser %pure-parser %locations -%error-verbose +%define parse.error verbose %defines /* %no-lines */ %parse-param { void * scanner } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0da9f702f4bb..39073725e9c4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tBool); } +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} struct CompareValues { @@ -555,7 +561,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - std::experimental::optional<std::string> outputHash; + std::optional<std::string> outputHash; std::string outputHashAlgo; bool outputHashRecursive = false; @@ -2169,6 +2175,7 @@ void EvalState::createBaseEnv() addPrimOp("__isInt", 1, prim_isInt); addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); addPrimOp("__addErrorContext", 2, prim_addErrorContext); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b46d2f258265..aaf02c856d4f 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -26,7 +26,7 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref<Store> store, const std::string & uri, - std::experimental::optional<std::string> ref, std::string rev, + std::optional<std::string> ref, std::string rev, const std::string & name) { if (evalSettings.pureEval && rev == "") @@ -190,7 +190,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::experimental::optional<std::string> ref; + std::optional<std::string> ref; std::string rev; std::string name = "source"; PathSet context; diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 44929f7eea06..91faea122ce1 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -75,6 +75,13 @@ public: } size_t totalSize() const; + + template<typename T> + void dump(T callback) + { + for (auto & s : symbols) + callback(s); + } }; } diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 38bc85c484eb..37b03dcd42c0 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 47ee8b48f4b4..dbadfacd51ef 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -12,6 +12,7 @@ #include "json.hh" #include "nar-info.hh" #include "parsed-derivations.hh" +#include "machines.hh" #include <algorithm> #include <iostream> @@ -802,6 +803,9 @@ private: /* Whether we're currently doing a chroot build. */ bool useChroot = false; + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite; + Path chrootRootDir; /* RAII object to delete the chroot directory. */ @@ -993,6 +997,13 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut , wantedOutputs(wantedOutputs) , buildMode(buildMode) { +#if __linux__ + needsHashRewrite = !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + needsHashRewrite = true; +#endif + state = &DerivationGoal::getDerivation; name = (format("building of '%1%'") % drvPath).str(); trace("created"); @@ -2072,7 +2083,7 @@ void DerivationGoal::startBuilder() #endif } - else { + if (needsHashRewrite) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -2413,7 +2424,7 @@ void DerivationGoal::writeStructuredAttrs() objects consisting entirely of those values. (So nested arrays or objects are not supported.) */ - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { if (value.is_string()) return shellEscape(value); @@ -2872,6 +2883,10 @@ void DerivationGoal::runChild() for (auto & i : missingPaths) { sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); } + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); + } sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -3050,7 +3065,9 @@ void DerivationGoal::registerOutputs() throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } else { + } + + if (needsHashRewrite) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() @@ -3311,8 +3328,8 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) struct Checks { bool ignoreSelfRefs = false; - std::experimental::optional<uint64_t> maxSize, maxClosureSize; - std::experimental::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + std::optional<uint64_t> maxSize, maxClosureSize; + std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; }; /* Compute the closure and closure size of some output. This @@ -3359,7 +3376,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) info.path, closureSize, *checks.maxClosureSize); } - auto checkRefs = [&](const std::experimental::optional<Strings> & value, bool allowed, bool recursive) + auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive) { if (!value) return; @@ -3413,7 +3430,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) if (maxClosureSize != output->end()) checks.maxClosureSize = maxClosureSize->get<uint64_t>(); - auto get = [&](const std::string & name) -> std::experimental::optional<Strings> { + auto get = [&](const std::string & name) -> std::optional<Strings> { auto i = output->find(name); if (i != output->end()) { Strings res; @@ -4411,6 +4428,11 @@ static void primeCache(Store & store, const PathSet & paths) PathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bbf05..22382ab1d6e8 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader writeFull(wakeupPipe.writeSide.get(), " "); } +#ifdef ENABLE_S3 + std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + void enqueueDownload(const DownloadRequest & request, Callback<DownloadResult> callback) override { @@ -622,12 +638,15 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 - S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable - auto slash = request.uri.find('/', 5); - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", request.uri); - std::string bucketName(request.uri, 5, slash - 5); - std::string key(request.uri, slash + 1); + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b415d5421476..d8a5da0d49e2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -129,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(); - if (roots.find(gcRoot) == roots.end()) + Roots roots = findRoots(false); + if (roots[storePath].count(gcRoot) == 0) printError( format( "warning: '%1%' is not in a directory where the garbage collector looks for roots; " @@ -197,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path) } -std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) -{ - std::set<std::pair<pid_t, Path>> tempRoots; +static std::string censored = "{censored}"; + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +{ /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { @@ -250,14 +251,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots.emplace(pid, root); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } fds.push_back(fd); /* keep open */ } - - return tempRoots; } @@ -266,7 +265,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); else printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); }; @@ -306,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = storeDir + "/" + baseNameOf(path); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); } } @@ -321,10 +320,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -Roots LocalStore::findRootsNoTemp() +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { - Roots roots; - /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); @@ -333,32 +330,22 @@ Roots LocalStore::findRootsNoTemp() NIX_ROOT_FINDER environment variable. This is typically used to add running programs to the set of roots (to prevent them from being garbage collected). */ - size_t n = 0; - for (auto & root : findRuntimeRoots()) - roots[fmt("{memory:%d}", n++)] = root; - - return roots; + findRuntimeRoots(roots, censor); } -Roots LocalStore::findRoots() +Roots LocalStore::findRoots(bool censor) { - Roots roots = findRootsNoTemp(); + Roots roots; + findRootsNoTemp(roots, censor); FDs fds; - pid_t prev = -1; - size_t n = 0; - for (auto & root : readTempRoots(fds)) { - if (prev != root.first) n = 0; - prev = root.first; - roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second; - } + findTempRoots(fds, roots, censor); return roots; } - -static void readProcLink(const string & file, StringSet & paths) +static void readProcLink(const string & file, Roots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -377,8 +364,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - paths.emplace(static_cast<char *>(buf), res); - return; + roots[std::string(static_cast<char *>(buf), res)] + .emplace(file); } static string quoteRegexChars(const string & raw) @@ -387,20 +374,20 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, StringSet & paths) +static void readFileRoots(const char * path, Roots & roots) { try { - paths.emplace(readFile(path)); + roots[readFile(path)].emplace(path); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; } } -PathSet LocalStore::findRuntimeRoots() +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - PathSet roots; - StringSet paths; + Roots unchecked; + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -410,10 +397,10 @@ PathSet LocalStore::findRuntimeRoots() while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); if (!fdDir) { if (errno == ENOENT || errno == EACCES) @@ -422,9 +409,8 @@ PathSet LocalStore::findRuntimeRoots() } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); - } + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); } if (errno) { if (errno == ESRCH) @@ -434,18 +420,19 @@ PathSet LocalStore::findRuntimeRoots() fdDir.reset(); try { - auto mapLines = - tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); - for (const auto& line : mapLines) { + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); + for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - paths.emplace(match[1]); + unchecked[match[1]].emplace(mapFile); } - auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true); + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - paths.emplace(i->str()); + unchecked[i->str()].emplace(envFile); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -465,7 +452,7 @@ PathSet LocalStore::findRuntimeRoots() for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - paths.emplace(match[1]); + unchecked[match[1]].emplace("{lsof}"); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -473,21 +460,23 @@ PathSet LocalStore::findRuntimeRoots() #endif #if defined(__linux__) - readFileRoots("/proc/sys/kernel/modprobe", paths); - readFileRoots("/proc/sys/kernel/fbsplash", paths); - readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & i : paths) - if (isInStore(i)) { - Path path = toStorePath(i); - if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { + for (auto & [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); + if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.insert(path); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } - - return roots; + } } @@ -754,16 +743,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printError(format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.second); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths can be added to the set of temporary roots. */ FDs fds; - for (auto & root : readTempRoots(fds)) - state.tempRoots.insert(root.second); + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto & root : tempRoots) + state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); /* After this point the set of roots or temporary roots cannot diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fce963433a5e..6b655647b031 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -180,11 +180,11 @@ private: typedef std::shared_ptr<AutoCloseFD> FDPtr; typedef list<FDPtr> FDs; - std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds); + void findTempRoots(FDs & fds, Roots & roots, bool censor); public: - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -267,9 +267,9 @@ private: void findRoots(const Path & path, unsigned char type, Roots & roots); - Roots findRootsNoTemp(); + void findRootsNoTemp(Roots & roots, bool censor); - PathSet findRuntimeRoots(); + void findRuntimeRoots(Roots & roots, bool censor); void removeUnusedLinks(const GCState & state); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index edd03d147832..f848582dafd4 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines) Machines getMachines() { - Machines machines; - - parseMachines(settings.builders, machines); - + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); return machines; } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faadcbe..6d67b1e03808 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index dc3286482736..17fde00a0167 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) } } -std::experimental::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const +std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); @@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const } } -std::experimental::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const +std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0a82c146172b..ed07dc652e8d 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -8,22 +8,22 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::experimental::optional<nlohmann::json> structuredAttrs; + std::optional<nlohmann::json> structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::experimental::optional<nlohmann::json> & getStructuredAttrs() const + const std::optional<nlohmann::json> & getStructuredAttrs() const { return structuredAttrs; } - std::experimental::optional<std::string> getStringAttr(const std::string & name) const; + std::optional<std::string> getStringAttr(const std::string & name) const; bool getBoolAttr(const std::string & name, bool def = false) const; - std::experimental::optional<Strings> getStringsAttr(const std::string & name) const; + std::optional<Strings> getStringsAttr(const std::string & name) const; StringSet getRequiredSystemFeatures() const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index def140cfbe18..15faf78a526d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -596,7 +596,7 @@ void RemoteStore::syncWithGC() } -Roots RemoteStore::findRoots() +Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); conn->to << wopFindRoots; @@ -606,7 +606,7 @@ Roots RemoteStore::findRoots() while (count--) { Path link = readString(conn->from); Path target = readStorePath(*this, conn->from); - result[link] = target; + result[target].emplace(link); } return result; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4f554b5980e8..80f18ab715d9 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -82,7 +82,7 @@ public: void syncWithGC() override; - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -149,7 +149,7 @@ public: private: ref<RemoteStore::Connection> openConnection() override; - std::experimental::optional<std::string> path; + std::optional<std::string> path; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 51de89e0d92f..cd547a964850 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -126,6 +126,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region res->endpointOverride = endpoint; } res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; res->retryStrategy = std::make_shared<RetryStrategy>(); res->caFile = settings.caFile; return res; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 913a1112141e..c13ff11564ec 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -842,12 +842,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) { auto uri(uri_); - Store::Params params(extraParams); + Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { @@ -873,6 +872,15 @@ ref<Store> openStore(const std::string & uri_, } uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f504735e0f84..7a1b31d0ff59 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,8 @@ #include <atomic> #include <limits> #include <map> +#include <unordered_map> +#include <unordered_set> #include <memory> #include <string> @@ -47,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map<Path, Path> Roots; +typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; struct GCOptions @@ -483,8 +485,10 @@ public: /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) { unsupported("findRoots"); } /* Perform a garbage collection. */ @@ -798,4 +802,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); + } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index bb68e82886d0..3aa120270970 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -331,7 +331,7 @@ struct RestoreSink : ParseSink filesystem doesn't support preallocation (e.g. on OpenSolaris). Since preallocation is just an optimisation, ignore it. */ - if (errno && errno != EINVAL) + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) throw SysError(format("preallocating file of %1% bytes") % len); } #endif diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 9b8290e634c9..8b83f842c324 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,7 +2,7 @@ #include <map> #include <list> -#include <experimental/optional> +#include <optional> namespace nix { @@ -64,7 +64,7 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - std::experimental::optional<Value> get(const Key & key) + std::optional<Value> get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0e75eeec2bfe..8201549fd7d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,7 +171,7 @@ std::unique_ptr<Source> sinkToSource( std::function<void(Sink &)> fun; std::function<void()> eof; - std::experimental::optional<coro_t::pull_type> coro; + std::optional<coro_t::pull_type> coro; bool started = false; SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7eca35577b01..e3dcd246c681 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -965,7 +965,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const std::experimental::optional<std::string> & input) + const std::optional<std::string> & input) { RunOptions opts(program, args); opts.searchPath = searchPath; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bda87bee433e..9f239bff371a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,7 +14,7 @@ #include <cstdio> #include <map> #include <sstream> -#include <experimental/optional> +#include <optional> #include <future> #ifndef HAVE_STRUCT_DIRENT_D_TYPE @@ -259,14 +259,14 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::experimental::optional<std::string> & input = {}); + const std::optional<std::string> & input = {}); struct RunOptions { Path program; bool searchPath = true; Strings args; - std::experimental::optional<std::string> input; + std::optional<std::string> input; Source * standardIn = nullptr; Sink * standardOut = nullptr; bool _killStderr = false; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8368c3266142..8d63b8f362ec 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -475,11 +475,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(); + Roots roots = store->findRoots(!trusted); logger->stopWork(); - to << roots.size(); + + size_t size = 0; for (auto & i : roots) - to << i.first << i.second; + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; + break; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 33138baff388..f324056bb3a1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -427,10 +427,11 @@ static void opQuery(Strings opFlags, Strings opArgs) maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } - Roots roots = store->findRoots(); - for (auto & i : roots) - if (referrers.find(i.second) != referrers.end()) - cout << format("%1%\n") % i.first; + Roots roots = store->findRoots(false); + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) + for (auto & link : links) + cout << format("%1% -> %2%\n") % link % target; break; } @@ -590,9 +591,14 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); if (printRoots) { - Roots roots = store->findRoots(); - for (auto & i : roots) - cout << i.first << " -> " << i.second << std::endl; + Roots roots = store->findRoots(false); + std::set<std::pair<Path, Path>> roots2; + // Transpose and sort the roots. + for (auto & [target, links] : roots) + for (auto & link : links) + roots2.emplace(link, target); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; } else { diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index d0003790d3b9..e86b96e3f3f2 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -8,7 +8,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { Path path; - std::experimental::optional<std::string> namePart; + std::optional<std::string> namePart; CmdAddToStore() { |