diff options
Diffstat (limited to 'src/libstore')
| -rw-r--r-- | src/libstore/binary-cache-store.cc | 72 | ||||
| -rw-r--r-- | src/libstore/binary-cache-store.hh | 1 | ||||
| -rw-r--r-- | src/libstore/build.cc | 423 | ||||
| -rw-r--r-- | src/libstore/download.cc | 4 | ||||
| -rw-r--r-- | src/libstore/globals.cc | 13 | ||||
| -rw-r--r-- | src/libstore/globals.hh | 19 | ||||
| -rw-r--r-- | src/libstore/machines.cc | 30 | ||||
| -rw-r--r-- | src/libstore/remote-fs-accessor.cc | 34 | ||||
| -rw-r--r-- | src/libstore/remote-fs-accessor.hh | 12 | ||||
| -rw-r--r-- | src/libstore/remote-store.cc | 2 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 24 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 5 |
12 files changed, 334 insertions, 305 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 556fa3d59355..67607ab3d43a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -17,66 +17,6 @@ namespace nix { -/* Given requests for a path /nix/store/<x>/<y>, this accessor will - first download the NAR for /nix/store/<x> from the binary cache, - build a NAR accessor for that NAR, and use that to access <y>. */ -struct BinaryCacheStoreAccessor : public FSAccessor -{ - ref<BinaryCacheStore> store; - - std::map<Path, ref<FSAccessor>> nars; - - BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) - : store(store) - { - } - - std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) - { - auto path = canonPath(path_); - - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); - - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); - - auto i = nars.find(storePath); - if (i != nars.end()) return {i->second, restPath}; - - StringSink sink; - store->narFromPath(storePath, sink); - - auto accessor = makeNarAccessor(sink.s); - nars.emplace(storePath, accessor); - return {accessor, restPath}; - } - - Stat stat(const Path & path) override - { - auto res = fetch(path); - return res.first->stat(res.second); - } - - StringSet readDirectory(const Path & path) override - { - auto res = fetch(path); - return res.first->readDirectory(res.second); - } - - std::string readFile(const Path & path) override - { - auto res = fetch(path); - return res.first->readFile(res.second); - } - - std::string readLink(const Path & path) override - { - auto res = fetch(path); - return res.first->readLink(res.second); - } -}; - BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) { @@ -161,7 +101,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); - auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); + auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -174,8 +114,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str auto narAccessor = makeNarAccessor(nar); - if (accessor_) + if (accessor_) { accessor_->nars.emplace(info.path, narAccessor); + accessor_->addToCache(info.path, *nar); + } std::function<void(const Path &, JSONPlaceholder &)> recurse; @@ -220,8 +162,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } else { - if (accessor_) + if (accessor_) { accessor_->nars.emplace(info.path, makeNarAccessor(nar)); + accessor_->addToCache(info.path, *nar); + } } /* Compress the NAR. */ @@ -379,7 +323,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, ref<FSAccessor> BinaryCacheStore::getFSAccessor() { - return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); } std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index f9c1c2cbe8a8..d3b0e0bd9332 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -18,6 +18,7 @@ public: const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; private: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9069d9b06e08..061682377257 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -18,6 +18,7 @@ #include <thread> #include <future> #include <chrono> +#include <regex> #include <limits.h> #include <sys/time.h> @@ -274,6 +275,10 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + Worker(LocalStore & store); ~Worker(); @@ -606,6 +611,10 @@ struct HookInstance /* The process ID of the hook. */ Pid pid; + FdSink sink; + + std::map<ActivityId, Activity> activities; + HookInstance(); ~HookInstance(); @@ -642,11 +651,7 @@ HookInstance::HookInstance() Strings args = { baseNameOf(settings.buildHook), - settings.thisSystem, - std::to_string(settings.maxSilentTime), - std::to_string(settings.buildTimeout), std::to_string(verbosity), - settings.builders }; execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); @@ -657,6 +662,11 @@ HookInstance::HookInstance() pid.setSeparatePG(true); fromHook.writeSide = -1; toHook.readSide = -1; + + sink = FdSink(toHook.writeSide.get()); + for (auto & setting : settings.getSettings()) + sink << 1 << setting.first << setting.second; + sink << 0; } @@ -762,6 +772,8 @@ private: std::string currentLogLine; size_t currentLogLinePos = 0; // to handle carriage return + std::string currentHookLine; + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -843,6 +855,9 @@ private: std::map<ActivityId, Activity> builderActivities; + /* The remote machine on which we're building. */ + std::string machineName; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -899,9 +914,6 @@ private: /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); - /* Handle the exportReferencesGraph attribute. */ - void doExportReferencesGraph(); - /* Run the builder's process. */ void runChild(); @@ -1390,8 +1402,15 @@ void DerivationGoal::tryToBuild() bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); auto started = [&]() { - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, - fmt("building '%s'", drvPath), Logger::Fields{drvPath}); + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", drvPath, curRound, nrRounds); + fmt("building '%s'", drvPath); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, + Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); }; @@ -1619,7 +1638,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || !useDerivation) return rpDecline; + if (!worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1633,21 +1652,29 @@ HookReply DerivationGoal::tryBuildHook() for (auto & i : features) checkStoreName(i); /* !!! abuse */ /* Send the request to the hook. */ - writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%") - % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv->platform % drvPath % concatStringsSep(",", features)).str()); + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << drvPath + << features; + worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ string reply; while (true) { string s = readLine(worker.hook->fromHook.readSide.get()); - if (string(s, 0, 2) == "# ") { + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { reply = string(s, 2); break; } - s += "\n"; - writeToStderr(s); + else { + s += "\n"; + writeToStderr(s); + } } debug(format("hook reply is '%1%'") % reply); @@ -1655,7 +1682,7 @@ HookReply DerivationGoal::tryBuildHook() if (reply == "decline") return rpDecline; else if (reply == "decline-permanently") { - settings.useBuildHook = false; + worker.tryBuildHook = false; worker.hook = 0; return rpDecline; } @@ -1674,18 +1701,19 @@ HookReply DerivationGoal::tryBuildHook() throw; } - printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - hook = std::move(worker.hook); + machineName = readLine(hook->fromHook.readSide.get()); + /* Tell the hook all the inputs that have to be copied to the remote system. */ - writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); + hook->sink << inputPaths; /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); + hook->sink << missingPaths; + hook->sink = FdSink(); hook->toHook.writeSide = -1; /* Create the log file and pipe. */ @@ -1714,16 +1742,44 @@ int childEntry(void * arg) } -void DerivationGoal::startBuilder() +PathSet exportReferences(Store & store, PathSet storePaths) { - auto f = format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : - "building path(s) %1%"); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - printInfo(f % showPaths(missingPaths) % curRound % nrRounds); + PathSet paths; + + for (auto storePath : storePaths) { + /* Check that the store path is valid. */ + if (!store.isInStore(storePath)) + throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") + % storePath); + storePath = store.toStorePath(storePath); + if (!store.isValidPath(storePath)) + throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") + % storePath); + + store.computeFSClosure(storePath, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths2(paths); + + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = store.derivationFromPath(j); + for (auto & k : drv.outputs) + store.computeFSClosure(k.second.path, paths); + } + } + + return paths; +} + + +void DerivationGoal::startBuilder() +{ /* Right platform? */ if (!drv->canBuildLocally()) { throw Error( @@ -1797,7 +1853,29 @@ void DerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - doExportReferencesGraph(); + if (!drv->env.count("__json")) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + Path storePath = *i++; + + /* Write closure info to <fileName>. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences(worker.store, {storePath}), false, false)); + } + } if (useChroot) { @@ -2266,88 +2344,127 @@ void DerivationGoal::initEnv() } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + void DerivationGoal::writeStructuredAttrs() { - auto json = drv->env.find("__json"); - if (json == drv->env.end()) return; + auto jsonAttr = drv->env.find("__json"); + if (jsonAttr == drv->env.end()) return; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites)); -} + try { + auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); -} + auto json = nlohmann::json::parse(jsonStr); + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(worker.store, storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + writeFile(tmpDir + "/.attrs.json", json.dump()); -void DerivationGoal::doExportReferencesGraph() -{ - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); - Strings ss = tokenizeString<Strings>(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - /* Check that the store path is valid. */ - Path storePath = *i++; - if (!worker.store.isInStore(storePath)) - throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") - % storePath); - storePath = worker.store.toStorePath(storePath); - if (!worker.store.isValidPath(storePath)) - throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") - % storePath); + auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { + if (value.is_string()) + return shellEscape(value); - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths, paths2; - worker.store.computeFSClosure(storePath, paths); - paths2 = paths; - - for (auto & j : paths2) { - if (isDerivation(j)) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); } - } - if (!drv->env.count("__json")) { + if (value.is_null()) + return std::string("''"); - /* Write closure info to <fileName>. */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration(paths, false, false)); + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - } else { + return {}; + }; - /* Write a more comprehensive JSON serialisation to - <fileName>. */ - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - worker.store.pathInfoToJSON(jsonRoot, paths, false, true); + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); } - writeFile(tmpDir + "/" + fileName, str.str()); + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } } + + writeFile(tmpDir + "/.attrs.sh", jsonSh); + + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); } } +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % path); +} + + void setupSeccomp() { #if __linux__ @@ -2402,64 +2519,6 @@ void setupSeccomp() } -struct BuilderLogger : Logger -{ - Logger & prevLogger; - - BuilderLogger(Logger & prevLogger) : prevLogger(prevLogger) { } - - void addFields(nlohmann::json & json, const Fields & fields) - { - if (fields.empty()) return; - auto & arr = json["fields"] = nlohmann::json::array(); - for (auto & f : fields) - if (f.type == Logger::Field::tInt) - arr.push_back(f.i); - else if (f.type == Logger::Field::tString) - arr.push_back(f.s); - else - abort(); - } - - void log(Verbosity lvl, const FormatOrString & fs) override - { - prevLogger.log(lvl, fs); - } - - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - nlohmann::json json; - json["action"] = "start"; - json["id"] = act; - json["level"] = lvl; - json["type"] = type; - json["text"] = s; - addFields(json, fields); - // FIXME: handle parent - log(lvlError, "@nix " + json.dump()); - } - - void stopActivity(ActivityId act) override - { - nlohmann::json json; - json["action"] = "stop"; - json["id"] = act; - log(lvlError, "@nix " + json.dump()); - } - - void result(ActivityId act, ResultType type, const Fields & fields) override - { - nlohmann::json json; - json["action"] = "result"; - json["id"] = act; - json["type"] = type; - addFields(json, fields); - log(lvlError, "@nix " + json.dump()); - } -}; - - void DerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite @@ -2868,7 +2927,7 @@ void DerivationGoal::runChild() /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { - logger = new BuilderLogger(*logger); + logger = makeJSONLogger(*logger); if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv, netrcData); else @@ -3309,8 +3368,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) if (logSink) (*logSink)(data); } - if (hook && fd == hook->fromHook.readSide.get()) - printError(chomp(data)); + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } } @@ -3321,56 +3386,10 @@ void DerivationGoal::handleEOF(int fd) } -static Logger::Fields getFields(nlohmann::json & json) -{ - Logger::Fields fields; - for (auto & f : json) { - if (f.type() == nlohmann::json::value_t::number_unsigned) - fields.emplace_back(Logger::Field(f.get<uint64_t>())); - else if (f.type() == nlohmann::json::value_t::string) - fields.emplace_back(Logger::Field(f.get<std::string>())); - else throw Error("unsupported JSON type %d", (int) f.type()); - } - return fields; -} - - void DerivationGoal::flushLine() { - if (hasPrefix(currentLogLine, "@nix ")) { - - try { - auto json = nlohmann::json::parse(std::string(currentLogLine, 5)); - - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (type == actDownload) - builderActivities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act->id)); - } - - else if (action == "stop") - builderActivities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = builderActivities.find((ActivityId) json["id"]); - if (i != builderActivities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act->result(resSetPhase, phase); - } - - } catch (std::exception & e) { - printError("bad log message from builder: %s", e.what()); - } - } + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; else { if (settings.verboseBuild && @@ -3683,8 +3702,6 @@ void SubstitutionGoal::tryToRun() return; } - printInfo(format("fetching path '%1%'...") % storePath); - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3992,7 +4009,7 @@ void Worker::run(const Goals & _topGoals) else { if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( "unable to start any build; either increase '--max-jobs' " - "or enable distributed builds"); + "or enable remote builds"); assert(!awake.empty()); } } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 608b8fd399b4..579a5e8c1b59 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -23,6 +23,8 @@ #include <cmath> #include <random> +using namespace std::string_literals; + namespace nix { double getTime() @@ -604,7 +606,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Path cacheDir = getCacheDir() + "/nix/tarballs"; createDirs(cacheDir); - string urlHash = hashString(htSHA256, url).to_string(Base32, false); + string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); Path dataFile = cacheDir + "/" + urlHash + ".info"; Path fileLink = cacheDir + "/" + urlHash + "-file"; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7da4bce87753..4fa02f92085a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -53,7 +53,12 @@ Settings::Settings() /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); - if (s != "") builderFiles = tokenizeString<Strings>(s, ":"); + if (s != "") { + Strings ss; + for (auto & p : tokenizeString<Strings>(s, ":")) + ss.push_back("@" + p); + builders = concatStringsSep(" ", ss); + } #if defined(__linux__) && defined(SANDBOX_SHELL) sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); @@ -111,17 +116,17 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s args.mkFlag() .longName(name) .description("Enable sandboxing.") - .handler([=](Strings ss) { value = smEnabled; }) + .handler([=](std::vector<std::string> ss) { value = smEnabled; }) .category(category); args.mkFlag() .longName("no-" + name) .description("Disable sandboxing.") - .handler([=](Strings ss) { value = smDisabled; }) + .handler([=](std::vector<std::string> ss) { value = smDisabled; }) .category(category); args.mkFlag() .longName("relaxed-" + name) .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](Strings ss) { value = smRelaxed; }) + .handler([=](std::vector<std::string> ss) { value = smRelaxed; }) .category(category); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 264e82a16e20..a4aa842d70fd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "config.hh" +#include "util.hh" #include <map> #include <limits> @@ -84,6 +85,9 @@ public: /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; + Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", + "The default Nix store to use."}; + Setting<bool> keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; @@ -128,19 +132,12 @@ public: "The maximum duration in seconds that a builder can run. " "0 means infinity.", {"build-timeout"}}; - Setting<bool> useBuildHook{this, true, "remote-builds", - "Whether to use build hooks (for distributed builds)."}; - PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", "The path of the helper program that executes builds to remote machines."}; - Setting<std::string> builders{this, "", "builders", + Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", "A semicolon-separated list of build machines, in the format of nix.machines."}; - Setting<Strings> builderFiles{this, - {nixConfDir + "/machines"}, "builder-files", - "A list of files specifying build machines."}; - Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; @@ -228,7 +225,7 @@ public: Setting<bool> restrictEval{this, false, "restrict-eval", "Whether to restrict file system access to paths in $NIX_PATH, " - "and to disallow fetching files from the network."}; + "and network access to the URI prefixes listed in 'allowed-uris'."}; Setting<size_t> buildRepeat{this, 0, "repeat", "The number of times to repeat a build in order to verify determinism.", @@ -274,7 +271,7 @@ public: "Number of parallel HTTP connections.", {"binary-caches-parallel-connections"}}; - Setting<bool> enableHttp2{this, true, "enable-http2", + Setting<bool> enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", @@ -356,6 +353,8 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<Strings> allowedUris{this, {}, "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; }; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 076c3cab3e90..edd03d147832 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -17,7 +17,11 @@ Machine::Machine(decltype(storeUri) storeUri, storeUri( // Backwards compatibility: if the URI is a hostname, // prepend ssh://. - storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto") + storeUri.find("://") != std::string::npos + || hasPrefix(storeUri, "local") + || hasPrefix(storeUri, "remote") + || hasPrefix(storeUri, "auto") + || hasPrefix(storeUri, "/") ? storeUri : "ssh://" + storeUri), systemTypes(systemTypes), @@ -47,9 +51,22 @@ bool Machine::mandatoryMet(const std::set<string> & features) const { void parseMachines(const std::string & s, Machines & machines) { for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { - chomp(line); + trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue; + + if (line[0] == '@') { + auto file = trim(std::string(line, 1)); + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + debug("cannot find machines file '%s'", file); + } + continue; + } + auto tokens = tokenizeString<std::vector<string>>(line); auto sz = tokens.size(); if (sz < 1) @@ -74,15 +91,6 @@ Machines getMachines() { Machines machines; - for (auto & file : settings.builderFiles.get()) { - try { - parseMachines(readFile(file), machines); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - } - parseMachines(settings.builders, machines); return machines; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 098151f8c0f6..ba9620a175bb 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -3,10 +3,29 @@ namespace nix { - -RemoteFSAccessor::RemoteFSAccessor(ref<Store> store) +RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) : store(store) + , cacheDir(cacheDir) { + if (cacheDir != "") + createDirs(cacheDir); +} + +Path RemoteFSAccessor::makeCacheFile(const Path & storePath) +{ + assert(cacheDir != ""); + return fmt("%s/%s.nar", cacheDir, storePathToHash(storePath)); +} + +void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar) +{ + try { + if (cacheDir == "") return; + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(storePath), nar); + } catch (...) { + ignoreException(); + } } std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) @@ -23,7 +42,16 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) if (i != nars.end()) return {i->second, restPath}; StringSink sink; - store->narFromPath(storePath, sink); + + try { + if (cacheDir != "") + *sink.s = nix::readFile(makeCacheFile(storePath)); + } catch (SysError &) { } + + if (sink.s->empty()) { + store->narFromPath(storePath, sink); + addToCache(storePath, *sink.s); + } auto accessor = makeNarAccessor(sink.s); nars.emplace(storePath, accessor); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 28f36c8296e1..2a3fc01eff58 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -12,10 +12,20 @@ class RemoteFSAccessor : public FSAccessor std::map<Path, ref<FSAccessor>> nars; + Path cacheDir; + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_); + + friend class BinaryCacheStore; + + Path makeCacheFile(const Path & storePath); + + void addToCache(const Path & storePath, const std::string & nar); + public: - RemoteFSAccessor(ref<Store> store); + RemoteFSAccessor(ref<Store> store, + const /* FIXME: use std::optional */ Path & cacheDir = ""); Stat stat(const Path & path) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b9076c0474d6..77b41b6bf8a8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -166,7 +166,7 @@ void RemoteStore::setOptions(Connection & conn) << verbosity << settings.maxBuildJobs << settings.maxSilentTime - << settings.useBuildHook + << true << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type << 0 /* obsolete print build trace */ diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa6ade75002a..c57e42fec00d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -565,8 +565,16 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - Activity act(*logger, lvlInfo, actCopyPath, fmt("copying path '%s'", storePath), - {storePath, srcStore->getUri(), dstStore->getUri()}); + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); PushActivity pact(act.id); auto info = srcStore->queryPathInfo(storePath); @@ -619,6 +627,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + if (missing.empty()) return; + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); std::atomic<size_t> nrDone{0}; @@ -833,7 +843,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir) { if (uri == "daemon") { return tDaemon; - } else if (uri == "local") { + } else if (uri == "local" || hasPrefix(uri, "/")) { return tLocal; } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -855,8 +865,12 @@ static RegisterStoreImplementation regStore([]( switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { case tDaemon: return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); - case tLocal: - return std::shared_ptr<Store>(std::make_shared<LocalStore>(params)); + case tLocal: { + Store::Params params2 = params; + if (hasPrefix(uri, "/")) + params2["root"] = uri; + return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); + } default: return nullptr; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5f3d8c7b9529..d1e1b5d6f452 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -716,7 +716,7 @@ void removeTempRoots(); You can pass parameters to the store implementation by appending ‘?key=value&key=value&...’ to the URI. */ -ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), +ref<Store> openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); @@ -727,7 +727,8 @@ enum StoreType { }; -StoreType getStoreType(const std::string & uri = getEnv("NIX_REMOTE"), const std::string & stateDir = settings.nixStateDir); +StoreType getStoreType(const std::string & uri = settings.storeUri.get(), + const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options like |