diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 423 |
1 files changed, 220 insertions, 203 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9069d9b06e08..061682377257 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -18,6 +18,7 @@ #include <thread> #include <future> #include <chrono> +#include <regex> #include <limits.h> #include <sys/time.h> @@ -274,6 +275,10 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + Worker(LocalStore & store); ~Worker(); @@ -606,6 +611,10 @@ struct HookInstance /* The process ID of the hook. */ Pid pid; + FdSink sink; + + std::map<ActivityId, Activity> activities; + HookInstance(); ~HookInstance(); @@ -642,11 +651,7 @@ HookInstance::HookInstance() Strings args = { baseNameOf(settings.buildHook), - settings.thisSystem, - std::to_string(settings.maxSilentTime), - std::to_string(settings.buildTimeout), std::to_string(verbosity), - settings.builders }; execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); @@ -657,6 +662,11 @@ HookInstance::HookInstance() pid.setSeparatePG(true); fromHook.writeSide = -1; toHook.readSide = -1; + + sink = FdSink(toHook.writeSide.get()); + for (auto & setting : settings.getSettings()) + sink << 1 << setting.first << setting.second; + sink << 0; } @@ -762,6 +772,8 @@ private: std::string currentLogLine; size_t currentLogLinePos = 0; // to handle carriage return + std::string currentHookLine; + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -843,6 +855,9 @@ private: std::map<ActivityId, Activity> builderActivities; + /* The remote machine on which we're building. */ + std::string machineName; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -899,9 +914,6 @@ private: /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); - /* Handle the exportReferencesGraph attribute. */ - void doExportReferencesGraph(); - /* Run the builder's process. */ void runChild(); @@ -1390,8 +1402,15 @@ void DerivationGoal::tryToBuild() bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); auto started = [&]() { - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, - fmt("building '%s'", drvPath), Logger::Fields{drvPath}); + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", drvPath, curRound, nrRounds); + fmt("building '%s'", drvPath); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, + Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); }; @@ -1619,7 +1638,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || !useDerivation) return rpDecline; + if (!worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1633,21 +1652,29 @@ HookReply DerivationGoal::tryBuildHook() for (auto & i : features) checkStoreName(i); /* !!! abuse */ /* Send the request to the hook. */ - writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%") - % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv->platform % drvPath % concatStringsSep(",", features)).str()); + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << drvPath + << features; + worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ string reply; while (true) { string s = readLine(worker.hook->fromHook.readSide.get()); - if (string(s, 0, 2) == "# ") { + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { reply = string(s, 2); break; } - s += "\n"; - writeToStderr(s); + else { + s += "\n"; + writeToStderr(s); + } } debug(format("hook reply is '%1%'") % reply); @@ -1655,7 +1682,7 @@ HookReply DerivationGoal::tryBuildHook() if (reply == "decline") return rpDecline; else if (reply == "decline-permanently") { - settings.useBuildHook = false; + worker.tryBuildHook = false; worker.hook = 0; return rpDecline; } @@ -1674,18 +1701,19 @@ HookReply DerivationGoal::tryBuildHook() throw; } - printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - hook = std::move(worker.hook); + machineName = readLine(hook->fromHook.readSide.get()); + /* Tell the hook all the inputs that have to be copied to the remote system. */ - writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); + hook->sink << inputPaths; /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); + hook->sink << missingPaths; + hook->sink = FdSink(); hook->toHook.writeSide = -1; /* Create the log file and pipe. */ @@ -1714,16 +1742,44 @@ int childEntry(void * arg) } -void DerivationGoal::startBuilder() +PathSet exportReferences(Store & store, PathSet storePaths) { - auto f = format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : - "building path(s) %1%"); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - printInfo(f % showPaths(missingPaths) % curRound % nrRounds); + PathSet paths; + + for (auto storePath : storePaths) { + /* Check that the store path is valid. */ + if (!store.isInStore(storePath)) + throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") + % storePath); + storePath = store.toStorePath(storePath); + if (!store.isValidPath(storePath)) + throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") + % storePath); + + store.computeFSClosure(storePath, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths2(paths); + + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = store.derivationFromPath(j); + for (auto & k : drv.outputs) + store.computeFSClosure(k.second.path, paths); + } + } + + return paths; +} + + +void DerivationGoal::startBuilder() +{ /* Right platform? */ if (!drv->canBuildLocally()) { throw Error( @@ -1797,7 +1853,29 @@ void DerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - doExportReferencesGraph(); + if (!drv->env.count("__json")) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + Path storePath = *i++; + + /* Write closure info to <fileName>. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences(worker.store, {storePath}), false, false)); + } + } if (useChroot) { @@ -2266,88 +2344,127 @@ void DerivationGoal::initEnv() } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + void DerivationGoal::writeStructuredAttrs() { - auto json = drv->env.find("__json"); - if (json == drv->env.end()) return; + auto jsonAttr = drv->env.find("__json"); + if (jsonAttr == drv->env.end()) return; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites)); -} + try { + auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); -} + auto json = nlohmann::json::parse(jsonStr); + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(worker.store, storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + writeFile(tmpDir + "/.attrs.json", json.dump()); -void DerivationGoal::doExportReferencesGraph() -{ - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); - Strings ss = tokenizeString<Strings>(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - /* Check that the store path is valid. */ - Path storePath = *i++; - if (!worker.store.isInStore(storePath)) - throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") - % storePath); - storePath = worker.store.toStorePath(storePath); - if (!worker.store.isValidPath(storePath)) - throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") - % storePath); + auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { + if (value.is_string()) + return shellEscape(value); - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths, paths2; - worker.store.computeFSClosure(storePath, paths); - paths2 = paths; - - for (auto & j : paths2) { - if (isDerivation(j)) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); } - } - if (!drv->env.count("__json")) { + if (value.is_null()) + return std::string("''"); - /* Write closure info to <fileName>. */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration(paths, false, false)); + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - } else { + return {}; + }; - /* Write a more comprehensive JSON serialisation to - <fileName>. */ - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - worker.store.pathInfoToJSON(jsonRoot, paths, false, true); + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); } - writeFile(tmpDir + "/" + fileName, str.str()); + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } } + + writeFile(tmpDir + "/.attrs.sh", jsonSh); + + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); } } +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % path); +} + + void setupSeccomp() { #if __linux__ @@ -2402,64 +2519,6 @@ void setupSeccomp() } -struct BuilderLogger : Logger -{ - Logger & prevLogger; - - BuilderLogger(Logger & prevLogger) : prevLogger(prevLogger) { } - - void addFields(nlohmann::json & json, const Fields & fields) - { - if (fields.empty()) return; - auto & arr = json["fields"] = nlohmann::json::array(); - for (auto & f : fields) - if (f.type == Logger::Field::tInt) - arr.push_back(f.i); - else if (f.type == Logger::Field::tString) - arr.push_back(f.s); - else - abort(); - } - - void log(Verbosity lvl, const FormatOrString & fs) override - { - prevLogger.log(lvl, fs); - } - - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - nlohmann::json json; - json["action"] = "start"; - json["id"] = act; - json["level"] = lvl; - json["type"] = type; - json["text"] = s; - addFields(json, fields); - // FIXME: handle parent - log(lvlError, "@nix " + json.dump()); - } - - void stopActivity(ActivityId act) override - { - nlohmann::json json; - json["action"] = "stop"; - json["id"] = act; - log(lvlError, "@nix " + json.dump()); - } - - void result(ActivityId act, ResultType type, const Fields & fields) override - { - nlohmann::json json; - json["action"] = "result"; - json["id"] = act; - json["type"] = type; - addFields(json, fields); - log(lvlError, "@nix " + json.dump()); - } -}; - - void DerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite @@ -2868,7 +2927,7 @@ void DerivationGoal::runChild() /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { - logger = new BuilderLogger(*logger); + logger = makeJSONLogger(*logger); if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv, netrcData); else @@ -3309,8 +3368,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) if (logSink) (*logSink)(data); } - if (hook && fd == hook->fromHook.readSide.get()) - printError(chomp(data)); + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } } @@ -3321,56 +3386,10 @@ void DerivationGoal::handleEOF(int fd) } -static Logger::Fields getFields(nlohmann::json & json) -{ - Logger::Fields fields; - for (auto & f : json) { - if (f.type() == nlohmann::json::value_t::number_unsigned) - fields.emplace_back(Logger::Field(f.get<uint64_t>())); - else if (f.type() == nlohmann::json::value_t::string) - fields.emplace_back(Logger::Field(f.get<std::string>())); - else throw Error("unsupported JSON type %d", (int) f.type()); - } - return fields; -} - - void DerivationGoal::flushLine() { - if (hasPrefix(currentLogLine, "@nix ")) { - - try { - auto json = nlohmann::json::parse(std::string(currentLogLine, 5)); - - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (type == actDownload) - builderActivities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act->id)); - } - - else if (action == "stop") - builderActivities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = builderActivities.find((ActivityId) json["id"]); - if (i != builderActivities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act->result(resSetPhase, phase); - } - - } catch (std::exception & e) { - printError("bad log message from builder: %s", e.what()); - } - } + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; else { if (settings.verboseBuild && @@ -3683,8 +3702,6 @@ void SubstitutionGoal::tryToRun() return; } - printInfo(format("fetching path '%1%'...") % storePath); - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3992,7 +4009,7 @@ void Worker::run(const Goals & _topGoals) else { if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( "unable to start any build; either increase '--max-jobs' " - "or enable distributed builds"); + "or enable remote builds"); assert(!awake.empty()); } } |