diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 177 |
1 files changed, 107 insertions, 70 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b61ea5298e1e..7fc6ff0df0f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,9 +17,9 @@ #include <sstream> #include <thread> #include <future> +#include <chrono> #include <limits.h> -#include <time.h> #include <sys/time.h> #include <sys/wait.h> #include <sys/types.h> @@ -187,6 +187,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) { } +typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; + + /* A mapping used to remember for each child process to what goal it belongs, and file descriptors for receiving log data and output path creation commands. */ @@ -197,8 +200,8 @@ struct Child set<int> fds; bool respectTimeouts; bool inBuildSlot; - time_t lastOutput; /* time we last got output on stdout/stderr */ - time_t timeStarted; + steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + steady_time_point timeStarted; }; @@ -238,7 +241,7 @@ private: WeakGoals waitingForAWhile; /* Last time the goals in `waitingForAWhile' where woken up. */ - time_t lastWokenUp; + steady_time_point lastWokenUp; /* Cache for pathContentsGood(). */ std::map<Path, bool> pathContentsGoodCache; @@ -254,7 +257,7 @@ public: LocalStore & store; - std::shared_ptr<HookInstance> hook; + std::unique_ptr<HookInstance> hook; Worker(LocalStore & store); ~Worker(); @@ -643,7 +646,7 @@ HookInstance::~HookInstance() { try { toHook.writeSide = -1; - pid.kill(true); + if (pid != -1) pid.kill(true); } catch (...) { ignoreException(); } @@ -748,7 +751,7 @@ private: Pipe userNamespaceSync; /* The build hook. */ - std::shared_ptr<HookInstance> hook; + std::unique_ptr<HookInstance> hook; /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -957,7 +960,7 @@ void DerivationGoal::killChild() child. */ ::kill(-pid, SIGKILL); /* ignore the result */ buildUser.kill(); - pid.wait(true); + pid.wait(); } else pid.kill(); @@ -1249,8 +1252,7 @@ void DerivationGoal::inputsRealised() } /* Second, the input sources. */ - for (auto & i : drv->inputSrcs) - worker.store.computeFSClosure(i, inputPaths); + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1270,6 +1272,8 @@ void DerivationGoal::inputsRealised() build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + + result = BuildResult(); } @@ -1343,6 +1347,7 @@ void DerivationGoal::tryToBuild() case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; return; case rpPostpone: @@ -1411,14 +1416,15 @@ void DerivationGoal::buildDone() /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have - simply have closed its end of the pipe --- just don't do that - :-) */ - /* !!! this could block! security problem! solution: kill the - child */ - int status = hook ? hook->pid.wait(true) : pid.wait(true); + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill(true) : pid.kill(true); debug(format("builder process for ‘%1%’ finished") % drvPath); + result.timesBuilt++; + result.stopTime = time(0); + /* So the child is gone now. */ worker.childTerminated(this); @@ -1558,7 +1564,7 @@ HookReply DerivationGoal::tryBuildHook() if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; if (!worker.hook) - worker.hook = std::make_shared<HookInstance>(); + worker.hook = std::make_unique<HookInstance>(); /* Tell the hook about system features (beyond the system type) required from the build machine. (The hook could parse the @@ -1593,8 +1599,7 @@ HookReply DerivationGoal::tryBuildHook() printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - hook = worker.hook; - worker.hook.reset(); + hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the remote system. This unfortunately has to contain the entire @@ -2102,7 +2107,11 @@ void DerivationGoal::startBuilder() /* Create a pipe to get the output of the builder. */ builderOut.create(); + result.startTime = time(0); + /* Fork a child to build the package. */ + ProcessOptions options; + #if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2144,7 +2153,6 @@ void DerivationGoal::startBuilder() userNamespaceSync.create(); - ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { @@ -2155,7 +2163,8 @@ void DerivationGoal::startBuilder() namespace, we can't drop additional groups; they will be mapped to nogroup in the child namespace. There does not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?)*/ + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); @@ -2179,7 +2188,7 @@ void DerivationGoal::startBuilder() _exit(0); }, options); - if (helper.wait(true) != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2210,7 +2219,6 @@ void DerivationGoal::startBuilder() } else #endif { - ProcessOptions options; options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); @@ -2284,12 +2292,8 @@ void DerivationGoal::runChild() outside of the namespace. Making a subtree private is local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ - Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); - for (auto & i : mounts) { - vector<string> fields = tokenizeString<vector<string> >(i, " "); - string fs = decodeOctalEscaped(fields.at(4)); - if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) - throw SysError(format("unable to make filesystem ‘%1%’ private") % fs); + if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { + throw SysError("unable to make ‘/’ private mount"); } /* Bind-mount chroot directory to itself, to treat it as a @@ -2329,6 +2333,7 @@ void DerivationGoal::runChild() ss.push_back("/etc/nsswitch.conf"); ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); + ss.push_back("/var/run/nscd/socket"); } for (auto & i : ss) dirsInChroot[i] = i; @@ -2555,15 +2560,18 @@ void DerivationGoal::runChild() */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { - if (i.first != i.second) + if (i.first != i.second.source) throw Error(format( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second); + % i.first % i.second.source); string path = i.first; struct stat st; - if (lstat(path.c_str(), &st)) + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; throw SysError(format("getting attributes of path ‘%1%’") % path); + } if (S_ISDIR(st.st_mode)) sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); else @@ -2674,7 +2682,9 @@ void DerivationGoal::registerOutputs() outputs to allow hard links between outputs. */ InodesSeen inodesSeen; - Path checkSuffix = "-check"; + Path checkSuffix = ".check"; + bool runDiffHook = settings.get("run-diff-hook", false); + bool keepPreviousRound = settings.keepFailed || runDiffHook; /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -2904,30 +2914,42 @@ void DerivationGoal::registerOutputs() assert(prevInfos.size() == infos.size()); for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { + result.isNonDeterministic = true; Path prev = i->path + checkSuffix; - if (pathExists(prev)) - throw NotDeterministic( - format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") - % i->path % drvPath % prev); - else - throw NotDeterministic( - format("output ‘%1%’ of ‘%2%’ differs from previous round") - % i->path % drvPath); + bool prevExists = keepPreviousRound && pathExists(prev); + auto msg = prevExists + ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) + : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); + + auto diffHook = settings.get("diff-hook", std::string("")); + if (prevExists && diffHook != "" && runDiffHook) { + try { + auto diff = runProgram(diffHook, true, {prev, i->path}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } + + if (settings.get("enforce-determinism", true)) + throw NotDeterministic(msg); + + printError(msg); + curRound = nrRounds; // we know enough, bail out early } - abort(); // shouldn't happen } - if (settings.keepFailed) { + /* If this is the first round of several, then move the output out + of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { Path prev = i.second.path + checkSuffix; deletePath(prev); - if (curRound < nrRounds) { - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); - } + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); } - } if (curRound < nrRounds) { @@ -2935,6 +2957,15 @@ void DerivationGoal::registerOutputs() return; } + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + } + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -3045,7 +3076,8 @@ void DerivationGoal::handleEOF(int fd) void DerivationGoal::flushLine() { - if (settings.verboseBuild) + if (settings.verboseBuild && + (settings.printRepeatedBuilds || curRound == 1)) printError(filterANSIEscapes(currentLogLine, true)); else { logTail.push_back(currentLogLine); @@ -3387,7 +3419,7 @@ Worker::Worker(LocalStore & store) if (working) abort(); working = true; nrLocalBuilds = 0; - lastWokenUp = 0; + lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; } @@ -3496,7 +3528,7 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds, child.goal = goal; child.goal2 = goal.get(); child.fds = fds; - child.timeStarted = child.lastOutput = time(0); + child.timeStarted = child.lastOutput = steady_time_point::clock::now(); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; children.emplace_back(child); @@ -3615,35 +3647,38 @@ void Worker::waitForInput() bool useTimeout = false; struct timeval timeout; timeout.tv_usec = 0; - time_t before = time(0); + auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there is a build timeout, then wait for input until the first deadline for any child. */ - assert(sizeof(time_t) >= sizeof(long)); - time_t nearest = LONG_MAX; // nearest deadline + auto nearest = steady_time_point::max(); // nearest deadline for (auto & i : children) { if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } - if (nearest != LONG_MAX) { - timeout.tv_sec = std::max((time_t) 1, nearest - before); + if (nearest != steady_time_point::max()) { + timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); useTimeout = true; - printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); } /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == 0) + if (lastWokenUp == steady_time_point::min()) printError("waiting for locks or build slots..."); - if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); - } else lastWokenUp = 0; + if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; + timeout.tv_sec = std::max(1L, + (long) std::chrono::duration_cast<std::chrono::seconds>( + lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); + } else lastWokenUp = steady_time_point::min(); + + if (useTimeout) + vomit("sleeping %d seconds", timeout.tv_sec); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) @@ -3663,9 +3698,10 @@ void Worker::waitForInput() throw SysError("waiting for input"); } - time_t after = time(0); + auto after = steady_time_point::clock::now(); - /* Process all available file descriptors. */ + /* Process all available file descriptors. FIXME: this is + O(children * fds). */ decltype(children)::iterator i; for (auto j = children.begin(); j != children.end(); j = i) { i = std::next(j); @@ -3701,7 +3737,7 @@ void Worker::waitForInput() if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && j->respectTimeouts && - after - j->lastOutput >= (time_t) settings.maxSilentTime) + after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { printError( format("%1% timed out after %2% seconds of silence") @@ -3712,7 +3748,7 @@ void Worker::waitForInput() else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && j->respectTimeouts && - after - j->timeStarted >= (time_t) settings.buildTimeout) + after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { printError( format("%1% timed out after %2% seconds") @@ -3721,7 +3757,7 @@ void Worker::waitForInput() } } - if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { GoalPtr goal = i.lock(); @@ -3783,12 +3819,13 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) worker.run(goals); PathSet failed; - for (auto & i : goals) - if (i->getExitCode() == Goal::ecFailed) { + for (auto & i : goals) { + if (i->getExitCode() != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } + } if (!failed.empty()) throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); |