diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 1657 |
1 files changed, 867 insertions, 790 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8c4412f11a8b..cca357dfb31b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2,16 +2,21 @@ #include "references.hh" #include "pathlocks.hh" -#include "misc.hh" #include "globals.hh" #include "local-store.hh" #include "util.hh" #include "archive.hh" #include "affinity.hh" +#include "builtins.hh" +#include "finally.hh" +#include "compression.hh" +#include <algorithm> +#include <iostream> #include <map> #include <sstream> -#include <algorithm> +#include <thread> +#include <future> #include <limits.h> #include <time.h> @@ -21,57 +26,35 @@ #include <sys/stat.h> #include <sys/utsname.h> #include <sys/select.h> +#include <sys/resource.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> -#include <stdio.h> #include <cstring> #include <pwd.h> #include <grp.h> -#include <bzlib.h> - -/* Includes required for chroot support. */ -#if HAVE_SYS_PARAM_H -#include <sys/param.h> -#endif -#if HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif -#if HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#if HAVE_SCHED_H -#include <sched.h> -#endif - -/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but - <linux/fs.h> does. */ -#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H -#include <linux/fs.h> -#endif - -#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root) - /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - #define SANDBOX_ENABLED 1 #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #else - #define SANDBOX_ENABLED 0 - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/bin" "/usr/bin" + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #endif -#if CHROOT_ENABLED +/* Includes required for chroot support. */ +#if __linux__ #include <sys/socket.h> #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#endif - -#if __linux__ #include <sys/personality.h> +#include <sys/mman.h> +#include <sched.h> +#include <sys/param.h> +#include <sys/mount.h> +#include <sys/syscall.h> +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif #if HAVE_STATVFS @@ -94,6 +77,7 @@ struct HookInstance; /* A pointer to a goal. */ class Goal; +class DerivationGoal; typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; @@ -184,10 +168,10 @@ public: return exitCode; } - /* Cancel the goal. It should wake up its waiters, get rid of any - running child processes that are being monitored by the worker - (important!), etc. */ - virtual void cancel(bool timeout) = 0; + /* Callback in case of a timeout. It should wake up its waiters, + get rid of any running child processes that are being monitored + by the worker (important!), etc. */ + virtual void timedOut() = 0; virtual string key() = 0; @@ -216,8 +200,6 @@ struct Child time_t timeStarted; }; -typedef map<pid_t, Child> Children; - /* The worker class. */ class Worker @@ -237,7 +219,7 @@ private: WeakGoals wantingToBuild; /* Child processes currently running. */ - Children children; + std::list<Child> children; /* Number of build slots occupied. This includes local builds and substitutions but not remote builds via the build hook. */ @@ -257,6 +239,9 @@ private: /* Last time the goals in `waitingForAWhile' where woken up. */ time_t lastWokenUp; + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + public: /* Set if at least one derivation had a BuildError (i.e. permanent @@ -275,6 +260,8 @@ public: /* Make a goal (with caching). */ GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath, + const BasicDerivation & drv, BuildMode buildMode = bmNormal); GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); /* Remove a dead goal. */ @@ -290,14 +277,14 @@ public: /* Registers a running child process. `inBuildSlot' means that the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, pid_t pid, - const set<int> & fds, bool inBuildSlot, bool respectTimeouts); + void childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts); /* Unregisters a running child process. `wakeSleepers' should be false if there is no sense in waking up goals that are sleeping because they can't run yet (e.g., there is no free build slot, or the hook would still say `postpone'). */ - void childTerminated(pid_t pid, bool wakeSleepers = true); + void childTerminated(GoalPtr goal, bool wakeSleepers = true); /* Put `goal' to sleep until a build slot becomes available (which might be right away). */ @@ -320,6 +307,12 @@ public: void waitForInput(); unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); }; @@ -330,8 +323,8 @@ void addToWeakGoals(WeakGoals & goals, GoalPtr p) { // FIXME: necessary? // FIXME: O(n) - foreach (WeakGoals::iterator, i, goals) - if (i->lock() == p) return; + for (auto & i : goals) + if (i.lock() == p) return; goals.push_back(p); } @@ -361,11 +354,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) /* If we failed and keepGoing is not set, we remove all remaining waitees. */ - foreach (Goals::iterator, i, waitees) { - GoalPtr goal = *i; + for (auto & goal : waitees) { WeakGoals waiters2; - foreach (WeakGoals::iterator, j, goal->waiters) - if (j->lock() != shared_from_this()) waiters2.push_back(*j); + for (auto & j : goal->waiters) + if (j.lock() != shared_from_this()) waiters2.push_back(j); goal->waiters = waiters2; } waitees.clear(); @@ -381,8 +373,8 @@ void Goal::amDone(ExitCode result) assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; - foreach (WeakGoals::iterator, i, waiters) { - GoalPtr goal = i->lock(); + for (auto & i : waiters) { + GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); } waiters.clear(); @@ -508,13 +500,13 @@ void UserLock::acquire() /* Find a user account that isn't currently in use for another build. */ - foreach (Strings::iterator, i, users) { - debug(format("trying user ‘%1%’") % *i); + for (auto & i : users) { + debug(format("trying user ‘%1%’") % i); - struct passwd * pw = getpwnam(i->c_str()); + struct passwd * pw = getpwnam(i.c_str()); if (!pw) throw Error(format("the user ‘%1%’ in the group ‘%2%’ does not exist") - % *i % settings.buildUsersGroup); + % i % settings.buildUsersGroup); createDirs(settings.nixStateDir + "/userpool"); @@ -532,7 +524,7 @@ void UserLock::acquire() if (lockFile(fd, ltWrite, false)) { fdUserLock = fd.borrow(); lockedPaths.insert(fnUserLock); - user = *i; + user = i; uid = pw->pw_uid; /* Sanity check... */ @@ -540,6 +532,7 @@ void UserLock::acquire() throw Error(format("the Nix user should not be a member of ‘%1%’") % settings.buildUsersGroup); +#if __linux__ /* Get the list of supplementary groups of this build user. This is usually either empty or contains a group such as "kvm". */ supplementaryGIDs.resize(10); @@ -550,6 +543,7 @@ void UserLock::acquire() throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name); supplementaryGIDs.resize(ngroups); +#endif return; } @@ -634,11 +628,14 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide, 4) == -1) throw SysError("dupping builder's stdout/stderr"); - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); + Strings args = { + baseNameOf(buildHook), + settings.thisSystem, + (format("%1%") % settings.maxSilentTime).str(), + (format("%1%") % settings.buildTimeout).str() + }; + + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); throw SysError(format("executing ‘%1%’") % buildHook); }); @@ -668,12 +665,12 @@ typedef map<string, string> HashRewrites; string rewriteHashes(string s, const HashRewrites & rewrites) { - foreach (HashRewrites::const_iterator, i, rewrites) { - assert(i->first.size() == i->second.size()); + for (auto & i : rewrites) { + assert(i.first.size() == i.second.size()); size_t j = 0; - while ((j = s.find(i->first, j)) != string::npos) { + while ((j = s.find(i.first, j)) != string::npos) { debug(format("rewriting @ %1%") % j); - s.replace(j, i->second.size(), i->second); + s.replace(j, i.second.size(), i.second); } } return s; @@ -690,6 +687,9 @@ class SubstitutionGoal; class DerivationGoal : public Goal { private: + /* Whether to use an on-disk .drv file. */ + bool useDerivation; + /* The path of the derivation. */ Path drvPath; @@ -698,14 +698,14 @@ private: StringSet wantedOutputs; /* Whether additional wanted outputs have been added. */ - bool needRestart; + bool needRestart = false; /* Whether to retry substituting the outputs after building the inputs. */ - bool retrySubstitution; + bool retrySubstitution = false; /* The derivation stored at drvPath. */ - Derivation drv; + std::unique_ptr<BasicDerivation> drv; /* The remainder is state held during the build. */ @@ -735,14 +735,22 @@ private: /* The temporary directory. */ Path tmpDir; + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + /* File descriptor for the log file. */ - FILE * fLogFile; - BZFILE * bzLogFile; AutoCloseFD fdLogFile; + std::shared_ptr<BufferedSink> logFileSink, logSink; /* Number of bytes received from the builder's stdout/stderr. */ unsigned long logSize; + /* The most recent log lines. */ + std::list<std::string> logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -750,16 +758,13 @@ private: std::shared_ptr<HookInstance> hook; /* Whether we're currently doing a chroot build. */ - bool useChroot; + bool useChroot = false; Path chrootRootDir; /* RAII object to delete the chroot directory. */ std::shared_ptr<AutoDelete> autoDelChroot; - /* All inputs that are regular files. */ - PathSet regularInputPaths; - /* Whether this is a fixed-output derivation. */ bool fixedOutput; @@ -772,6 +777,12 @@ private: typedef map<string, string> Environment; Environment env; +#if __APPLE__ + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; + AutoDelete autoDelSandbox; +#endif + /* Hash rewriting. */ HashRewrites rewritesToTmp, rewritesFromTmp; typedef map<Path, Path> RedirectedOutputs; @@ -784,18 +795,29 @@ private: temporary paths. */ PathSet redirectedBadOutputs; - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; + BuildResult result; + + /* The current round, if we're building multiple times. */ + unsigned int curRound = 1; + + unsigned int nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + ValidPathInfos prevInfos; public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); - void cancel(bool timeout); + void timedOut() override; - string key() + string key() override { /* Ensure that derivations get built in order of their name, i.e. a derivation named "aardvark" always comes before @@ -804,7 +826,7 @@ public: return "b$" + storePathToName(drvPath) + "$" + drvPath; } - void work(); + void work() override; Path getDrvPath() { @@ -814,9 +836,12 @@ public: /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); + BuildResult getResult() { return result; } + private: /* The states. */ - void init(); + void getDerivation(); + void loadDerivation(); void haveDerivation(); void outputsSubstituted(); void closureRepaired(); @@ -849,8 +874,9 @@ private: void deleteTmpDir(bool force); /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data); - void handleEOF(int fd); + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; + void flushLine(); /* Return the set of (in)valid paths. */ PathSet checkPathValidity(bool returnValid, bool checkHash); @@ -864,26 +890,43 @@ private: Path addHashRewrite(const Path & path); void repairClosure(); + + void done(BuildResult::Status status, const string & msg = ""); }; -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) +DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + Worker & worker, BuildMode buildMode) : Goal(worker) + , useDerivation(true) + , drvPath(drvPath) , wantedOutputs(wantedOutputs) - , needRestart(false) - , retrySubstitution(false) - , fLogFile(0) - , bzLogFile(0) - , useChroot(false) , buildMode(buildMode) { - this->drvPath = drvPath; - state = &DerivationGoal::init; + state = &DerivationGoal::getDerivation; name = (format("building of ‘%1%’") % drvPath).str(); trace("created"); } +DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(false) + , drvPath(drvPath) + , buildMode(buildMode) +{ + this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); + state = &DerivationGoal::haveDerivation; + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + trace("created"); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(drvPath); +} + + DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a @@ -897,7 +940,7 @@ DerivationGoal::~DerivationGoal() void DerivationGoal::killChild() { if (pid != -1) { - worker.childTerminated(pid); + worker.childTerminated(shared_from_this()); if (buildUser.enabled()) { /* If we're using a build user, then there is a tricky @@ -919,12 +962,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::cancel(bool timeout) +void DerivationGoal::timedOut() { - if (settings.printBuildTrace && timeout) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); killChild(); - amDone(ecFailed); + done(BuildResult::TimedOut); } @@ -943,42 +984,39 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs) wantedOutputs.clear(); needRestart = true; } else - foreach (StringSet::const_iterator, i, outputs) - if (wantedOutputs.find(*i) == wantedOutputs.end()) { - wantedOutputs.insert(*i); + for (auto & i : outputs) + if (wantedOutputs.find(i) == wantedOutputs.end()) { + wantedOutputs.insert(i); needRestart = true; } } -void DerivationGoal::init() +void DerivationGoal::getDerivation() { trace("init"); - if (settings.readOnlyMode) - throw Error(format("cannot build derivation ‘%1%’ - no write access to the Nix store") % drvPath); - /* The first thing to do is to make sure that the derivation exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - haveDerivation(); + loadDerivation(); return; } addWaitee(worker.makeSubstitutionGoal(drvPath)); - state = &DerivationGoal::haveDerivation; + state = &DerivationGoal::loadDerivation; } -void DerivationGoal::haveDerivation() +void DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { printMsg(lvlError, format("cannot build missing derivation ‘%1%’") % drvPath); - amDone(ecFailed); + done(BuildResult::MiscFailure); return; } @@ -990,31 +1028,43 @@ void DerivationGoal::haveDerivation() assert(worker.store.isValidPath(drvPath)); /* Get the derivation. */ - drv = derivationFromPath(worker.store, drvPath); + drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath))); + + haveDerivation(); +} + + +void DerivationGoal::haveDerivation() +{ + trace("have derivation"); - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.addTempRoot(i->second.path); + for (auto & i : drv->outputs) + worker.store.addTempRoot(i.second.path); /* Check what outputs paths are not already valid. */ PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0 && buildMode == bmNormal) { - amDone(ecSuccess); + done(BuildResult::AlreadyValid); return; } - /* Check whether any output previously failed to build. If so, - don't bother. */ - foreach (PathSet::iterator, i, invalidOutputs) - if (pathFailed(*i)) return; + /* Reject doing a hash build of anything other than a fixed-output + derivation. */ + if (buildMode == bmHash) { + if (drv->outputs.size() != 1 || + drv->outputs.find("out") == drv->outputs.end() || + drv->outputs["out"].hashAlgo == "") + throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); + } /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && substitutesAllowed(drv)) - foreach (PathSet::iterator, i, invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair)); + if (settings.useSubstitutes && drv->substitutesAllowed()) + for (auto & i : invalidOutputs) + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1045,7 +1095,7 @@ void DerivationGoal::outputsSubstituted() unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); if (buildMode == bmNormal && nrInvalid == 0) { - amDone(ecSuccess); + done(BuildResult::Substituted); return; } if (buildMode == bmRepair && nrInvalid == 0) { @@ -1063,11 +1113,17 @@ void DerivationGoal::outputsSubstituted() wantedOutputs = PathSet(); /* The inputs must be built before we can build this goal. */ - foreach (DerivationInputs::iterator, i, drv.inputDrvs) - addWaitee(worker.makeDerivationGoal(i->first, i->second, buildMode == bmRepair ? bmRepair : bmNormal)); + if (useDerivation) + for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - foreach (PathSet::iterator, i, drv.inputSrcs) - addWaitee(worker.makeSubstitutionGoal(*i)); + for (auto & i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) continue; + if (!settings.useSubstitutes) + throw Error(format("dependency of ‘%1%’ of ‘%2%’ does not exist, and substitution is disabled") + % i % drvPath); + addWaitee(worker.makeSubstitutionGoal(i)); + } if (waitees.empty()) /* to prevent hang (no wake-up event) */ inputsRealised(); @@ -1085,40 +1141,42 @@ void DerivationGoal::repairClosure() /* Get the output closure. */ PathSet outputClosure; - foreach (DerivationOutputs::iterator, i, drv.outputs) - computeFSClosure(worker.store, i->second.path, outputClosure); + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second.path, outputClosure); + } /* Filter out our own outputs (which we have already checked). */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - outputClosure.erase(i->second.path); + for (auto & i : drv->outputs) + outputClosure.erase(i.second.path); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output closure. */ PathSet inputClosure; - computeFSClosure(worker.store, drvPath, inputClosure); + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); std::map<Path, Path> outputsToDrv; - foreach (PathSet::iterator, i, inputClosure) - if (isDerivation(*i)) { - Derivation drv = derivationFromPath(worker.store, *i); - foreach (DerivationOutputs::iterator, j, drv.outputs) - outputsToDrv[j->second.path] = *i; + for (auto & i : inputClosure) + if (isDerivation(i)) { + Derivation drv = worker.store.derivationFromPath(i); + for (auto & j : drv.outputs) + outputsToDrv[j.second.path] = i; } /* Check each path (slow!). */ PathSet broken; - foreach (PathSet::iterator, i, outputClosure) { - if (worker.store.pathContentsGood(*i)) continue; - printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % *i % drvPath); - Path drvPath2 = outputsToDrv[*i]; + for (auto & i : outputClosure) { + if (worker.pathContentsGood(i)) continue; + printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); + Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") - addWaitee(worker.makeSubstitutionGoal(*i, true)); + addWaitee(worker.makeSubstitutionGoal(i, true)); else addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); } if (waitees.empty()) { - amDone(ecSuccess); + done(BuildResult::AlreadyValid); return; } @@ -1131,7 +1189,7 @@ void DerivationGoal::closureRepaired() trace("closure repaired"); if (nrFailed > 0) throw Error(format("some paths in the output closure of derivation ‘%1%’ could not be repaired") % drvPath); - amDone(ecSuccess); + done(BuildResult::AlreadyValid); } @@ -1140,10 +1198,12 @@ void DerivationGoal::inputsRealised() trace("all inputs realised"); if (nrFailed != 0) { + if (!useDerivation) + throw Error(format("some dependencies of ‘%1%’ are missing") % drvPath); printMsg(lvlError, format("cannot build derivation ‘%1%’: %2% dependencies couldn't be built") % drvPath % nrFailed); - amDone(ecFailed); + done(BuildResult::DependencyFailed); return; } @@ -1156,32 +1216,33 @@ void DerivationGoal::inputsRealised() running the build hook. */ /* The outputs are referenceable paths. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - debug(format("building path ‘%1%’") % i->second.path); - allPaths.insert(i->second.path); + for (auto & i : drv->outputs) { + debug(format("building path ‘%1%’") % i.second.path); + allPaths.insert(i.second.path); } /* Determine the full set of input paths. */ /* First, the input derivations. */ - foreach (DerivationInputs::iterator, i, drv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `*i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(i->first)); - Derivation inDrv = derivationFromPath(worker.store, i->first); - foreach (StringSet::iterator, j, i->second) - if (inDrv.outputs.find(*j) != inDrv.outputs.end()) - computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); - else - throw Error( - format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") - % drvPath % *j % i->first); - } + if (useDerivation) + for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i.first)); + Derivation inDrv = worker.store.derivationFromPath(i.first); + for (auto & j : i.second) + if (inDrv.outputs.find(j) != inDrv.outputs.end()) + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + else + throw Error( + format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") + % drvPath % j % i.first); + } /* Second, the input sources. */ - foreach (PathSet::iterator, i, drv.inputSrcs) - computeFSClosure(worker.store, *i, inputPaths); + for (auto & i : drv->inputSrcs) + worker.store.computeFSClosure(i, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1189,8 +1250,12 @@ void DerivationGoal::inputsRealised() /* Is this a fixed-output derivation? */ fixedOutput = true; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (i->second.hash == "") fixedOutput = false; + for (auto & i : drv->outputs) + if (i.second.hash == "") fixedOutput = false; + + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1200,35 +1265,6 @@ void DerivationGoal::inputsRealised() } -static bool canBuildLocally(const string & platform) -{ - return platform == settings.thisSystem -#if __linux__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") -#endif - ; -} - - -static string get(const StringPairs & map, const string & key, const string & def = "") -{ - StringPairs::const_iterator i = map.find(key); - return i == map.end() ? def : i->second; -} - - -bool willBuildLocally(const Derivation & drv) -{ - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); -} - - -bool substitutesAllowed(const Derivation & drv) -{ - return get(drv.env, "allowSubstitutes", "1") == "1"; -} - - void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1238,10 +1274,10 @@ void DerivationGoal::tryToBuild() (It can't happen between here and the lockPaths() call below because we're not allowing multi-threading.) If so, put this goal to sleep until another goal finishes, then try again. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (pathIsLockedByMe(i->second.path)) { + for (auto & i : drv->outputs) + if (pathIsLockedByMe(i.second.path)) { debug(format("putting derivation ‘%1%’ to sleep because ‘%2%’ is locked by another goal") - % drvPath % i->second.path); + % drvPath % i.second.path); worker.waitForAnyGoal(shared_from_this()); return; } @@ -1251,7 +1287,7 @@ void DerivationGoal::tryToBuild() can't acquire the lock, then continue; hopefully some other goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ - if (!outputLocks.lockPaths(outputPaths(drv), "", false)) { + if (!outputLocks.lockPaths(drv->outputPaths(), "", false)) { worker.waitForAWhile(shared_from_this()); return; } @@ -1264,38 +1300,30 @@ void DerivationGoal::tryToBuild() now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); - assert(buildMode != bmCheck || validPaths.size() == drv.outputs.size()); - if (buildMode != bmCheck && validPaths.size() == drv.outputs.size()) { + if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { debug(format("skipping build of derivation ‘%1%’, someone beat us to it") % drvPath); outputLocks.setDeletion(true); - amDone(ecSuccess); + done(BuildResult::AlreadyValid); return; } - missingPaths = outputPaths(drv); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) - foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i); + for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; + for (auto & i : drv->outputs) { + Path path = i.second.path; if (worker.store.isValidPath(path)) continue; - if (!pathExists(path)) continue; debug(format("removing invalid path ‘%1%’") % path); deletePath(path); } - /* Check again whether any output previously failed to build, - because some other process may have tried and failed before we - acquired the lock. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (pathFailed(i->second.path)) return; - /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || willBuildLocally(drv); + bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); /* Is the build hook willing to accept this job? */ if (!buildLocally) { @@ -1336,11 +1364,8 @@ void DerivationGoal::tryToBuild() printMsg(lvlError, e.msg()); outputLocks.unlock(); buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 0 % e.msg()); worker.permanentFailure = true; - amDone(ecFailed); + done(BuildResult::InputRejected, e.msg()); return; } @@ -1361,11 +1386,13 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) throw SysError(format("moving ‘%1%’ to ‘%2%’") % tmpPath % storePath); - if (pathExists(oldPath)) - deletePath(oldPath); + deletePath(oldPath); } +MakeError(NotDeterministic, BuildError) + + void DerivationGoal::buildDone() { trace("build done"); @@ -1374,22 +1401,14 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe --- just don't do that :-) */ - int status; - pid_t savedPid; - if (hook) { - savedPid = hook->pid; - status = hook->pid.wait(true); - } else { - /* !!! this could block! security problem! solution: kill the - child */ - savedPid = pid; - status = pid.wait(true); - } + /* !!! this could block! security problem! solution: kill the + child */ + int status = hook ? hook->pid.wait(true) : pid.wait(true); debug(format("builder process for ‘%1%’ finished") % drvPath); /* So the child is gone now. */ - worker.childTerminated(savedPid); + worker.childTerminated(shared_from_this()); /* Close the read side of the logger pipe. */ if (hook) { @@ -1436,15 +1455,23 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - foreach (PathSet::iterator, i, missingPaths) - if (pathExists(chrootRootDir + *i)) - rename((chrootRootDir + *i).c_str(), i->c_str()); + for (auto & i : missingPaths) + if (pathExists(chrootRootDir + i)) + rename((chrootRootDir + i).c_str(), i.c_str()); + + std::string msg = (format("builder for ‘%1%’ %2%") + % drvPath % statusToString(status)).str(); + + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto & line : logTail) + msg += "\n " + line; + } if (diskFull) - printMsg(lvlError, "note: build failure may have been caused by lack of free disk space"); + msg += "\nnote: build failure may have been caused by lack of free disk space"; - throw BuildError(format("builder for ‘%1%’ %2%") - % drvPath % statusToString(status)); + throw BuildError(msg); } /* Compute the FS closure of the outputs and register them as @@ -1452,19 +1479,28 @@ void DerivationGoal::buildDone() registerOutputs(); if (buildMode == bmCheck) { - amDone(ecSuccess); + done(BuildResult::Built); return; } /* Delete unused redirected outputs (when doing hash rewriting). */ - foreach (RedirectedOutputs::iterator, i, redirectedOutputs) - if (pathExists(i->second)) deletePath(i->second); + for (auto & i : redirectedOutputs) + deletePath(i.second); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ deleteTmpDir(true); + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + buildUser.release(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old @@ -1478,67 +1514,50 @@ void DerivationGoal::buildDone() outputLocks.unlock(); buildUser.release(); - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); - worker.timedOut = true; - } + BuildResult::Status st = BuildResult::MiscFailure; + + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) + st = BuildResult::TimedOut; else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") - % drvPath % status % e.msg()); } else { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 1 % e.msg()); - worker.permanentFailure = !fixedOutput && !diskFull; - - /* Register the outputs of this build as "failed" so we - won't try to build them again (negative caching). - However, don't do this for fixed-output derivations, - since they're likely to fail for transient reasons - (e.g., fetchurl not being able to access the network). - Hook errors (like communication problems with the - remote machine) shouldn't be cached either. */ - if (settings.cacheFailure && !fixedOutput && !diskFull) - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.registerFailedPath(i->second.path); + st = + dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + fixedOutput || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; } - amDone(ecFailed); + done(st, e.msg()); return; } /* Release the build user, if applicable. */ buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); - - amDone(ecSuccess); + done(BuildResult::Built); } HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; + if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; if (!worker.hook) - worker.hook = std::shared_ptr<HookInstance>(new HookInstance); + worker.hook = std::make_shared<HookInstance>(); /* Tell the hook about system features (beyond the system type) required from the build machine. (The hook could parse the drv file itself, but this is easier.) */ - Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures")); - foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ + Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures")); + for (auto & i : features) checkStoreName(i); /* !!! abuse */ /* Send the request to the hook. */ writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv.platform % drvPath % concatStringsSep(",", features)).str()); + % drv->platform % drvPath % concatStringsSep(",", features)).str()); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ @@ -1572,16 +1591,16 @@ HookReply DerivationGoal::tryBuildHook() list it since the remote system *probably* already has it.) */ PathSet allInputs; allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(worker.store, drvPath, allInputs); + worker.store.computeFSClosure(drvPath, allInputs); string s; - foreach (PathSet::iterator, i, allInputs) { s += *i; s += ' '; } + for (auto & i : allInputs) { s += i; s += ' '; } writeLine(hook->toHook.writeSide, s); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ s = ""; - foreach (PathSet::iterator, i, missingPaths) { s += *i; s += ' '; } + for (auto & i : missingPaths) { s += i; s += ' '; } writeLine(hook->toHook.writeSide, s); hook->toHook.writeSide.close(); @@ -1592,11 +1611,7 @@ HookReply DerivationGoal::tryBuildHook() set<int> fds; fds.insert(hook->fromHook.readSide); fds.insert(hook->builderOut.readSide); - worker.childStarted(shared_from_this(), hook->pid, fds, false, false); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); + worker.childStarted(shared_from_this(), fds, false, false); return rpAccept; } @@ -1618,21 +1633,55 @@ int childEntry(void * arg) void DerivationGoal::startBuilder() { - startNest(nest, lvlInfo, format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - "building path(s) %1%") % showPaths(missingPaths)); + auto f = format( + buildMode == bmRepair ? "repairing path(s) %1%" : + buildMode == bmCheck ? "checking path(s) %1%" : + nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : + "building path(s) %1%"); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + printMsg(lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ - if (!canBuildLocally(drv.platform)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform); + if (!drv->canBuildLocally()) { throw Error( format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’") - % drv.platform % settings.thisSystem % drvPath); + % drv->platform % settings.thisSystem % drvPath); + } + +#if __APPLE__ + additionalSandboxProfile = get(drv->env, "__sandboxProfile"); +#endif + + /* Are we doing a chroot build? Note that fixed-output + derivations are never done in a chroot, mainly so that + functions like fetchurl (which needs a proper /etc/resolv.conf) + work properly. Purity checking for fixed-output derivations + is somewhat pointless anyway. */ + { + string x = settings.get("build-use-sandbox", + /* deprecated alias */ + settings.get("build-use-chroot", string("false"))); + if (x != "true" && x != "false" && x != "relaxed") + throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); + if (x == "true") { + if (get(drv->env, "__noChroot") == "1") + throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " + "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error(format("derivation ‘%1%’ specifies a sandbox profile, " + "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath); +#endif + useChroot = true; + } + else if (x == "false") + useChroot = false; + else if (x == "relaxed") + useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; } /* Construct the environment passed to the builder. */ + env.clear(); /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when PATH is not set. We don't want this, so we fill it in with some dummy @@ -1659,38 +1708,44 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700); + auto drvName = storePathToName(drvPath); + tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + + /* In a sandbox, for determinism, always use the same temporary + directory. */ + tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; /* Add all bindings specified in the derivation via the environments, except those listed in the passAsFile attribute. Those are passed as file names pointing to temporary files containing the contents. */ PathSet filesToChown; - StringSet passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile")); + StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); int fileNr = 0; - for (auto & i : drv.env) { + for (auto & i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - Path p = tmpDir + "/.attr-" + int2String(fileNr++); + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; writeFile(p, i.second); filesToChown.insert(p); - env[i.first + "Path"] = p; + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } } /* For convenience, set an environment pointing to the top build directory. */ - env["NIX_BUILD_TOP"] = tmpDir; + env["NIX_BUILD_TOP"] = tmpDirInSandbox; /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; /* Explicitly set PWD to prevent problems with chroot builds. In particular, dietlibc cannot figure out the cwd because the inode of the current directory doesn't appear in .. (because getdents returns the inode of the mount point). */ - env["PWD"] = tmpDir; + env["PWD"] = tmpDirInSandbox; /* Compatibility hack with Nix <= 0.7: if this is a fixed-output derivation, tell the builder, so that for instance `fetchurl' @@ -1708,8 +1763,8 @@ void DerivationGoal::startBuilder() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars")); - foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); + Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); + for (auto & i : varNames) env[i] = getEnv(i); } /* The `exportReferencesGraph' feature allows the references graph @@ -1719,7 +1774,7 @@ void DerivationGoal::startBuilder() temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - string s = get(drv.env, "exportReferencesGraph"); + string s = get(drv->env, "exportReferencesGraph"); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s); @@ -1742,14 +1797,14 @@ void DerivationGoal::startBuilder() like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ PathSet paths, paths2; - computeFSClosure(worker.store, storePath, paths); + worker.store.computeFSClosure(storePath, paths); paths2 = paths; - foreach (PathSet::iterator, j, paths2) { - if (isDerivation(*j)) { - Derivation drv = derivationFromPath(worker.store, *j); - foreach (DerivationOutputs::iterator, k, drv.outputs) - computeFSClosure(worker.store, k->second.path, paths); + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputs) + worker.store.computeFSClosure(k.second.path, paths); } } @@ -1779,40 +1834,27 @@ void DerivationGoal::startBuilder() } - /* Are we doing a chroot build? Note that fixed-output - derivations are never done in a chroot, mainly so that - functions like fetchurl (which needs a proper /etc/resolv.conf) - work properly. Purity checking for fixed-output derivations - is somewhat pointless anyway. */ - { - string x = settings.get("build-use-chroot", string("false")); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-chroot’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { - if (get(drv.env, "__noChroot") == "1") - throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, but that's not allowed when ‘build-use-chroot’ is ‘true’") % drvPath); - useChroot = true; - } - else if (x == "false") - useChroot = false; - else if (x == "relaxed") - useChroot = !fixedOutput && get(drv.env, "__noChroot") != "1"; - } - if (useChroot) { string defaultChrootDirs; -#if CHROOT_ENABLED +#if __linux__ if (isInStore(BASH_PATH)) defaultChrootDirs = "/bin/sh=" BASH_PATH; #endif /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", defaultChrootDirs)); - PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string(""))); + PathSet dirs = tokenizeString<StringSet>( + settings.get("build-sandbox-paths", + /* deprecated alias with lower priority */ + settings.get("build-chroot-dirs", defaultChrootDirs))); + PathSet dirs2 = tokenizeString<StringSet>( + settings.get("build-extra-chroot-dirs", + settings.get("build-extra-sandbox-paths", string("")))); dirs.insert(dirs2.begin(), dirs2.end()); + dirsInChroot.clear(); + for (auto & i : dirs) { size_t p = i.find('='); if (p == string::npos) @@ -1820,13 +1862,13 @@ void DerivationGoal::startBuilder() else dirsInChroot[string(i, 0, p)] = string(i, p + 1); } - dirsInChroot[tmpDir] = tmpDir; + dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ PathSet closure; for (auto & i : dirsInChroot) if (isInStore(i.second)) - computeFSClosure(worker.store, toStorePath(i.second), closure); + worker.store.computeFSClosure(toStorePath(i.second), closure); for (auto & i : closure) dirsInChroot[i] = i; @@ -1834,7 +1876,7 @@ void DerivationGoal::startBuilder() PathSet allowedPaths = tokenizeString<StringSet>(allowed); /* This works like the above, except on a per-derivation level */ - Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps")); + Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); for (auto & i : impurePaths) { bool found = false; @@ -1851,21 +1893,21 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); dirsInChroot[i] = i; } -#if CHROOT_ENABLED +#if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ chrootRootDir = drvPath + ".chroot"; - if (pathExists(chrootRootDir)) deletePath(chrootRootDir); + deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ - autoDelChroot = std::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); + autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); printMsg(lvlChatty, format("setting up chroot environment in ‘%1%’") % chrootRootDir); @@ -1918,44 +1960,42 @@ void DerivationGoal::startBuilder() if (chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1) throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir); - foreach (PathSet::iterator, i, inputPaths) { + for (auto & i : inputPaths) { struct stat st; - if (lstat(i->c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % *i); + if (lstat(i.c_str(), &st)) + throw SysError(format("getting attributes of path ‘%1%’") % i); if (S_ISDIR(st.st_mode)) - dirsInChroot[*i] = *i; + dirsInChroot[i] = i; else { - Path p = chrootRootDir + *i; - if (link(i->c_str(), p.c_str()) == -1) { + Path p = chrootRootDir + i; + if (link(i.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a `nix-store --optimise'. */ if (errno != EMLINK) - throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % *i); + throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % i); StringSink sink; - dumpPath(*i, sink); - StringSource source(sink.s); + dumpPath(i, sink); + StringSource source(*sink.s); restorePath(p, source); } - - regularInputPaths.insert(*i); } } - /* If we're repairing or checking, it's possible that we're + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're rebuilding a path that is in settings.dirsInChroot (typically the dependencies of /bin/sh). Throw them out. */ - if (buildMode != bmNormal) - foreach (DerivationOutputs::iterator, i, drv.outputs) - dirsInChroot.erase(i->second.path); + for (auto & i : drv->outputs) + dirsInChroot.erase(i.second.path); -#elif SANDBOX_ENABLED +#elif __APPLE__ /* We don't really have any parent prep work to do (yet?) All work happens in the child, instead. */ #else - throw Error("chroot builds are not supported on this platform"); + throw Error("sandboxing builds is not supported on this platform"); #endif } @@ -1974,16 +2014,16 @@ void DerivationGoal::startBuilder() contents of the new outputs to replace the dummy strings with the actual hashes. */ if (validPaths.size() > 0) - foreach (PathSet::iterator, i, validPaths) - addHashRewrite(*i); + for (auto & i : validPaths) + addHashRewrite(i); /* If we're repairing, then we don't want to delete the corrupt outputs in advance. So rewrite them as well. */ if (buildMode == bmRepair) - foreach (PathSet::iterator, i, missingPaths) - if (worker.store.isValidPath(*i) && pathExists(*i)) { - addHashRewrite(*i); - redirectedBadOutputs.insert(*i); + for (auto & i : missingPaths) + if (worker.store.isValidPath(i) && pathExists(i)) { + addHashRewrite(i); + redirectedBadOutputs.insert(i); } } @@ -2001,10 +2041,10 @@ void DerivationGoal::startBuilder() auto lastPos = std::string::size_type{0}; for (auto nlPos = lines.find('\n'); nlPos != string::npos; nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos}; + auto line = std::string{lines, lastPos, nlPos - lastPos}; lastPos = nlPos + 1; if (state == stBegin) { - if (line == "extra-chroot-dirs") { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { throw Error(format("unknown pre-build hook command ‘%1%’") @@ -2025,7 +2065,7 @@ void DerivationGoal::startBuilder() } /* Run the builder. */ - printMsg(lvlChatty, format("executing builder ‘%1%’") % drv.builder); + printMsg(lvlChatty, format("executing builder ‘%1%’") % drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2034,7 +2074,7 @@ void DerivationGoal::startBuilder() builderOut.create(); /* Fork a child to build the package. */ -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2072,16 +2112,19 @@ void DerivationGoal::startBuilder() ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { - char stack[32 * 1024]; + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; - pid_t child = clone(childEntry, stack + sizeof(stack) - 8, flags, this); + pid_t child = clone(childEntry, stack + stackSize, flags, this); if (child == -1 && errno == EINVAL) /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + sizeof(stack) - 8, flags & ~CLONE_NEWPID, this); + child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); if (child == -1) throw SysError("cloning builder process"); - writeFull(builderOut.writeSide, int2String(child) + "\n"); + writeFull(builderOut.writeSide, std::to_string(child) + "\n"); _exit(0); }, options); if (helper.wait(true) != 0) @@ -2093,7 +2136,7 @@ void DerivationGoal::startBuilder() #endif { ProcessOptions options; - options.allowVfork = !buildUser.enabled(); + options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); }, options); @@ -2102,16 +2145,16 @@ void DerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); builderOut.writeSide.close(); - worker.childStarted(shared_from_this(), pid, - singleton<set<int> >(builderOut.readSide), true, true); + worker.childStarted(shared_from_this(), {builderOut.readSide}, true, true); /* Check if setting up the build environment failed. */ - string msg = readLine(builderOut.readSide); - if (!msg.empty()) throw Error(msg); - - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); + while (true) { + string msg = readLine(builderOut.readSide); + if (string(msg, 0, 1) == "\1") { + if (msg.size() == 1) break; + throw Error(string(msg, 1)); + } + printMsg(lvlDebug, msg); } } @@ -2125,7 +2168,7 @@ void DerivationGoal::runChild() commonChildInit(builderOut); -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Initialise the loopback interface. */ @@ -2157,8 +2200,8 @@ void DerivationGoal::runChild() local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); - foreach (Strings::iterator, i, mounts) { - vector<string> fields = tokenizeString<vector<string> >(*i, " "); + for (auto & i : mounts) { + vector<string> fields = tokenizeString<vector<string> >(i, " "); string fs = decodeOctalEscaped(fields.at(4)); if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) throw SysError(format("unable to make filesystem ‘%1%’ private") % fs); @@ -2206,10 +2249,10 @@ void DerivationGoal::runChild() /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - foreach (DirsInChroot::iterator, i, dirsInChroot) { + for (auto & i : dirsInChroot) { struct stat st; - Path source = i->second; - Path target = chrootRootDir + i->first; + Path source = i.second; + Path target = chrootRootDir + i.first; if (source == "/proc") continue; // backwards compatibility debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target); if (stat(source.c_str(), &st) == -1) @@ -2258,10 +2301,8 @@ void DerivationGoal::runChild() if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) if (pivot_root(".", "real-root") == -1) throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root")); -#undef pivot_root if (chroot(".") == -1) throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir); @@ -2274,7 +2315,7 @@ void DerivationGoal::runChild() } #endif - if (chdir(tmpDir.c_str()) == -1) + if (chdir(tmpDirInSandbox.c_str()) == -1) throw SysError(format("changing into ‘%1%’") % tmpDir); /* Close all other file descriptors. */ @@ -2285,7 +2326,7 @@ void DerivationGoal::runChild() i686-linux build on an x86_64-linux machine. */ struct utsname utsbuf; uname(&utsbuf); - if (drv.platform == "i686-linux" && + if (drv->platform == "i686-linux" && (settings.thisSystem == "x86_64-linux" || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { if (personality(PER_LINUX32) == -1) @@ -2294,7 +2335,7 @@ void DerivationGoal::runChild() /* Impersonate a Linux 2.6 machine to get some determinism in builds that depend on the kernel version. */ - if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { int cur = personality(0xffffffff); if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); } @@ -2305,10 +2346,16 @@ void DerivationGoal::runChild() if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); #endif + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + /* Fill in the environment. */ Strings envStrs; - foreach (Environment::const_iterator, i, env) - envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); + for (auto & i : env) + envStrs.push_back(rewriteHashes(i.first + "=" + i.second, rewritesToTmp)); /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop all root @@ -2319,7 +2366,8 @@ void DerivationGoal::runChild() if (buildUser.enabled()) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (setgroups(buildUser.getSupplementaryGIDs().size(), + if (!buildUser.getSupplementaryGIDs().empty() && + setgroups(buildUser.getSupplementaryGIDs().size(), buildUser.getSupplementaryGIDs().data()) == -1) throw SysError("cannot set supplementary groups of build user"); @@ -2340,7 +2388,10 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (useChroot && SANDBOX_ENABLED) { + if (drv->isBuiltin()) { + ; +#if __APPLE__ + } else if (useChroot) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ PathSet ancestry; @@ -2367,8 +2418,7 @@ void DerivationGoal::runChild() for (auto & i : inputPaths) dirsInChroot[i] = i; - - /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */ + /* This has to appear before import statements */ sandboxProfile += "(version 1)\n"; /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2378,15 +2428,6 @@ void DerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(allow file-read* file-write-data (literal \"/dev/null\"))\n"; - - sandboxProfile += "(allow file-read-metadata\n" - "\t(literal \"/var\")\n" - "\t(literal \"/tmp\")\n" - "\t(literal \"/etc\")\n" - "\t(literal \"/etc/nix\")\n" - "\t(literal \"/etc/nix/nix.conf\"))\n"; - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); @@ -2394,20 +2435,6 @@ void DerivationGoal::runChild() /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - /* This is where our temp folders are and where most of the building will happen, so we want rwx on it. */ - sandboxProfile += (format("(allow file-read* file-write* process-exec (subpath \"%1%\") (subpath \"/private/tmp\"))\n") % globalTmpDir).str(); - - sandboxProfile += "(allow process-fork)\n"; - sandboxProfile += "(allow sysctl-read)\n"; - sandboxProfile += "(allow signal (target same-sandbox))\n"; - - /* Enables getpwuid (used by git and others) */ - sandboxProfile += "(allow mach-lookup (global-name \"com.apple.system.notification_center\") (global-name \"com.apple.system.opendirectoryd.libinfo\"))\n"; - - /* Allow local networking operations, mostly because lots of test suites use it and it seems mostly harmless */ - sandboxProfile += "(allow network* (local ip) (remote unix-socket))"; - - /* Our rwx outputs */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : missingPaths) { @@ -2416,15 +2443,15 @@ void DerivationGoal::runChild() sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) - Note that the sandbox profile allows file-write* even though it isn't seemingly necessary. First of all, nix's standard user permissioning - mechanism still prevents builders from writing to input directories, so no security/purity is lost. The reason we allow file-write* is that - denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since - it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of - these is the entire GHC Haskell ecosystem. */ + + without file-write* allowed, access() incorrectly returns EPERM + */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second) - throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin")); + throw Error(format( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") + % i.first % i.second); string path = i.first; struct stat st; @@ -2437,46 +2464,66 @@ void DerivationGoal::runChild() } sandboxProfile += ")\n"; - /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that, - you open up the entire filesystem because you end up with (subpath "/") */ - sandboxProfile += "(allow file-read-metadata\n"; + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); } sandboxProfile += ")\n"; + sandboxProfile += additionalSandboxProfile; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + Path sandboxFile = drvPath + ".sb"; + deletePath(sandboxFile); + autoDelSandbox.reset(sandboxFile, false); + + writeFile(sandboxFile, sandboxProfile); + builder = "/usr/bin/sandbox-exec"; args.push_back("sandbox-exec"); - args.push_back("-p"); - args.push_back(sandboxProfile); - args.push_back(drv.builder); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back(drv->builder); +#endif } else { - builder = drv.builder.c_str(); - string builderBasename = baseNameOf(drv.builder); + builder = drv->builder.c_str(); + string builderBasename = baseNameOf(drv->builder); args.push_back(builderBasename); } - foreach (Strings::iterator, i, drv.args) - args.push_back(rewriteHashes(*i, rewritesToTmp)); + for (auto & i : drv->args) + args.push_back(rewriteHashes(i, rewritesToTmp)); restoreSIGPIPE(); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, "\n"); + writeFull(STDERR_FILENO, string("\1\n")); - /* This needs to be after that fateful '\n', and I didn't want to duplicate code */ - if (useChroot && SANDBOX_ENABLED) { - printMsg(lvlDebug, "Generated sandbox profile:"); - printMsg(lvlDebug, sandboxProfile); + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(*drv); + else + throw Error(format("unsupported builtin function ‘%1%’") % string(drv->builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } } - /* Execute the program. This should not return. */ execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing ‘%1%’") % drv.builder); + throw SysError(format("executing ‘%1%’") % drv->builder); } catch (std::exception & e) { - writeFull(STDERR_FILENO, "while setting up the build environment: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); _exit(1); } } @@ -2485,18 +2532,17 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) +PathSet parseReferenceSpecifiers(const BasicDerivation & drv, string attr) { PathSet result; Paths paths = tokenizeString<Paths>(attr); - foreach (Strings::iterator, i, paths) { - if (isStorePath(*i)) - result.insert(*i); - else if (drv.outputs.find(*i) != drv.outputs.end()) - result.insert(drv.outputs.find(*i)->second.path); + for (auto & i : paths) { + if (isStorePath(i)) + result.insert(i); + else if (drv.outputs.find(i) != drv.outputs.end()) + result.insert(drv.outputs.find(i)->second.path); else throw BuildError( - format("derivation contains an illegal reference specifier ‘%1%’") - % *i); + format("derivation contains an illegal reference specifier ‘%1%’") % i); } return result; } @@ -2509,18 +2555,25 @@ void DerivationGoal::registerOutputs() to do anything here. */ if (hook) { bool allValid = true; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!worker.store.isValidPath(i->second.path)) allValid = false; + for (auto & i : drv->outputs) + if (!worker.store.isValidPath(i.second.path)) allValid = false; if (allValid) return; } ValidPathInfos infos; + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = "-check"; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; + for (auto & i : drv->outputs) { + Path path = i.second.path; if (missingPaths.find(path) == missingPaths.end()) continue; Path actualPath = path; @@ -2532,7 +2585,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), path.c_str()) == -1) - throw SysError(format("moving build output ‘%1%’ from the chroot to the Nix store") % path); + throw SysError(format("moving build output ‘%1%’ from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = path; } else { @@ -2541,7 +2594,7 @@ void DerivationGoal::registerOutputs() && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && pathExists(redirected)) replaceValidPath(path, redirected); - if (buildMode == bmCheck) + if (buildMode == bmCheck && redirected != "") actualPath = redirected; } @@ -2578,23 +2631,22 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); deletePath(actualPath); - sink.s = rewriteHashes(sink.s, rewritesFromTmp); - StringSource source(sink.s); + sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp)); + StringSource source(*sink.s); restorePath(actualPath, source); rewritten = true; } - startNest(nest, lvlTalkative, - format("scanning for references inside ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - if (i->second.hash != "") { + if (i.second.hash != "") { bool recursive; HashType ht; Hash h; - i->second.parseHashInfo(recursive, ht, h); + i.second.parseHashInfo(recursive, ht, h); if (!recursive) { /* The output path should be a regular file without @@ -2604,12 +2656,28 @@ void DerivationGoal::registerOutputs() format("output path ‘%1%’ should be a non-executable regular file") % path); } - /* Check the hash. */ + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); - if (h != h2) - throw BuildError( - format("output path ‘%1%’ should have %2% hash ‘%3%’, instead has ‘%4%’") - % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); + if (buildMode == bmHash) { + Path dest = makeFixedOutputPath(recursive, ht, h2, drv->env["name"]); + printMsg(lvlError, format("build produced path ‘%1%’ with %2% hash ‘%3%’") + % dest % printHashType(ht) % printHash16or32(h2)); + if (worker.store.isValidPath(dest)) + return; + if (actualPath != dest) { + PathLocks outputLocks({dest}); + deletePath(dest); + if (rename(actualPath.c_str(), dest.c_str()) == -1) + throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest); + } + path = actualPath = dest; + } else { + if (h != h2) + throw BuildError( + format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") + % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); + } } /* Get rid of all weird permissions. This also checks that @@ -2625,27 +2693,47 @@ void DerivationGoal::registerOutputs() PathSet references = scanForReferences(actualPath, allPaths, hash); if (buildMode == bmCheck) { - ValidPathInfo info = worker.store.queryPathInfo(path); - if (hash.first != info.hash) - throw Error(format("derivation ‘%1%’ may not be deterministic: hash mismatch in output ‘%2%’") % drvPath % path); + if (!worker.store.isValidPath(path)) continue; + auto info = *worker.store.queryPathInfo(path); + if (hash.first != info.narHash) { + if (settings.keepFailed) { + Path dst = path + checkSuffix; + deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst); + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’") + % drvPath % path % dst); + } else + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") + % drvPath % path); + } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + continue; } /* For debugging, print out the referenced and unreferenced paths. */ - foreach (PathSet::iterator, i, inputPaths) { - PathSet::iterator j = references.find(*i); + for (auto & i : inputPaths) { + PathSet::iterator j = references.find(i); if (j == references.end()) - debug(format("unreferenced input: ‘%1%’") % *i); + debug(format("unreferenced input: ‘%1%’") % i); else - debug(format("referenced input: ‘%1%’") % *i); + debug(format("referenced input: ‘%1%’") % i); } /* Enforce `allowedReferences' and friends. */ auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - if (drv.env.find(attrName) == drv.env.end()) return; + if (drv->env.find(attrName) == drv->env.end()) return; - PathSet spec = parseReferenceSpecifiers(drv, get(drv.env, attrName)); + PathSet spec = parseReferenceSpecifiers(*drv, get(drv->env, attrName)); PathSet used; if (recursive) { @@ -2653,18 +2741,29 @@ void DerivationGoal::registerOutputs() for (auto & i : references) /* Don't call computeFSClosure on ourselves. */ if (actualPath != i) - computeFSClosure(worker.store, i, used); + worker.store.computeFSClosure(i, used); } else used = references; + PathSet badPaths; + for (auto & i : used) if (allowed) { if (spec.find(i) == spec.end()) - throw BuildError(format("output (‘%1%’) is not allowed to refer to path ‘%2%’") % actualPath % i); + badPaths.insert(i); } else { if (spec.find(i) != spec.end()) - throw BuildError(format("output (‘%1%’) is not allowed to refer to path ‘%2%’") % actualPath % i); + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n\t"; + badPathsStr += i; } + throw BuildError(format("output ‘%1%’ is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); + } }; checkRefs("allowedReferences", true, false); @@ -2672,21 +2771,63 @@ void DerivationGoal::registerOutputs() checkRefs("disallowedReferences", false, false); checkRefs("disallowedRequisites", false, true); - worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + if (curRound == nrRounds) { + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.markContentsGood(path); + } ValidPathInfo info; info.path = path; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; info.references = references; info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + infos.push_back(info); } if (buildMode == bmCheck) return; + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + Path prev = i->path + checkSuffix; + if (pathExists(prev)) + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") + % i->path % drvPath % prev); + else + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from previous round") + % i->path % drvPath); + } + assert(false); // shouldn't happen + } + + if (settings.keepFailed) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + if (curRound < nrRounds) { + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); + } + } + + } + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -2709,46 +2850,31 @@ Path DerivationGoal::openLogFile() Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); createDirs(dir); - if (settings.compressLog) { + Path logFileName = (format("%1%/%2%%3%") + % dir + % string(baseName, 2) + % (settings.compressLog ? ".bz2" : "")).str(); - Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); - AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fd == -1) throw SysError(format("creating log file ‘%1%’") % logFileName); - closeOnExec(fd); + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + if (fdLogFile == -1) throw SysError(format("creating log file ‘%1%’") % logFileName); - if (!(fLogFile = fdopen(fd.borrow(), "w"))) - throw SysError(format("opening file ‘%1%’") % logFileName); + logFileSink = std::make_shared<FdSink>(fdLogFile); - int err; - if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) - throw Error(format("cannot open compressed log file ‘%1%’") % logFileName); - - return logFileName; + if (settings.compressLog) + logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink)); + else + logSink = logFileSink; - } else { - Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fdLogFile == -1) throw SysError(format("creating log file ‘%1%’") % logFileName); - closeOnExec(fdLogFile); - return logFileName; - } + return logFileName; } void DerivationGoal::closeLogFile() { - if (bzLogFile) { - int err; - BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0); - bzLogFile = 0; - if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err); - } - - if (fLogFile) { - fclose(fLogFile); - fLogFile = 0; - } - + auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink); + if (logSink2) logSink2->finish(); + if (logFileSink) logFileSink->flush(); + logSink = logFileSink = 0; fdLogFile.close(); } @@ -2779,59 +2905,61 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) printMsg(lvlError, format("%1% killed after writing more than %2% bytes of log output") % getName() % settings.maxLogSize); - cancel(true); // not really a timeout, but close enough + killChild(); + done(BuildResult::LogLimitExceeded); return; } - if (verbosity >= settings.buildVerbosity) - writeToStderr(filterANSIEscapes(data, true)); - if (bzLogFile) { - int err; - BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); - if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); - } else if (fdLogFile != -1) - writeFull(fdLogFile, data); + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + + if (logSink) (*logSink)(data); } if (hook && fd == hook->fromHook.readSide) - writeToStderr(data); + printMsg(lvlError, data); // FIXME? } void DerivationGoal::handleEOF(int fd) { + if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); } -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +void DerivationGoal::flushLine() { - PathSet result; - foreach (DerivationOutputs::iterator, i, drv.outputs) { - if (!wantOutput(i->first, wantedOutputs)) continue; - bool good = - worker.store.isValidPath(i->second.path) && - (!checkHash || worker.store.pathContentsGood(i->second.path)); - if (good == returnValid) result.insert(i->second.path); + if (settings.verboseBuild) + printMsg(lvlInfo, filterANSIEscapes(currentLogLine, true)); + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); } - return result; + currentLogLine = ""; + currentLogLinePos = 0; } -bool DerivationGoal::pathFailed(const Path & path) +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { - if (!settings.cacheFailure) return false; - - if (!worker.store.hasPathFailed(path)) return false; - - printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); - - worker.permanentFailure = true; - amDone(ecFailed); - - return true; + PathSet result; + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + bool good = + worker.store.isValidPath(i.second.path) && + (!checkHash || worker.pathContentsGood(i.second.path)); + if (good == returnValid) result.insert(i.second.path); + } + return result; } @@ -2840,7 +2968,7 @@ Path DerivationGoal::addHashRewrite(const Path & path) string h1 = string(path, settings.nixStore.size() + 1, 32); string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33); - if (pathExists(p)) deletePath(p); + deletePath(p); assert(path.size() == p.size()); rewritesToTmp[h1] = h2; rewritesFromTmp[h2] = h1; @@ -2849,6 +2977,18 @@ Path DerivationGoal::addHashRewrite(const Path & path) } +void DerivationGoal::done(BuildResult::Status status, const string & msg) +{ + result.status = status; + result.errorMsg = msg; + amDone(result.success() ? ecSuccess : ecFailed); + if (result.status == BuildResult::TimedOut) + worker.timedOut = true; + if (result.status == BuildResult::PermanentFailure) + worker.permanentFailure = true; +} + + ////////////////////////////////////////////////////////////////////// @@ -2861,28 +3001,24 @@ private: Path storePath; /* The remaining substituters. */ - Paths subs; + std::list<ref<Store>> subs; /* The current substituter. */ - Path sub; + std::shared_ptr<Store> sub; - /* Whether any substituter can realise this path */ + /* Whether any substituter can realise this path. */ bool hasSubstitute; /* Path info returned by the substituter's query info operation. */ - SubstitutablePathInfo info; + std::shared_ptr<const ValidPathInfo> info; /* Pipe for the substituter's standard output. */ Pipe outPipe; - /* Pipe for the substituter's standard error. */ - Pipe logPipe; + /* The substituter thread. */ + std::thread thr; - /* The process ID of the builder. */ - Pid pid; - - /* Lock on the store path. */ - std::shared_ptr<PathLocks> outputLock; + std::promise<void> promise; /* Whether to try to repair a valid path. */ bool repair; @@ -2898,7 +3034,7 @@ public: SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); ~SubstitutionGoal(); - void cancel(bool timeout); + void timedOut() { abort(); }; string key() { @@ -2939,20 +3075,14 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool SubstitutionGoal::~SubstitutionGoal() { - if (pid != -1) worker.childTerminated(pid); -} - - -void SubstitutionGoal::cancel(bool timeout) -{ - if (settings.printBuildTrace && timeout) - printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); - if (pid != -1) { - pid_t savedPid = pid; - pid.kill(); - worker.childTerminated(savedPid); + try { + if (thr.joinable()) { + thr.join(); + worker.childTerminated(shared_from_this()); + } + } catch (...) { + ignoreException(); } - amDone(ecFailed); } @@ -2977,7 +3107,7 @@ void SubstitutionGoal::init() if (settings.readOnlyMode) throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath); - subs = settings.substituters; + subs = getDefaultSubstituters(); tryNext(); } @@ -2991,6 +3121,7 @@ void SubstitutionGoal::tryNext() /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path ‘%1%’ is required, but there is no substituter that can build it") % storePath); + /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ @@ -3001,19 +3132,31 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - SubstitutablePathInfos infos; - PathSet dummy(singleton<PathSet>(storePath)); - worker.store.querySubstitutablePathInfos(sub, dummy, infos); - SubstitutablePathInfos::iterator k = infos.find(storePath); - if (k == infos.end()) { tryNext(); return; } - info = k->second; + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath &) { + tryNext(); + return; + } + hasSubstitute = true; + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (worker.store.requireSigs && !info->checkSignatures(worker.store.publicKeys)) { + printMsg(lvlInfo, format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’") + % sub->getUri() % storePath); + tryNext(); + return; + } + /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - foreach (PathSet::iterator, i, info.references) - if (*i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(*i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -3032,9 +3175,9 @@ void SubstitutionGoal::referencesValid() return; } - foreach (PathSet::iterator, i, info.references) - if (*i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(*i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); state = &SubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); @@ -3054,76 +3197,29 @@ void SubstitutionGoal::tryToRun() return; } - /* Maybe a derivation goal has already locked this path - (exceedingly unlikely, since it should have used a substitute - first, but let's be defensive). */ - outputLock.reset(); // make sure this goal's lock is gone - if (pathIsLockedByMe(storePath)) { - debug(format("restarting substitution of ‘%1%’ because it's locked by another goal") - % storePath); - worker.waitForAnyGoal(shared_from_this()); - return; /* restart in the tryToRun() state when another goal finishes */ - } - - /* Acquire a lock on the output path. */ - outputLock = std::shared_ptr<PathLocks>(new PathLocks); - if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Check again whether the path is invalid. */ - if (!repair && worker.store.isValidPath(storePath)) { - debug(format("store path ‘%1%’ has become valid") % storePath); - outputLock->setDeletion(true); - amDone(ecSuccess); - return; - } - printMsg(lvlInfo, format("fetching path ‘%1%’...") % storePath); outPipe.create(); - logPipe.create(); - - destPath = repair ? storePath + ".tmp" : storePath; - - /* Remove the (stale) output path if it exists. */ - if (pathExists(destPath)) - deletePath(destPath); - - worker.store.setSubstituterEnv(); - - /* Fill in the arguments. */ - Strings args; - args.push_back(baseNameOf(sub)); - args.push_back("--substitute"); - args.push_back(storePath); - args.push_back(destPath); - - /* Fork the substitute program. */ - pid = startProcess([&]() { - commonChildInit(logPipe); + promise = std::promise<void>(); - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide.close(); }); - execv(sub.c_str(), stringsToCharPtrs(args).data()); + copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), + storePath, repair); - throw SysError(format("executing ‘%1%’") % sub); + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } }); - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - outPipe.writeSide.close(); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton<set<int> >(logPipe.readSide), true, true); + worker.childStarted(shared_from_this(), {outPipe.readSide}, true, false); state = &SubstitutionGoal::finished; - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); } @@ -3131,110 +3227,40 @@ void SubstitutionGoal::finished() { trace("substitute finished"); - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - pid_t savedPid = pid; - int status = pid.wait(true); + thr.join(); + worker.childTerminated(shared_from_this()); - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); - - /* Get the hash info from stdout. */ - string dummy = readLine(outPipe.readSide); - string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; - outPipe.readSide.close(); - - /* Check the exit status and the build result. */ - HashResult hash; try { - - if (!statusOk(status)) - throw SubstError(format("fetching path ‘%1%’ %2%") - % storePath % statusToString(status)); - - if (!pathExists(destPath)) - throw SubstError(format("substitute did not produce path ‘%1%’") % destPath); - - hash = hashPath(htSHA256, destPath); - - /* Verify the expected hash we got from the substituer. */ - if (expectedHashStr != "") { - size_t n = expectedHashStr.find(':'); - if (n == string::npos) - throw Error(format("bad hash from substituter: %1%") % expectedHashStr); - HashType hashType = parseHashType(string(expectedHashStr, 0, n)); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm in ‘%1%’") % expectedHashStr); - Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); - Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) - throw SubstError(format("hash mismatch in downloaded path ‘%1%’: expected %2%, got %3%") - % storePath % printHash(expectedHash) % printHash(actualHash)); - } - - } catch (SubstError & e) { - + promise.get_future().get(); + } catch (Error & e) { printMsg(lvlInfo, e.msg()); - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") - % storePath % status % e.msg()); - } - /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } - if (repair) replaceValidPath(storePath, destPath); - - canonicalisePathMetaData(storePath, -1); - - worker.store.optimisePath(storePath); // FIXME: combine with hashPath() - - ValidPathInfo info2; - info2.path = storePath; - info2.hash = hash.first; - info2.narSize = hash.second; - info2.references = info.references; - info2.deriver = info.deriver; - worker.store.registerValidPath(info2); - - outputLock->setDeletion(true); - outputLock.reset(); - - worker.store.markContentsGood(storePath); + worker.markContentsGood(storePath); printMsg(lvlChatty, format("substitution of path ‘%1%’ succeeded") % storePath); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); - amDone(ecSuccess); } void SubstitutionGoal::handleChildOutput(int fd, const string & data) { - assert(fd == logPipe.readSide); - if (verbosity >= settings.buildVerbosity) writeToStderr(data); - /* Don't write substitution output to a log file for now. We - probably should, though. */ } void SubstitutionGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + if (fd == outPipe.readSide) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -3266,11 +3292,12 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, BuildMode buildMode) +GoalPtr Worker::makeDerivationGoal(const Path & path, + const StringSet & wantedOutputs, BuildMode buildMode) { GoalPtr goal = derivationGoals[path].lock(); if (!goal) { - goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, buildMode)); + goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); derivationGoals[path] = goal; wakeUp(goal); } else @@ -3279,11 +3306,20 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOu } +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drvPath, + const BasicDerivation & drv, BuildMode buildMode) +{ + auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); + wakeUp(goal); + return goal; +} + + GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) { GoalPtr goal = substitutionGoals[path].lock(); if (!goal) { - goal = GoalPtr(new SubstitutionGoal(path, *this, repair)); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair); substitutionGoals[path] = goal; wakeUp(goal); } @@ -3318,8 +3354,8 @@ void Worker::removeGoal(GoalPtr goal) } /* Wake up goals waiting for any goal to finish. */ - foreach (WeakGoals::iterator, i, waitingForAnyGoal) { - GoalPtr goal = i->lock(); + for (auto & i : waitingForAnyGoal) { + GoalPtr goal = i.lock(); if (goal) wakeUp(goal); } @@ -3340,9 +3376,8 @@ unsigned Worker::getNrLocalBuilds() } -void Worker::childStarted(GoalPtr goal, - pid_t pid, const set<int> & fds, bool inBuildSlot, - bool respectTimeouts) +void Worker::childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; @@ -3350,30 +3385,29 @@ void Worker::childStarted(GoalPtr goal, child.timeStarted = child.lastOutput = time(0); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; - children[pid] = child; + children.emplace_back(child); if (inBuildSlot) nrLocalBuilds++; } -void Worker::childTerminated(pid_t pid, bool wakeSleepers) +void Worker::childTerminated(GoalPtr goal, bool wakeSleepers) { - assert(pid != -1); /* common mistake */ - - Children::iterator i = children.find(pid); + auto i = std::find_if(children.begin(), children.end(), + [&](const Child & child) { return child.goal.lock() == goal; }); assert(i != children.end()); - if (i->second.inBuildSlot) { + if (i->inBuildSlot) { assert(nrLocalBuilds > 0); nrLocalBuilds--; } - children.erase(pid); + children.erase(i); if (wakeSleepers) { /* Wake up goals waiting for a build slot. */ - foreach (WeakGoals::iterator, i, wantingToBuild) { - GoalPtr goal = i->lock(); + for (auto & j : wantingToBuild) { + GoalPtr goal = j.lock(); if (goal) wakeUp(goal); } @@ -3408,9 +3442,9 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i); + for (auto & i : _topGoals) topGoals.insert(i); - startNest(nest, lvlDebug, format("entered goal loop")); + Activity act(*logger, lvlDebug, "entered goal loop"); while (1) { @@ -3474,12 +3508,12 @@ void Worker::waitForInput() deadline for any child. */ assert(sizeof(time_t) >= sizeof(long)); time_t nearest = LONG_MAX; // nearest deadline - foreach (Children::iterator, i, children) { - if (!i->second.respectTimeouts) continue; + for (auto & i : children) { + if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); } if (nearest != LONG_MAX) { timeout.tv_sec = std::max((time_t) 1, nearest - before); @@ -3497,17 +3531,16 @@ void Worker::waitForInput() timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); } else lastWokenUp = 0; - using namespace std; /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ fd_set fds; FD_ZERO(&fds); int fdMax = 0; - foreach (Children::iterator, i, children) { - foreach (set<int>::iterator, j, i->second.fds) { - FD_SET(*j, &fds); - if (*j >= fdMax) fdMax = *j + 1; + for (auto & i : children) { + for (auto & j : i.fds) { + FD_SET(j, &fds); + if (j >= fdMax) fdMax = j + 1; } } @@ -3519,73 +3552,65 @@ void Worker::waitForInput() time_t after = time(0); /* Process all available file descriptors. */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); - /* Since goals may be canceled from inside the loop below (causing - them go be erased from the `children' map), we have to be - careful that we don't keep iterators alive across calls to - cancel(). */ - set<pid_t> pids; - foreach (Children::iterator, i, children) pids.insert(i->first); - - foreach (set<pid_t>::iterator, i, pids) { checkInterrupt(); - Children::iterator j = children.find(*i); - if (j == children.end()) continue; // child destroyed - GoalPtr goal = j->second.goal.lock(); + + GoalPtr goal = j->goal.lock(); assert(goal); - set<int> fds2(j->second.fds); - foreach (set<int>::iterator, k, fds2) { - if (FD_ISSET(*k, &fds)) { + set<int> fds2(j->fds); + for (auto & k : fds2) { + if (FD_ISSET(k, &fds)) { unsigned char buffer[4096]; - ssize_t rd = read(*k, buffer, sizeof(buffer)); + ssize_t rd = read(k, buffer, sizeof(buffer)); if (rd == -1) { if (errno != EINTR) throw SysError(format("reading from %1%") % goal->getName()); } else if (rd == 0) { debug(format("%1%: got EOF") % goal->getName()); - goal->handleEOF(*k); - j->second.fds.erase(*k); + goal->handleEOF(k); + j->fds.erase(k); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); string data((char *) buffer, rd); - j->second.lastOutput = after; - goal->handleChildOutput(*k, data); + j->lastOutput = after; + goal->handleChildOutput(k, data); } } } if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && - j->second.respectTimeouts && - after - j->second.lastOutput >= (time_t) settings.maxSilentTime) + j->respectTimeouts && + after - j->lastOutput >= (time_t) settings.maxSilentTime) { printMsg(lvlError, format("%1% timed out after %2% seconds of silence") % goal->getName() % settings.maxSilentTime); - goal->cancel(true); - timedOut = true; + goal->timedOut(); } else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && - j->second.respectTimeouts && - after - j->second.timeStarted >= (time_t) settings.buildTimeout) + j->respectTimeouts && + after - j->timeStarted >= (time_t) settings.buildTimeout) { printMsg(lvlError, format("%1% timed out after %2% seconds") % goal->getName() % settings.buildTimeout); - goal->cancel(true); - timedOut = true; + goal->timedOut(); } } - if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { lastWokenUp = after; - foreach (WeakGoals::iterator, i, waitingForAWhile) { - GoalPtr goal = i->lock(); + for (auto & i : waitingForAWhile) { + GoalPtr goal = i.lock(); if (goal) wakeUp(goal); } waitingForAWhile.clear(); @@ -3599,33 +3624,56 @@ unsigned int Worker::exitStatus() } +bool Worker::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path ‘%1%’...") % path); + auto info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info->narHash.type, path); + Hash nullHash(htSHA256); + res = info->narHash == nullHash || info->narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); + return res; +} + + +void Worker::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + ////////////////////////////////////////////////////////////////////// void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - startNest(nest, lvlDebug, - format("building %1%") % showPaths(drvPaths)); - Worker worker(*this); Goals goals; - foreach (PathSet::const_iterator, i, drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + for (auto & i : drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); if (isDerivation(i2.first)) goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(*i, buildMode)); + goals.insert(worker.makeSubstitutionGoal(i, buildMode)); } worker.run(goals); PathSet failed; - foreach (Goals::iterator, i, goals) - if ((*i)->getExitCode() == Goal::ecFailed) { - DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get()); + for (auto & i : goals) + if (i->getExitCode() == Goal::ecFailed) { + DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath()); + else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } if (!failed.empty()) @@ -3633,6 +3681,26 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) } +BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + Worker worker(*this); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + + BuildResult result; + + try { + worker.run(Goals{goal}); + result = goal->getResult(); + } catch (Error & e) { + result.status = BuildResult::MiscFailure; + result.errorMsg = e.msg(); + } + + return result; +} + + void LocalStore::ensurePath(const Path & path) { /* If the path is already valid, we're done. */ @@ -3640,7 +3708,7 @@ void LocalStore::ensurePath(const Path & path) Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = singleton<Goals>(goal); + Goals goals = {goal}; worker.run(goals); @@ -3653,12 +3721,21 @@ void LocalStore::repairPath(const Path & path) { Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path, true); - Goals goals = singleton<Goals>(goal); + Goals goals = {goal}; worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + auto deriver = queryPathInfo(path)->deriver; + if (deriver != "" && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + } } |