diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 222 |
1 files changed, 113 insertions, 109 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 968e291129f3..44cae3431fc2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -34,13 +34,6 @@ #include <pwd.h> #include <grp.h> -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - /* Includes required for chroot support. */ #if __linux__ #include <sys/socket.h> @@ -127,6 +120,8 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + Activity act; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -175,7 +170,8 @@ public: virtual string key() = 0; protected: - void amDone(ExitCode result); + + virtual void amDone(ExitCode result); }; @@ -469,7 +465,7 @@ UserLock::UserLock() assert(settings.buildUsersGroup != ""); /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist") % settings.buildUsersGroup); @@ -590,11 +586,7 @@ struct HookInstance HookInstance::HookInstance() { - debug("starting build hook"); - - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; - buildHook = canonPath(buildHook); + debug("starting build hook ‘%s’", settings.buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -621,15 +613,17 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(buildHook), + baseNameOf(settings.buildHook), settings.thisSystem, - (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.buildTimeout).str() + std::to_string(settings.maxSilentTime), + std::to_string(settings.buildTimeout), + std::to_string(verbosity), + settings.builders }; - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError(format("executing ‘%1%’") % buildHook); + throw SysError("executing ‘%s’", settings.buildHook); }); pid.setSeparatePG(true); @@ -911,6 +905,12 @@ private: void repairClosure(); + void amDone(ExitCode result) + { + logger->event(evBuildFinished, act, result == ecSuccess); + Goal::amDone(result); + } + void done(BuildResult::Status status, const string & msg = ""); }; @@ -929,6 +929,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut state = &DerivationGoal::getDerivation; name = (format("building of ‘%1%’") % drvPath).str(); trace("created"); + + logger->event(evBuildCreated, act, drvPath); } @@ -944,6 +946,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); trace("created"); + logger->event(evBuildCreated, act, drvPath); + /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ worker.store.addTempRoot(drvPath); @@ -1075,12 +1079,8 @@ void DerivationGoal::haveDerivation() /* Reject doing a hash build of anything other than a fixed-output derivation. */ - if (buildMode == bmHash) { - if (drv->outputs.size() != 1 || - drv->outputs.find("out") == drv->outputs.end() || - drv->outputs["out"].hashAlgo == "") - throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); - } + if (buildMode == bmHash && !drv->isFixedOutput()) + throw Error("cannot do a hash build of non-fixed-output derivation ‘%1%’", drvPath); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1279,7 +1279,7 @@ void DerivationGoal::inputsRealised() /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1575,7 +1575,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; + if (!settings.useBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1608,8 +1608,15 @@ HookReply DerivationGoal::tryBuildHook() debug(format("hook reply is ‘%1%’") % reply); - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + settings.useBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; else if (reply != "accept") throw Error(format("bad hook reply ‘%1%’") % reply); @@ -1628,23 +1635,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - worker.store.computeFSClosure(drvPath, allInputs); - - string s; - for (auto & i : allInputs) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + remote system. */ + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - s = ""; - for (auto & i : missingPaths) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); hook->toHook.writeSide = -1; @@ -1697,12 +1693,7 @@ void DerivationGoal::startBuilder() /* Are we doing a chroot build? */ { - string x = settings.get("build-use-sandbox", - /* deprecated alias */ - settings.get("build-use-chroot", string("false"))); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { + if (settings.sandboxMode == smEnabled) { if (get(drv->env, "__noChroot") == "1") throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); @@ -1713,9 +1704,9 @@ void DerivationGoal::startBuilder() #endif useChroot = true; } - else if (x == "false") + else if (settings.sandboxMode == smDisabled) useChroot = false; - else if (x == "relaxed") + else if (settings.sandboxMode == smRelaxed) useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; } @@ -1739,7 +1730,14 @@ void DerivationGoal::startBuilder() /* In a sandbox, for determinism, always use the same temporary directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#elif __APPLE__ + // On Darwin, we canonize /tmp because its probably a symlink to /private/tmp. tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ @@ -1756,21 +1754,10 @@ void DerivationGoal::startBuilder() if (useChroot) { - string defaultChrootDirs; -#if __linux__ - if (worker.store.isInStore(BASH_PATH)) - defaultChrootDirs = "/bin/sh=" BASH_PATH; -#endif - /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = tokenizeString<StringSet>( - settings.get("build-sandbox-paths", - /* deprecated alias with lower priority */ - settings.get("build-chroot-dirs", defaultChrootDirs))); - PathSet dirs2 = tokenizeString<StringSet>( - settings.get("build-extra-chroot-dirs", - settings.get("build-extra-sandbox-paths", string("")))); + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; dirs.insert(dirs2.begin(), dirs2.end()); dirsInChroot.clear(); @@ -1796,14 +1783,14 @@ void DerivationGoal::startBuilder() try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + } catch (InvalidPath & e) { } catch (Error & e) { throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what()); } for (auto & i : closure) dirsInChroot[i] = i; - string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); - PathSet allowedPaths = tokenizeString<StringSet>(allowed); + PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); @@ -1823,7 +1810,7 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps") % drvPath % i); dirsInChroot[i] = i; } @@ -1859,11 +1846,11 @@ void DerivationGoal::startBuilder() Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - writeFile(chrootRootDir + "/etc/passwd", - (format( - "root:x:0:0:Nix build user:/:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n") % sandboxUid % sandboxGid).str()); + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -1900,6 +1887,7 @@ void DerivationGoal::startBuilder() dirsInChroot[i] = r; else { Path p = chrootRootDir + i; + debug("linking ‘%1%’ to ‘%2%’", p, r); if (link(r.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), @@ -2137,6 +2125,8 @@ void DerivationGoal::startBuilder() } debug(msg); } + + logger->event(evBuildStarted, act); } @@ -2382,10 +2372,8 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); -#ifdef __linux__ if (pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); -#endif ss.push_back("/dev/null"); ss.push_back("/dev/random"); ss.push_back("/dev/tty"); @@ -2414,17 +2402,14 @@ void DerivationGoal::runChild() /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - for (auto & i : dirsInChroot) { - struct stat st; - Path source = i.second.source; - Path target = chrootRootDir + i.first; - if (source == "/proc") continue; // backwards compatibility + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target); + struct stat st; if (stat(source.c_str(), &st) == -1) { - if (i.second.optional && errno == ENOENT) - continue; + if (optional && errno == ENOENT) + return; else - throw SysError(format("getting attributes of path ‘%1%’") % source); + throw SysError("getting attributes of path ‘%1%’", source); } if (S_ISDIR(st.st_mode)) createDirs(target); @@ -2433,7 +2418,12 @@ void DerivationGoal::runChild() writeFile(target, ""); } if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError(format("bind mount from ‘%1%’ to ‘%2%’ failed") % source % target); + throw SysError("bind mount from ‘%1%’ to ‘%2%’ failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ @@ -2444,7 +2434,7 @@ void DerivationGoal::runChild() /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1) + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) throw SysError("mounting /dev/shm"); /* Mount a new devpts on /dev/pts. Note that this @@ -2455,13 +2445,19 @@ void DerivationGoal::runChild() !pathExists(chrootRootDir + "/dev/ptmx") && !dirsInChroot.count("/dev/pts")) { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1) - throw SysError("mounting /dev/pts"); - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", "/dev/pts"); + doBind("/dev/ptmx", "/dev/ptmx"); + } } /* Do the chroot(). */ @@ -2602,7 +2598,7 @@ void DerivationGoal::runChild() sandboxProfile += "(version 1)\n"; /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.get("darwin-log-sandbox-violations", false)) { + if (settings.darwinLogSandboxViolations) { sandboxProfile += "(deny default)\n"; } else { sandboxProfile += "(deny default (with no-log))\n"; @@ -2749,7 +2745,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.get("run-diff-hook", false); + bool runDiffHook = settings.runDiffHook; bool keepPreviousRound = settings.keepFailed || runDiffHook; /* Check whether the output paths were created, and grep each @@ -2876,7 +2872,7 @@ void DerivationGoal::registerOutputs() contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); + debug("scanning for references inside ‘%1%’", path); HashResult hash; PathSet references = scanForReferences(actualPath, allPaths, hash); @@ -2990,7 +2986,7 @@ void DerivationGoal::registerOutputs() ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); - auto diffHook = settings.get("diff-hook", std::string("")); + auto diffHook = settings.diffHook; if (prevExists && diffHook != "" && runDiffHook) { try { auto diff = runProgram(diffHook, true, {prev, i->path}); @@ -3001,7 +2997,7 @@ void DerivationGoal::registerOutputs() } } - if (settings.get("enforce-determinism", true)) + if (settings.enforceDeterminism) throw NotDeterministic(msg); printError(msg); @@ -3051,13 +3047,11 @@ Path DerivationGoal::openLogFile() string baseName = baseNameOf(drvPath); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); createDirs(dir); - Path logFileName = (format("%1%/%2%%3%") - % dir - % string(baseName, 2) - % (settings.compressLog ? ".bz2" : "")).str(); + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName); @@ -3131,7 +3125,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide.get()) - printError(data); // FIXME? + printError(chomp(data)); } @@ -3151,6 +3145,7 @@ void DerivationGoal::flushLine() logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); } + logger->event(evBuildOutput, act, currentLogLine); currentLogLine = ""; currentLogLinePos = 0; } @@ -3265,6 +3260,12 @@ public: void handleEOF(int fd); Path getStorePath() { return storePath; } + + void amDone(ExitCode result) + { + logger->event(evSubstitutionFinished, act, result == ecSuccess); + Goal::amDone(result); + } }; @@ -3277,6 +3278,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool state = &SubstitutionGoal::init; name = (format("substitution of ‘%1%’") % storePath).str(); trace("created"); + logger->event(evSubstitutionCreated, act, storePath); } @@ -3402,16 +3404,18 @@ void SubstitutionGoal::tryToRun() trace("trying to run"); /* Make sure that we are allowed to start a build. Note that even - is maxBuildJobs == 0 (no local builds allowed), we still allow + if maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } printInfo(format("fetching path ‘%1%’...") % storePath); + logger->event(evSubstitutionStarted, act); + outPipe.create(); promise = std::promise<void>(); @@ -3658,7 +3662,7 @@ void Worker::run(const Goals & _topGoals) { for (auto & i : _topGoals) topGoals.insert(i); - Activity act(*logger, lvlDebug, "entered goal loop"); + debug("entered goal loop"); while (1) { @@ -3686,7 +3690,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && settings.maxBuildJobs == 0) throw Error( + if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( "unable to start any build; either increase ‘--max-jobs’ " "or enable distributed builds"); assert(!awake.empty()); @@ -3723,9 +3727,9 @@ void Worker::waitForInput() auto nearest = steady_time_point::max(); // nearest deadline for (auto & i : children) { if (!i.respectTimeouts) continue; - if (settings.maxSilentTime != 0) + if (0 != settings.maxSilentTime) nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (settings.buildTimeout != 0) + if (0 != settings.buildTimeout) nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { @@ -3803,7 +3807,7 @@ void Worker::waitForInput() } if (goal->getExitCode() == Goal::ecBusy && - settings.maxSilentTime != 0 && + 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { @@ -3814,7 +3818,7 @@ void Worker::waitForInput() } else if (goal->getExitCode() == Goal::ecBusy && - settings.buildTimeout != 0 && + 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { |