diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 179 |
1 files changed, 114 insertions, 65 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dde87dcda8ed..e1ccb1eaf136 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -62,7 +62,7 @@ #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #else #define SANDBOX_ENABLED 0 - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/bin" "/usr/bin" + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #endif #if CHROOT_ENABLED @@ -74,6 +74,7 @@ #if __linux__ #include <sys/personality.h> +#include <sys/mman.h> #endif #if HAVE_STATVFS @@ -777,6 +778,12 @@ private: DirsInChroot dirsInChroot; typedef map<string, string> Environment; Environment env; +#if SANDBOX_ENABLED + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; + + AutoDelete autoDelSandbox; +#endif /* Hash rewriting. */ HashRewrites rewritesToTmp, rewritesFromTmp; @@ -790,13 +797,19 @@ private: temporary paths. */ PathSet redirectedBadOutputs; - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - BuildResult result; + /* The current round, if we're building multiple times. */ + unsigned int curRound = 1; + + unsigned int nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + ValidPathInfos prevInfos; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -1237,6 +1250,10 @@ void DerivationGoal::inputsRealised() for (auto & i : drv->outputs) if (i.second.hash == "") fixedOutput = false; + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; + /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ @@ -1245,11 +1262,22 @@ void DerivationGoal::inputsRealised() } -static bool canBuildLocally(const string & platform) +static bool isBuiltin(const BasicDerivation & drv) +{ + return string(drv.builder, 0, 8) == "builtin:"; +} + + +static bool canBuildLocally(const BasicDerivation & drv) { - return platform == settings.thisSystem + return drv.platform == settings.thisSystem + || isBuiltin(drv) #if __linux__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") + || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") + || (drv.platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") +#elif __FreeBSD__ + || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") + || (drv.platform == "i686-linux" && settings.thisSystem == "i686-freebsd") #endif ; } @@ -1264,7 +1292,7 @@ static string get(const StringPairs & map, const string & key, const string & de bool willBuildLocally(const BasicDerivation & drv) { - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); + return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv); } @@ -1274,12 +1302,6 @@ bool substitutesAllowed(const BasicDerivation & drv) } -static bool isBuiltin(const BasicDerivation & drv) -{ - return string(drv.builder, 0, 8) == "builtin:"; -} - - void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1417,6 +1439,9 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) } +MakeError(NotDeterministic, BuildError) + + void DerivationGoal::buildDone() { trace("build done"); @@ -1516,6 +1541,15 @@ void DerivationGoal::buildDone() deleteTmpDir(true); + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + buildUser.release(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old @@ -1549,6 +1583,7 @@ void DerivationGoal::buildDone() % drvPath % 1 % e.msg()); st = + dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : fixedOutput || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; @@ -1675,13 +1710,16 @@ int childEntry(void * arg) void DerivationGoal::startBuilder() { - startNest(nest, lvlInfo, format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - "building path(s) %1%") % showPaths(missingPaths)); + auto f = format( + buildMode == bmRepair ? "repairing path(s) %1%" : + buildMode == bmCheck ? "checking path(s) %1%" : + nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : + "building path(s) %1%"); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ - if (!canBuildLocally(drv->platform)) { + if (!canBuildLocally(*drv)) { if (settings.printBuildTrace) printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform); throw Error( @@ -1690,6 +1728,7 @@ void DerivationGoal::startBuilder() } /* Construct the environment passed to the builder. */ + env.clear(); /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when PATH is not set. We don't want this, so we fill it in with some dummy @@ -1729,7 +1768,7 @@ void DerivationGoal::startBuilder() if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - Path p = tmpDir + "/.attr-" + int2String(fileNr++); + Path p = tmpDir + "/.attr-" + std::to_string(fileNr++); writeFile(p, i.second); filesToChown.insert(p); env[i.first + "Path"] = p; @@ -1877,6 +1916,8 @@ void DerivationGoal::startBuilder() settings.get("build-extra-sandbox-paths", string("")))); dirs.insert(dirs2.begin(), dirs2.end()); + dirsInChroot.clear(); + for (auto & i : dirs) { size_t p = i.find('='); if (p == string::npos) @@ -1894,6 +1935,9 @@ void DerivationGoal::startBuilder() for (auto & i : closure) dirsInChroot[i] = i; +#if SANDBOX_ENABLED + additionalSandboxProfile = get(drv->env, "__sandboxProfile"); +#endif string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); PathSet allowedPaths = tokenizeString<StringSet>(allowed); @@ -2063,7 +2107,7 @@ void DerivationGoal::startBuilder() auto lastPos = std::string::size_type{0}; for (auto nlPos = lines.find('\n'); nlPos != string::npos; nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos}; + auto line = std::string{lines, lastPos, nlPos - lastPos}; lastPos = nlPos + 1; if (state == stBegin) { if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { @@ -2134,16 +2178,19 @@ void DerivationGoal::startBuilder() ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { - char stack[32 * 1024]; + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (!stack) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; - pid_t child = clone(childEntry, stack + sizeof(stack) - 8, flags, this); + pid_t child = clone(childEntry, stack + stackSize, flags, this); if (child == -1 && errno == EINVAL) /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + sizeof(stack) - 8, flags & ~CLONE_NEWPID, this); + child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); if (child == -1) throw SysError("cloning builder process"); - writeFull(builderOut.writeSide, int2String(child) + "\n"); + writeFull(builderOut.writeSide, std::to_string(child) + "\n"); _exit(0); }, options); if (helper.wait(true) != 0) @@ -2411,9 +2458,10 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (isBuiltin(*drv)) + if (isBuiltin(*drv)) { ; - else if (useChroot && SANDBOX_ENABLED) { +#if SANDBOX_ENABLED + } else if (useChroot) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ PathSet ancestry; @@ -2440,7 +2488,7 @@ void DerivationGoal::runChild() for (auto & i : inputPaths) dirsInChroot[i] = i; - /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */ + /* This has to appear before import statements */ sandboxProfile += "(version 1)\n"; /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2450,15 +2498,6 @@ void DerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(allow file-read* file-write-data (literal \"/dev/null\"))\n"; - - sandboxProfile += "(allow file-read-metadata\n" - "\t(literal \"/var\")\n" - "\t(literal \"/tmp\")\n" - "\t(literal \"/etc\")\n" - "\t(literal \"/etc/nix\")\n" - "\t(literal \"/etc/nix/nix.conf\"))\n"; - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); @@ -2466,20 +2505,6 @@ void DerivationGoal::runChild() /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - /* This is where our temp folders are and where most of the building will happen, so we want rwx on it. */ - sandboxProfile += (format("(allow file-read* file-write* process-exec (subpath \"%1%\") (subpath \"/private/tmp\"))\n") % globalTmpDir).str(); - - sandboxProfile += "(allow process-fork)\n"; - sandboxProfile += "(allow sysctl-read)\n"; - sandboxProfile += "(allow signal (target same-sandbox))\n"; - - /* Enables getpwuid (used by git and others) */ - sandboxProfile += "(allow mach-lookup (global-name \"com.apple.system.notification_center\") (global-name \"com.apple.system.opendirectoryd.libinfo\"))\n"; - - /* Allow local networking operations, mostly because lots of test suites use it and it seems mostly harmless */ - sandboxProfile += "(allow network* (local ip) (remote unix-socket))"; - - /* Our rwx outputs */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : missingPaths) { @@ -2488,11 +2513,9 @@ void DerivationGoal::runChild() sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) - Note that the sandbox profile allows file-write* even though it isn't seemingly necessary. First of all, nix's standard user permissioning - mechanism still prevents builders from writing to input directories, so no security/purity is lost. The reason we allow file-write* is that - denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since - it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of - these is the entire GHC Haskell ecosystem. */ + + without file-write* allowed, access() incorrectly returns EPERM + */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second) @@ -2509,22 +2532,32 @@ void DerivationGoal::runChild() } sandboxProfile += ")\n"; - /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that, - you open up the entire filesystem because you end up with (subpath "/") */ - sandboxProfile += "(allow file-read-metadata\n"; + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); } sandboxProfile += ")\n"; + sandboxProfile += additionalSandboxProfile; + debug("Generated sandbox profile:"); debug(sandboxProfile); + Path sandboxFile = drvPath + ".sb"; + if (pathExists(sandboxFile)) deletePath(sandboxFile); + autoDelSandbox.reset(sandboxFile, false); + + writeFile(sandboxFile, sandboxProfile); + builder = "/usr/bin/sandbox-exec"; args.push_back("sandbox-exec"); - args.push_back("-p"); - args.push_back(sandboxProfile); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); args.push_back(drv->builder); +#endif } else { builder = drv->builder.c_str(); string builderBasename = baseNameOf(drv->builder); @@ -2598,6 +2631,11 @@ void DerivationGoal::registerOutputs() ValidPathInfos infos; + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -2690,7 +2728,7 @@ void DerivationGoal::registerOutputs() Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); if (h != h2) throw BuildError( - format("output path ‘%1%’ should have %2% hash ‘%3%’, instead has ‘%4%’") + format("Nix expects output path ‘%1%’ to have %2% hash ‘%3%’, instead it has ‘%4%’") % path % i.second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); } @@ -2769,6 +2807,16 @@ void DerivationGoal::registerOutputs() if (buildMode == bmCheck) return; + if (curRound > 1 && prevInfos != infos) + throw NotDeterministic( + format("result of ‘%1%’ differs from previous round; rejecting as non-deterministic") + % drvPath); + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -2861,7 +2909,8 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) printMsg(lvlError, format("%1% killed after writing more than %2% bytes of log output") % getName() % settings.maxLogSize); - timedOut(); // not really a timeout, but close enough + killChild(); + done(BuildResult::LogLimitExceeded); return; } if (verbosity >= settings.buildVerbosity) |