diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 264 |
1 files changed, 162 insertions, 102 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d6671f45b231..b41daac99159 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -34,47 +34,27 @@ #include <bzlib.h> -/* Includes required for chroot support. */ -#if HAVE_SYS_PARAM_H -#include <sys/param.h> -#endif -#if HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif -#if HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#if HAVE_SCHED_H -#include <sched.h> -#endif - -/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but - <linux/fs.h> does. */ -#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H -#include <linux/fs.h> -#endif - -#define CHROOT_ENABLED HAVE_CHROOT && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root) - /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - #define SANDBOX_ENABLED 1 #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #else - #define SANDBOX_ENABLED 0 - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/bin" "/usr/bin" + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #endif -#if CHROOT_ENABLED +/* Includes required for chroot support. */ +#if __linux__ #include <sys/socket.h> #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#endif - -#if __linux__ #include <sys/personality.h> #include <sys/mman.h> +#include <sched.h> +#include <sys/param.h> +#include <sys/mount.h> +#include <sys/syscall.h> +#include <linux/fs.h> +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif #if HAVE_STATVFS @@ -745,6 +725,9 @@ private: /* The temporary directory. */ Path tmpDir; + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + /* File descriptor for the log file. */ FILE * fLogFile = 0; BZFILE * bzLogFile = 0; @@ -779,6 +762,12 @@ private: typedef map<string, string> Environment; Environment env; +#if __APPLE__ + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; + AutoDelete autoDelSandbox; +#endif + /* Hash rewriting. */ HashRewrites rewritesToTmp, rewritesFromTmp; typedef map<Path, Path> RedirectedOutputs; @@ -791,13 +780,19 @@ private: temporary paths. */ PathSet redirectedBadOutputs; - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; - BuildResult result; + /* The current round, if we're building multiple times. */ + unsigned int curRound = 1; + + unsigned int nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + ValidPathInfos prevInfos; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -1238,6 +1233,10 @@ void DerivationGoal::inputsRealised() for (auto & i : drv->outputs) if (i.second.hash == "") fixedOutput = false; + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; + /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ @@ -1259,6 +1258,9 @@ static bool canBuildLocally(const BasicDerivation & drv) #if __linux__ || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") || (drv.platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") +#elif __FreeBSD__ + || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") + || (drv.platform == "i686-linux" && settings.thisSystem == "i686-freebsd") #endif ; } @@ -1420,6 +1422,9 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) } +MakeError(NotDeterministic, BuildError) + + void DerivationGoal::buildDone() { trace("build done"); @@ -1519,6 +1524,15 @@ void DerivationGoal::buildDone() deleteTmpDir(true); + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + buildUser.release(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old @@ -1552,6 +1566,7 @@ void DerivationGoal::buildDone() % drvPath % 1 % e.msg()); st = + dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : fixedOutput || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; @@ -1678,10 +1693,13 @@ int childEntry(void * arg) void DerivationGoal::startBuilder() { - startNest(nest, lvlInfo, format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - "building path(s) %1%") % showPaths(missingPaths)); + auto f = format( + buildMode == bmRepair ? "repairing path(s) %1%" : + buildMode == bmCheck ? "checking path(s) %1%" : + nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : + "building path(s) %1%"); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ if (!canBuildLocally(*drv)) { @@ -1692,7 +1710,40 @@ void DerivationGoal::startBuilder() % drv->platform % settings.thisSystem % drvPath); } +#if __APPLE__ + additionalSandboxProfile = get(drv->env, "__sandboxProfile"); +#endif + + /* Are we doing a chroot build? Note that fixed-output + derivations are never done in a chroot, mainly so that + functions like fetchurl (which needs a proper /etc/resolv.conf) + work properly. Purity checking for fixed-output derivations + is somewhat pointless anyway. */ + { + string x = settings.get("build-use-sandbox", + /* deprecated alias */ + settings.get("build-use-chroot", string("false"))); + if (x != "true" && x != "false" && x != "relaxed") + throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); + if (x == "true") { + if (get(drv->env, "__noChroot") == "1") + throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " + "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error(format("derivation ‘%1%’ specifies a sandbox profile, " + "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath); +#endif + useChroot = true; + } + else if (x == "false") + useChroot = false; + else if (x == "relaxed") + useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; + } + /* Construct the environment passed to the builder. */ + env.clear(); /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when PATH is not set. We don't want this, so we fill it in with some dummy @@ -1719,7 +1770,12 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700); + auto drvName = storePathToName(drvPath); + tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + + /* In a sandbox, for determinism, always use the same temporary + directory. */ + tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; /* Add all bindings specified in the derivation via the environments, except those listed in the passAsFile @@ -1732,25 +1788,26 @@ void DerivationGoal::startBuilder() if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - Path p = tmpDir + "/.attr-" + std::to_string(fileNr++); + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; writeFile(p, i.second); filesToChown.insert(p); - env[i.first + "Path"] = p; + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } } /* For convenience, set an environment pointing to the top build directory. */ - env["NIX_BUILD_TOP"] = tmpDir; + env["NIX_BUILD_TOP"] = tmpDirInSandbox; /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; /* Explicitly set PWD to prevent problems with chroot builds. In particular, dietlibc cannot figure out the cwd because the inode of the current directory doesn't appear in .. (because getdents returns the inode of the mount point). */ - env["PWD"] = tmpDir; + env["PWD"] = tmpDirInSandbox; /* Compatibility hack with Nix <= 0.7: if this is a fixed-output derivation, tell the builder, so that for instance `fetchurl' @@ -1839,40 +1896,27 @@ void DerivationGoal::startBuilder() } - /* Are we doing a chroot build? Note that fixed-output - derivations are never done in a chroot, mainly so that - functions like fetchurl (which needs a proper /etc/resolv.conf) - work properly. Purity checking for fixed-output derivations - is somewhat pointless anyway. */ - { - string x = settings.get("build-use-chroot", string("false")); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-chroot’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { - if (get(drv->env, "__noChroot") == "1") - throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, but that's not allowed when ‘build-use-chroot’ is ‘true’") % drvPath); - useChroot = true; - } - else if (x == "false") - useChroot = false; - else if (x == "relaxed") - useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; - } - if (useChroot) { string defaultChrootDirs; -#if CHROOT_ENABLED +#if __linux__ if (isInStore(BASH_PATH)) defaultChrootDirs = "/bin/sh=" BASH_PATH; #endif /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", defaultChrootDirs)); - PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string(""))); + PathSet dirs = tokenizeString<StringSet>( + settings.get("build-sandbox-paths", + /* deprecated alias with lower priority */ + settings.get("build-chroot-dirs", defaultChrootDirs))); + PathSet dirs2 = tokenizeString<StringSet>( + settings.get("build-extra-chroot-dirs", + settings.get("build-extra-sandbox-paths", string("")))); dirs.insert(dirs2.begin(), dirs2.end()); + dirsInChroot.clear(); + for (auto & i : dirs) { size_t p = i.find('='); if (p == string::npos) @@ -1880,7 +1924,7 @@ void DerivationGoal::startBuilder() else dirsInChroot[string(i, 0, p)] = string(i, p + 1); } - dirsInChroot[tmpDir] = tmpDir; + dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ PathSet closure; @@ -1911,12 +1955,12 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); dirsInChroot[i] = i; } -#if CHROOT_ENABLED +#if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory @@ -2009,11 +2053,11 @@ void DerivationGoal::startBuilder() for (auto & i : drv->outputs) dirsInChroot.erase(i.second.path); -#elif SANDBOX_ENABLED +#elif __APPLE__ /* We don't really have any parent prep work to do (yet?) All work happens in the child, instead. */ #else - throw Error("chroot builds are not supported on this platform"); + throw Error("sandboxing builds is not supported on this platform"); #endif } @@ -2062,7 +2106,7 @@ void DerivationGoal::startBuilder() auto line = std::string{lines, lastPos, nlPos - lastPos}; lastPos = nlPos + 1; if (state == stBegin) { - if (line == "extra-chroot-dirs") { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { throw Error(format("unknown pre-build hook command ‘%1%’") @@ -2092,7 +2136,7 @@ void DerivationGoal::startBuilder() builderOut.create(); /* Fork a child to build the package. */ -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2133,7 +2177,7 @@ void DerivationGoal::startBuilder() size_t stackSize = 1 * 1024 * 1024; char * stack = (char *) mmap(0, stackSize, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (!stack) throw SysError("allocating stack"); + if (stack == MAP_FAILED) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); @@ -2194,7 +2238,7 @@ void DerivationGoal::runChild() commonChildInit(builderOut); -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Initialise the loopback interface. */ @@ -2327,10 +2371,8 @@ void DerivationGoal::runChild() if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) if (pivot_root(".", "real-root") == -1) throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root")); -#undef pivot_root if (chroot(".") == -1) throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir); @@ -2343,7 +2385,7 @@ void DerivationGoal::runChild() } #endif - if (chdir(tmpDir.c_str()) == -1) + if (chdir(tmpDirInSandbox.c_str()) == -1) throw SysError(format("changing into ‘%1%’") % tmpDir); /* Close all other file descriptors. */ @@ -2410,9 +2452,10 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (isBuiltin(*drv)) + if (isBuiltin(*drv)) { ; - else if (useChroot && SANDBOX_ENABLED) { +#if __APPLE__ + } else if (useChroot) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ PathSet ancestry; @@ -2442,8 +2485,6 @@ void DerivationGoal::runChild() /* This has to appear before import statements */ sandboxProfile += "(version 1)\n"; - sandboxProfile += (format("(import \"%1%/nix/sandbox-defaults.sb\")\n") % settings.nixDataDir).str(); - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ if (settings.get("darwin-log-sandbox-violations", false)) { sandboxProfile += "(deny default)\n"; @@ -2466,15 +2507,15 @@ void DerivationGoal::runChild() sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) - Note that the sandbox profile allows file-write* even though it isn't seemingly necessary. First of all, nix's standard user permissioning - mechanism still prevents builders from writing to input directories, so no security/purity is lost. The reason we allow file-write* is that - denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since - it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of - these is the entire GHC Haskell ecosystem. */ - sandboxProfile += "(allow file-read* file-write* process-exec mach-priv-task-port\n"; + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second) - throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin")); + throw Error(format( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") + % i.first % i.second); string path = i.first; struct stat st; @@ -2487,28 +2528,32 @@ void DerivationGoal::runChild() } sandboxProfile += ")\n"; - /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that, - you open up the entire filesystem because you end up with (subpath "/") - Note: file-read-metadata* is not sufficiently permissive for GHC. file-read* is but may - be a security hazard. - TODO: figure out a more appropriate directive. - */ + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); } sandboxProfile += ")\n"; + sandboxProfile += additionalSandboxProfile; + debug("Generated sandbox profile:"); debug(sandboxProfile); + Path sandboxFile = drvPath + ".sb"; + if (pathExists(sandboxFile)) deletePath(sandboxFile); + autoDelSandbox.reset(sandboxFile, false); + + writeFile(sandboxFile, sandboxProfile); + builder = "/usr/bin/sandbox-exec"; args.push_back("sandbox-exec"); - args.push_back("-p"); - args.push_back(sandboxProfile); + args.push_back("-f"); + args.push_back(sandboxFile); args.push_back("-D"); - args.push_back((format("_GLOBAL_TMP_DIR=%1%") % globalTmpDir).str()); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); args.push_back(drv->builder); +#endif } else { builder = drv->builder.c_str(); string builderBasename = baseNameOf(drv->builder); @@ -2582,6 +2627,11 @@ void DerivationGoal::registerOutputs() ValidPathInfos infos; + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -2598,7 +2648,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), path.c_str()) == -1) - throw SysError(format("moving build output ‘%1%’ from the chroot to the Nix store") % path); + throw SysError(format("moving build output ‘%1%’ from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = path; } else { @@ -2674,8 +2724,8 @@ void DerivationGoal::registerOutputs() Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); if (h != h2) throw BuildError( - format("output path ‘%1%’ should have %2% hash ‘%3%’, instead has ‘%4%’") - % path % i.second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); + format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") + % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); } /* Get rid of all weird permissions. This also checks that @@ -2753,6 +2803,16 @@ void DerivationGoal::registerOutputs() if (buildMode == bmCheck) return; + if (curRound > 1 && prevInfos != infos) + throw NotDeterministic( + format("result of ‘%1%’ differs from previous round; rejecting as non-deterministic") + % drvPath); + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ |