diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 235 |
1 files changed, 142 insertions, 93 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e00a07d3b1a0..180a558dc2e9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2,7 +2,6 @@ #include "references.hh" #include "pathlocks.hh" -#include "misc.hh" #include "globals.hh" #include "local-store.hh" #include "util.hh" @@ -621,11 +620,15 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide, 4) == -1) throw SysError("dupping builder's stdout/stderr"); - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); + Strings args = { + baseNameOf(buildHook), + settings.thisSystem, + (format("%1%") % settings.maxSilentTime).str(), + (format("%1%") % settings.printBuildTrace).str(), + (format("%1%") % settings.buildTimeout).str() + }; + + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); throw SysError(format("executing ‘%1%’") % buildHook); }); @@ -902,7 +905,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv { this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = (format("building of %1%") % showPaths(outputPaths(drv))).str(); + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); trace("created"); /* Prevent the .chroot directory from being @@ -1014,7 +1017,7 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); /* Get the derivation. */ - drv = std::unique_ptr<BasicDerivation>(new Derivation(derivationFromPath(worker.store, drvPath))); + drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath))); haveDerivation(); } @@ -1041,10 +1044,19 @@ void DerivationGoal::haveDerivation() for (auto & i : invalidOutputs) if (pathFailed(i)) return; + /* Reject doing a hash build of anything other than a fixed-output + derivation. */ + if (buildMode == bmHash) { + if (drv->outputs.size() != 1 || + drv->outputs.find("out") == drv->outputs.end() || + drv->outputs["out"].hashAlgo == "") + throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); + } + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && substitutesAllowed(*drv)) + if (settings.useSubstitutes && drv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair)); @@ -1123,8 +1135,10 @@ void DerivationGoal::repairClosure() /* Get the output closure. */ PathSet outputClosure; - for (auto & i : drv->outputs) - computeFSClosure(worker.store, i.second.path, outputClosure); + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second.path, outputClosure); + } /* Filter out our own outputs (which we have already checked). */ for (auto & i : drv->outputs) @@ -1134,11 +1148,11 @@ void DerivationGoal::repairClosure() derivation is responsible for which path in the output closure. */ PathSet inputClosure; - if (useDerivation) computeFSClosure(worker.store, drvPath, inputClosure); + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); std::map<Path, Path> outputsToDrv; for (auto & i : inputClosure) if (isDerivation(i)) { - Derivation drv = derivationFromPath(worker.store, i); + Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) outputsToDrv[j.second.path] = i; } @@ -1210,10 +1224,10 @@ void DerivationGoal::inputsRealised() `i' as input paths. Only add the closures of output paths that are specified as inputs. */ assert(worker.store.isValidPath(i.first)); - Derivation inDrv = derivationFromPath(worker.store, i.first); + Derivation inDrv = worker.store.derivationFromPath(i.first); for (auto & j : i.second) if (inDrv.outputs.find(j) != inDrv.outputs.end()) - computeFSClosure(worker.store, inDrv.outputs[j].path, inputPaths); + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); else throw Error( format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") @@ -1222,7 +1236,7 @@ void DerivationGoal::inputsRealised() /* Second, the input sources. */ for (auto & i : drv->inputSrcs) - computeFSClosure(worker.store, i, inputPaths); + worker.store.computeFSClosure(i, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1245,46 +1259,6 @@ void DerivationGoal::inputsRealised() } -static bool isBuiltin(const BasicDerivation & drv) -{ - return string(drv.builder, 0, 8) == "builtin:"; -} - - -static bool canBuildLocally(const BasicDerivation & drv) -{ - return drv.platform == settings.thisSystem - || isBuiltin(drv) -#if __linux__ - || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") - || (drv.platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") -#elif __FreeBSD__ - || (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") - || (drv.platform == "i686-linux" && settings.thisSystem == "i686-freebsd") -#endif - ; -} - - -static string get(const StringPairs & map, const string & key, const string & def = "") -{ - StringPairs::const_iterator i = map.find(key); - return i == map.end() ? def : i->second; -} - - -bool willBuildLocally(const BasicDerivation & drv) -{ - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv); -} - - -bool substitutesAllowed(const BasicDerivation & drv) -{ - return get(drv.env, "allowSubstitutes", "1") == "1"; -} - - void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1307,7 +1281,7 @@ void DerivationGoal::tryToBuild() can't acquire the lock, then continue; hopefully some other goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ - if (!outputLocks.lockPaths(outputPaths(*drv), "", false)) { + if (!outputLocks.lockPaths(drv->outputPaths(), "", false)) { worker.waitForAWhile(shared_from_this()); return; } @@ -1320,7 +1294,6 @@ void DerivationGoal::tryToBuild() now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); - assert(buildMode != bmCheck || validPaths.size() == drv->outputs.size()); if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { debug(format("skipping build of derivation ‘%1%’, someone beat us to it") % drvPath); outputLocks.setDeletion(true); @@ -1328,7 +1301,7 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = outputPaths(*drv); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); @@ -1351,7 +1324,7 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || willBuildLocally(*drv); + bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); /* Is the build hook willing to accept this job? */ if (!buildLocally) { @@ -1647,7 +1620,7 @@ HookReply DerivationGoal::tryBuildHook() list it since the remote system *probably* already has it.) */ PathSet allInputs; allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(worker.store, drvPath, allInputs); + worker.store.computeFSClosure(drvPath, allInputs); string s; for (auto & i : allInputs) { s += i; s += ' '; } @@ -1702,7 +1675,7 @@ void DerivationGoal::startBuilder() startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ - if (!canBuildLocally(*drv)) { + if (!drv->canBuildLocally()) { if (settings.printBuildTrace) printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform); throw Error( @@ -1710,6 +1683,10 @@ void DerivationGoal::startBuilder() % drv->platform % settings.thisSystem % drvPath); } +#if __APPLE__ + additionalSandboxProfile = get(drv->env, "__sandboxProfile"); +#endif + /* Are we doing a chroot build? Note that fixed-output derivations are never done in a chroot, mainly so that functions like fetchurl (which needs a proper /etc/resolv.conf) @@ -1723,7 +1700,13 @@ void DerivationGoal::startBuilder() throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); if (x == "true") { if (get(drv->env, "__noChroot") == "1") - throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); + throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " + "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error(format("derivation ‘%1%’ specifies a sandbox profile, " + "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath); +#endif useChroot = true; } else if (x == "false") @@ -1849,14 +1832,14 @@ void DerivationGoal::startBuilder() like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ PathSet paths, paths2; - computeFSClosure(worker.store, storePath, paths); + worker.store.computeFSClosure(storePath, paths); paths2 = paths; for (auto & j : paths2) { if (isDerivation(j)) { - Derivation drv = derivationFromPath(worker.store, j); + Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - computeFSClosure(worker.store, k.second.path, paths); + worker.store.computeFSClosure(k.second.path, paths); } } @@ -1920,13 +1903,10 @@ void DerivationGoal::startBuilder() PathSet closure; for (auto & i : dirsInChroot) if (isInStore(i.second)) - computeFSClosure(worker.store, toStorePath(i.second), closure); + worker.store.computeFSClosure(toStorePath(i.second), closure); for (auto & i : closure) dirsInChroot[i] = i; -#if __APPLE__ - additionalSandboxProfile = get(drv->env, "__sandboxProfile"); -#endif string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); PathSet allowedPaths = tokenizeString<StringSet>(allowed); @@ -1948,7 +1928,7 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); dirsInChroot[i] = i; } @@ -2170,7 +2150,7 @@ void DerivationGoal::startBuilder() size_t stackSize = 1 * 1024 * 1024; char * stack = (char *) mmap(0, stackSize, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (!stack) throw SysError("allocating stack"); + if (stack == MAP_FAILED) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); @@ -2191,7 +2171,7 @@ void DerivationGoal::startBuilder() #endif { ProcessOptions options; - options.allowVfork = !buildUser.enabled() && !isBuiltin(*drv); + options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); }, options); @@ -2445,7 +2425,7 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (isBuiltin(*drv)) { + if (drv->isBuiltin()) { ; #if __APPLE__ } else if (useChroot) { @@ -2562,7 +2542,7 @@ void DerivationGoal::runChild() writeFull(STDERR_FILENO, string("\1\n")); /* Execute the program. This should not return. */ - if (isBuiltin(*drv)) { + if (drv->isBuiltin()) { try { logType = ltFlat; if (drv->builder == "builtin:fetchurl") @@ -2625,6 +2605,8 @@ void DerivationGoal::registerOutputs() outputs to allow hard links between outputs. */ InodesSeen inodesSeen; + Path checkSuffix = "-check"; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -2650,7 +2632,7 @@ void DerivationGoal::registerOutputs() && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && pathExists(redirected)) replaceValidPath(path, redirected); - if (buildMode == bmCheck) + if (buildMode == bmCheck && redirected != "") actualPath = redirected; } @@ -2713,12 +2695,29 @@ void DerivationGoal::registerOutputs() format("output path ‘%1%’ should be a non-executable regular file") % path); } - /* Check the hash. */ + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); - if (h != h2) - throw BuildError( - format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") - % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); + if (buildMode == bmHash) { + Path dest = makeFixedOutputPath(recursive, ht, h2, drv->env["name"]); + printMsg(lvlError, format("build produced path ‘%1%’ with %2% hash ‘%3%’") + % dest % printHashType(ht) % printHash16or32(h2)); + if (worker.store.isValidPath(dest)) + return; + if (actualPath != dest) { + PathLocks outputLocks({dest}); + if (pathExists(dest)) + deletePath(dest); + if (rename(actualPath.c_str(), dest.c_str()) == -1) + throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest); + } + path = actualPath = dest; + } else { + if (h != h2) + throw BuildError( + format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") + % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); + } } /* Get rid of all weird permissions. This also checks that @@ -2734,9 +2733,20 @@ void DerivationGoal::registerOutputs() PathSet references = scanForReferences(actualPath, allPaths, hash); if (buildMode == bmCheck) { + if (!worker.store.isValidPath(path)) continue; ValidPathInfo info = worker.store.queryPathInfo(path); - if (hash.first != info.hash) - throw Error(format("derivation ‘%1%’ may not be deterministic: hash mismatch in output ‘%2%’") % drvPath % path); + if (hash.first != info.hash) { + if (settings.keepFailed) { + Path dst = path + checkSuffix; + if (pathExists(dst)) deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst); + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’") + % drvPath % path % dst); + } else + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") + % drvPath % path); + } continue; } @@ -2762,7 +2772,7 @@ void DerivationGoal::registerOutputs() for (auto & i : references) /* Don't call computeFSClosure on ourselves. */ if (actualPath != i) - computeFSClosure(worker.store, i, used); + worker.store.computeFSClosure(i, used); } else used = references; @@ -2781,9 +2791,11 @@ void DerivationGoal::registerOutputs() checkRefs("disallowedReferences", false, false); checkRefs("disallowedRequisites", false, true); - worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + if (curRound == nrRounds) { + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.store.markContentsGood(path); + } ValidPathInfo info; info.path = path; @@ -2796,10 +2808,37 @@ void DerivationGoal::registerOutputs() if (buildMode == bmCheck) return; - if (curRound > 1 && prevInfos != infos) - throw NotDeterministic( - format("result of ‘%1%’ differs from previous round; rejecting as non-deterministic") - % drvPath); + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + Path prev = i->path + checkSuffix; + if (pathExists(prev)) + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") + % i->path % drvPath % prev); + else + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from previous round") + % i->path % drvPath); + } + assert(false); // shouldn't happen + } + + if (settings.keepFailed) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + if (pathExists(prev)) deletePath(prev); + if (curRound < nrRounds) { + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); + } + } + + } if (curRound < nrRounds) { prevInfos = infos; @@ -3122,6 +3161,7 @@ void SubstitutionGoal::tryNext() /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path ‘%1%’ is required, but there is no substituter that can build it") % storePath); + /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ @@ -3721,7 +3761,7 @@ void Worker::waitForInput() } } - if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { GoalPtr goal = i.lock(); @@ -3817,8 +3857,17 @@ void LocalStore::repairPath(const Path & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + Path deriver = queryDeriver(path); + if (deriver != "" && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + } } |