diff options
Diffstat (limited to 'src/libstore/build.cc')
| -rw-r--r-- | src/libstore/build.cc | 168 |
1 files changed, 96 insertions, 72 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index af2c908c30db..e7054a05a53a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -193,6 +193,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) { struct Child { WeakGoalPtr goal; + Goal * goal2; // ugly hackery set<int> fds; bool respectTimeouts; bool inBuildSlot; @@ -284,7 +285,7 @@ public: false if there is no sense in waking up goals that are sleeping because they can't run yet (e.g., there is no free build slot, or the hook would still say `postpone'). */ - void childTerminated(GoalPtr goal, bool wakeSleepers = true); + void childTerminated(Goal * goal, bool wakeSleepers = true); /* Put `goal' to sleep until a build slot becomes available (which might be right away). */ @@ -652,18 +653,15 @@ HookInstance::~HookInstance() ////////////////////////////////////////////////////////////////////// -typedef map<string, string> HashRewrites; +typedef map<std::string, std::string> StringRewrites; -string rewriteHashes(string s, const HashRewrites & rewrites) +std::string rewriteStrings(std::string s, const StringRewrites & rewrites) { for (auto & i : rewrites) { - assert(i.first.size() == i.second.size()); size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) { - debug(format("rewriting @ %1%") % j); - s.replace(j, i.second.size(), i.second); - } + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); } return s; } @@ -782,7 +780,7 @@ private: #endif /* Hash rewriting. */ - HashRewrites rewritesToTmp, rewritesFromTmp; + StringRewrites inputRewrites, outputRewrites; typedef map<Path, Path> RedirectedOutputs; RedirectedOutputs redirectedOutputs; @@ -806,6 +804,9 @@ private: result. */ ValidPathInfos prevInfos; + const uid_t sandboxUid = 1000; + const gid_t sandboxGid = 100; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -938,7 +939,7 @@ DerivationGoal::~DerivationGoal() void DerivationGoal::killChild() { if (pid != -1) { - worker.childTerminated(shared_from_this()); + worker.childTerminated(this); if (buildUser.enabled()) { /* If we're using a build user, then there is a tricky @@ -1013,7 +1014,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printMsg(lvlError, format("cannot build missing derivation ‘%1%’") % drvPath); + printError(format("cannot build missing derivation ‘%1%’") % drvPath); done(BuildResult::MiscFailure); return; } @@ -1167,7 +1168,7 @@ void DerivationGoal::repairClosure() PathSet broken; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); + printError(format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") addWaitee(worker.makeSubstitutionGoal(i, true)); @@ -1200,7 +1201,7 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error(format("some dependencies of ‘%1%’ are missing") % drvPath); - printMsg(lvlError, + printError( format("cannot build derivation ‘%1%’: %2% dependencies couldn't be built") % drvPath % nrFailed); done(BuildResult::DependencyFailed); @@ -1365,7 +1366,7 @@ void DerivationGoal::tryToBuild() startBuilder(); } catch (BuildError & e) { - printMsg(lvlError, e.msg()); + printError(e.msg()); outputLocks.unlock(); buildUser.release(); worker.permanentFailure = true; @@ -1412,7 +1413,7 @@ void DerivationGoal::buildDone() debug(format("builder process for ‘%1%’ finished") % drvPath); /* So the child is gone now. */ - worker.childTerminated(shared_from_this()); + worker.childTerminated(this); /* Close the read side of the logger pipe. */ if (hook) { @@ -1514,7 +1515,7 @@ void DerivationGoal::buildDone() } catch (BuildError & e) { if (!hook) - printMsg(lvlError, e.msg()); + printError(e.msg()); outputLocks.unlock(); buildUser.release(); @@ -1643,7 +1644,7 @@ void DerivationGoal::startBuilder() nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : "building path(s) %1%"); f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - printMsg(lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); + printInfo(f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ if (!drv->canBuildLocally()) { @@ -1774,6 +1775,10 @@ void DerivationGoal::startBuilder() for (auto & i : varNames) env[i] = getEnv(i); } + /* Substitute output placeholders with the actual output paths. */ + for (auto & output : drv->outputs) + inputRewrites[hashPlaceholder(output.first)] = output.second.path; + /* The `exportReferencesGraph' feature allows the references graph to be passed to a builder. This attribute should be a list of pairs [name1 path1 name2 path2 ...]. The references graph of @@ -1872,8 +1877,12 @@ void DerivationGoal::startBuilder() /* Add the closure of store paths to the chroot. */ PathSet closure; for (auto & i : dirsInChroot) - if (worker.store.isInStore(i.second)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second), closure); + try { + if (worker.store.isInStore(i.second)) + worker.store.computeFSClosure(worker.store.toStorePath(i.second), closure); + } catch (Error & e) { + throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what()); + } for (auto & i : closure) dirsInChroot[i] = i; @@ -1935,14 +1944,18 @@ void DerivationGoal::startBuilder() createDirs(chrootRootDir + "/etc"); writeFile(chrootRootDir + "/etc/passwd", - "root:x:0:0:Nix build user:/:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n"); + (format( + "root:x:0:0:Nix build user:/:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n") % sandboxUid % sandboxGid).str()); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ writeFile(chrootRootDir + "/etc/group", - "root:x:0:\n" - "nobody:x:65534:\n"); + (format( + "root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n") % sandboxGid).str()); /* Create /etc/hosts with localhost entry. */ if (!fixedOutput) @@ -2124,7 +2137,12 @@ void DerivationGoal::startBuilder() Pid helper = startProcess([&]() { /* Drop additional groups here because we can't do it - after we've created the new user namespace. */ + after we've created the new user namespace. FIXME: + this means that if we're not root in the parent + namespace, we can't drop additional groups; they will + be mapped to nogroup in the child namespace. There does + not seem to be a workaround for this. (But who can tell + from reading user_namespaces(7)?)*/ if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); @@ -2157,19 +2175,19 @@ void DerivationGoal::startBuilder() if (!string2Int<pid_t>(readLine(builderOut.readSide.get()), tmp)) abort(); pid = tmp; - /* Set the UID/GID mapping of the builder's user - namespace such that root maps to the build user, or to the - calling user (if build users are disabled). */ - uid_t targetUid = buildUser.enabled() ? buildUser.getUID() : getuid(); - uid_t targetGid = buildUser.enabled() ? buildUser.getGID() : getgid(); + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser.enabled() ? buildUser.getUID() : getuid(); + uid_t hostGid = buildUser.enabled() ? buildUser.getGID() : getgid(); writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("0 %d 1") % targetUid).str()); + (format("%d %d 1") % sandboxUid % hostUid).str()); writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("0 %d 1") % targetGid).str()); + (format("%d %d 1") % sandboxGid % hostGid).str()); /* Signal the builder that we've updated its user namespace. */ @@ -2198,7 +2216,7 @@ void DerivationGoal::startBuilder() if (msg.size() == 1) break; throw Error(string(msg, 1)); } - printMsg(lvlDebug, msg); + debug(msg); } } @@ -2328,13 +2346,15 @@ void DerivationGoal::runChild() /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ - if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, 0) == -1) + if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1) throw SysError("mounting /dev/shm"); /* Mount a new devpts on /dev/pts. Note that this requires the kernel to be compiled with CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case if /dev/ptx/ptmx exists). */ +#if 0 if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") && dirsInChroot.find("/dev/pts") == dirsInChroot.end()) @@ -2347,6 +2367,7 @@ void DerivationGoal::runChild() Linux versions, it is created with permissions 0. */ chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); } +#endif /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) @@ -2367,11 +2388,12 @@ void DerivationGoal::runChild() if (rmdir("real-root") == -1) throw SysError("cannot remove real-root directory"); - /* Become root in the user namespace, which corresponds to - the build user or calling user in the parent namespace. */ - if (setgid(0) == -1) + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid) == -1) throw SysError("setgid failed"); - if (setuid(0) == -1) + if (setuid(sandboxUid) == -1) throw SysError("setuid failed"); setUser = false; @@ -2418,7 +2440,7 @@ void DerivationGoal::runChild() /* Fill in the environment. */ Strings envStrs; for (auto & i : env) - envStrs.push_back(rewriteHashes(i.first + "=" + i.second, rewritesToTmp)); + envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop all root @@ -2560,7 +2582,7 @@ void DerivationGoal::runChild() } for (auto & i : drv->args) - args.push_back(rewriteHashes(i, rewritesToTmp)); + args.push_back(rewriteStrings(i, inputRewrites)); restoreSIGPIPE(); @@ -2682,8 +2704,8 @@ void DerivationGoal::registerOutputs() /* Apply hash rewriting if necessary. */ bool rewritten = false; - if (!rewritesFromTmp.empty()) { - printMsg(lvlError, format("warning: rewriting hashes in ‘%1%’; cross fingers") % path); + if (!outputRewrites.empty()) { + printError(format("warning: rewriting hashes in ‘%1%’; cross fingers") % path); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -2694,7 +2716,7 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); deletePath(actualPath); - sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp)); + sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); StringSource source(*sink.s); restorePath(actualPath, source); @@ -2706,8 +2728,8 @@ void DerivationGoal::registerOutputs() hash). */ if (i.second.hash != "") { - bool recursive; HashType ht; Hash h; - i.second.parseHashInfo(recursive, ht, h); + bool recursive; Hash h; + i.second.parseHashInfo(recursive, h); if (!recursive) { /* The output path should be a regular file without @@ -2719,11 +2741,11 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); + Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); if (buildMode == bmHash) { - Path dest = worker.store.makeFixedOutputPath(recursive, ht, h2, drv->env["name"]); - printMsg(lvlError, format("build produced path ‘%1%’ with %2% hash ‘%3%’") - % dest % printHashType(ht) % printHash16or32(h2)); + Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); + printError(format("build produced path ‘%1%’ with %2% hash ‘%3%’") + % dest % printHashType(h.type) % printHash16or32(h2)); if (worker.store.isValidPath(dest)) return; Path actualDest = worker.store.toRealPath(dest); @@ -2910,7 +2932,7 @@ Path DerivationGoal::openLogFile() string baseName = baseNameOf(drvPath); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % drvsLogDir % string(baseName, 0, 2)).str(); createDirs(dir); Path logFileName = (format("%1%/%2%%3%") @@ -2946,7 +2968,7 @@ void DerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { if (settings.keepFailed && !force) { - printMsg(lvlError, + printError( format("note: keeping build directory ‘%2%’") % drvPath % tmpDir); chmod(tmpDir.c_str(), 0755); @@ -2965,7 +2987,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - printMsg(lvlError, + printError( format("%1% killed after writing more than %2% bytes of log output") % getName() % settings.maxLogSize); killChild(); @@ -2988,7 +3010,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide.get()) - printMsg(lvlError, data); // FIXME? + printError(data); // FIXME? } @@ -3002,7 +3024,7 @@ void DerivationGoal::handleEOF(int fd) void DerivationGoal::flushLine() { if (settings.verboseBuild) - printMsg(lvlInfo, filterANSIEscapes(currentLogLine, true)); + printInfo(filterANSIEscapes(currentLogLine, true)); else { logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); @@ -3033,8 +3055,8 @@ Path DerivationGoal::addHashRewrite(const Path & path) Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); deletePath(p); assert(path.size() == p.size()); - rewritesToTmp[h1] = h2; - rewritesFromTmp[h2] = h1; + inputRewrites[h1] = h2; + outputRewrites[h2] = h1; redirectedOutputs[path] = p; return p; } @@ -3140,8 +3162,9 @@ SubstitutionGoal::~SubstitutionGoal() { try { if (thr.joinable()) { + // FIXME: signal worker thread to quit. thr.join(); - //worker.childTerminated(shared_from_this()); // FIXME + worker.childTerminated(this); } } catch (...) { ignoreException(); @@ -3213,8 +3236,8 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs && !info->checkSignatures(worker.store.publicKeys)) { - printMsg(lvlInfo, format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’") + if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) { + printInfo(format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’") % sub->getUri() % storePath); tryNext(); return; @@ -3265,7 +3288,7 @@ void SubstitutionGoal::tryToRun() return; } - printMsg(lvlInfo, format("fetching path ‘%1%’...") % storePath); + printInfo(format("fetching path ‘%1%’...") % storePath); outPipe.create(); @@ -3296,12 +3319,12 @@ void SubstitutionGoal::finished() trace("substitute finished"); thr.join(); - worker.childTerminated(shared_from_this()); + worker.childTerminated(this); try { promise.get_future().get(); } catch (Error & e) { - printMsg(lvlInfo, e.msg()); + printInfo(e.msg()); /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; @@ -3449,6 +3472,7 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds, { Child child; child.goal = goal; + child.goal2 = goal.get(); child.fds = fds; child.timeStarted = child.lastOutput = time(0); child.inBuildSlot = inBuildSlot; @@ -3458,11 +3482,11 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds, } -void Worker::childTerminated(GoalPtr goal, bool wakeSleepers) +void Worker::childTerminated(Goal * goal, bool wakeSleepers) { auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal.lock() == goal; }); - assert(i != children.end()); + [&](const Child & child) { return child.goal2 == goal; }); + if (i == children.end()) return; if (i->inBuildSlot) { assert(nrLocalBuilds > 0); @@ -3594,7 +3618,7 @@ void Worker::waitForInput() if (!waitingForAWhile.empty()) { useTimeout = true; if (lastWokenUp == 0) - printMsg(lvlError, "waiting for locks or build slots..."); + printError("waiting for locks or build slots..."); if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); } else lastWokenUp = 0; @@ -3657,7 +3681,7 @@ void Worker::waitForInput() j->respectTimeouts && after - j->lastOutput >= (time_t) settings.maxSilentTime) { - printMsg(lvlError, + printError( format("%1% timed out after %2% seconds of silence") % goal->getName() % settings.maxSilentTime); goal->timedOut(); @@ -3668,7 +3692,7 @@ void Worker::waitForInput() j->respectTimeouts && after - j->timeStarted >= (time_t) settings.buildTimeout) { - printMsg(lvlError, + printError( format("%1% timed out after %2% seconds") % goal->getName() % settings.buildTimeout); goal->timedOut(); @@ -3696,7 +3720,7 @@ bool Worker::pathContentsGood(const Path & path) { std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); + printInfo(format("checking path ‘%1%’...") % path); auto info = store.queryPathInfo(path); bool res; if (!pathExists(path)) @@ -3707,7 +3731,7 @@ bool Worker::pathContentsGood(const Path & path) res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); + if (!res) printError(format("path ‘%1%’ is corrupted or missing!") % path); return res; } @@ -3745,7 +3769,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) } if (!failed.empty()) - throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); + throw Error(worker.exitStatus(), "build of %s failed",showPaths(failed)); } @@ -3781,7 +3805,7 @@ void LocalStore::ensurePath(const Path & path) worker.run(goals); if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("path ‘%1%’ does not exist and cannot be created") % path, worker.exitStatus()); + throw Error(worker.exitStatus(), "path ‘%s’ does not exist and cannot be created", path); } @@ -3802,7 +3826,7 @@ void LocalStore::repairPath(const Path & path) goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); worker.run(goals); } else - throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + throw Error(worker.exitStatus(), "cannot repair path ‘%s’", path); } } |