diff options
Diffstat (limited to 'src/libstore')
| -rw-r--r-- | src/libstore/build.cc | 521 | ||||
| -rw-r--r-- | src/libstore/derivations.cc | 22 | ||||
| -rw-r--r-- | src/libstore/gc.cc | 78 | ||||
| -rw-r--r-- | src/libstore/globals.cc | 211 | ||||
| -rw-r--r-- | src/libstore/globals.hh | 223 | ||||
| -rw-r--r-- | src/libstore/local-store.cc | 310 | ||||
| -rw-r--r-- | src/libstore/local-store.hh | 70 | ||||
| -rw-r--r-- | src/libstore/misc.cc | 112 | ||||
| -rw-r--r-- | src/libstore/optimise-store.cc | 30 | ||||
| -rw-r--r-- | src/libstore/remote-store.cc | 162 | ||||
| -rw-r--r-- | src/libstore/remote-store.hh | 10 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 16 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 32 | ||||
| -rw-r--r-- | src/libstore/worker-protocol.hh | 9 |
14 files changed, 1049 insertions, 757 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 290635695e..0972d6e193 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -64,7 +64,7 @@ namespace nix { using std::map; - + static string pathNullDevice = "/dev/null"; @@ -94,10 +94,10 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap; class Goal : public boost::enable_shared_from_this<Goal> { public: - typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode; - + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode; + protected: - + /* Backlink to the worker. */ Worker & worker; @@ -111,6 +111,10 @@ protected: /* Number of goals we are/were waiting for that have failed. */ unsigned int nrFailed; + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; + /* Name of this goal for debugging purposes. */ string name; @@ -119,7 +123,7 @@ protected: Goal(Worker & worker) : worker(worker) { - nrFailed = 0; + nrFailed = nrNoSubstituters = 0; exitCode = ecBusy; } @@ -151,7 +155,7 @@ public: { return name; } - + ExitCode getExitCode() { return exitCode; @@ -213,7 +217,7 @@ private: /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; - + /* Goals sleeping for a few seconds (polling a lock). */ WeakGoals waitingForAWhile; @@ -225,8 +229,6 @@ private: public: - bool cacheFailure; - /* Set if at least one derivation had a BuildError (i.e. permanent failure). */ bool permanentFailure; @@ -234,7 +236,7 @@ public: LocalStore & store; boost::shared_ptr<HookInstance> hook; - + Worker(LocalStore & store); ~Worker(); @@ -271,13 +273,13 @@ public: /* Wait for any goal to finish. Pretty indiscriminate way to wait for some resource that some other goal is holding. */ void waitForAnyGoal(GoalPtr goal); - + /* Wait for a few seconds and then retry this goal. Used when waiting for a lock held by another process. This kind of polling is inefficient, but POSIX doesn't really provide a way to wait for multiple locks in the main select() loop. */ void waitForAWhile(GoalPtr goal); - + /* Loop until the specified top-level goals have finished. */ void run(const Goals & topGoals); @@ -305,10 +307,12 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) trace(format("waitee `%1%' done; %2% left") % waitee->name % waitees.size()); - - if (result == ecFailed) ++nrFailed; - - if (waitees.empty() || (result == ecFailed && !keepGoing)) { + + if (result == ecFailed || result == ecNoSubstituters) ++nrFailed; + + if (result == ecNoSubstituters) ++nrNoSubstituters; + + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { /* If we failed and keepGoing is not set, we remove all remaining waitees. */ @@ -330,7 +334,7 @@ void Goal::amDone(ExitCode result) { trace("done"); assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters); exitCode = result; foreach (WeakGoals::iterator, i, waiters) { GoalPtr goal = i->lock(); @@ -360,12 +364,13 @@ void commonChildInit(Pipe & logPipe) terminal signals. */ if (setsid() == -1) throw SysError(format("creating a new session")); - + /* Dup the write side of the logger pipe into stderr. */ if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) throw SysError("cannot pipe standard error into log file"); logPipe.readSide.close(); - + logPipe.writeSide.close(); + /* Dup stderr to stdout. */ if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) throw SysError("cannot dup stderr into stdout"); @@ -394,7 +399,7 @@ const char * * strings2CharPtrs(const Strings & ss) /* Restore default handling of SIGPIPE, otherwise some programs will randomly say "Broken pipe". */ -static void restoreSIGPIPE() +static void restoreSIGPIPE() { struct sigaction act, oact; act.sa_handler = SIG_DFL; @@ -421,7 +426,7 @@ private: string user; uid_t uid; gid_t gid; - + public: UserLock(); ~UserLock(); @@ -436,7 +441,7 @@ public: uid_t getGID() { return gid; } bool enabled() { return uid != 0; } - + }; @@ -459,14 +464,13 @@ void UserLock::acquire() { assert(uid == 0); - string buildUsersGroup = querySetting("build-users-group", ""); - assert(buildUsersGroup != ""); + assert(settings.buildUsersGroup != ""); /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); if (!gr) throw Error(format("the group `%1%' specified in `build-users-group' does not exist") - % buildUsersGroup); + % settings.buildUsersGroup); gid = gr->gr_gid; /* Copy the result of getgrnam. */ @@ -478,7 +482,7 @@ void UserLock::acquire() if (users.empty()) throw Error(format("the build users group `%1%' has no members") - % buildUsersGroup); + % settings.buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ @@ -488,16 +492,16 @@ void UserLock::acquire() struct passwd * pw = getpwnam(i->c_str()); if (!pw) throw Error(format("the user `%1%' in the group `%2%' does not exist") - % *i % buildUsersGroup); + % *i % settings.buildUsersGroup); - createDirs(nixStateDir + "/userpool"); - - fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str(); + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); if (lockedPaths.find(fnUserLock) != lockedPaths.end()) /* We already have a lock on this one. */ continue; - + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT, 0600); if (fd == -1) throw SysError(format("opening user lock `%1%'") % fnUserLock); @@ -512,15 +516,15 @@ void UserLock::acquire() /* Sanity check... */ if (uid == getuid() || uid == geteuid()) throw Error(format("the Nix user should not be a member of `%1%'") - % buildUsersGroup); - + % settings.buildUsersGroup); + return; } } throw Error(format("all build users are currently in use; " "consider creating additional users and adding them to the `%1%' group") - % buildUsersGroup); + % settings.buildUsersGroup); } @@ -539,8 +543,8 @@ static void runSetuidHelper(const string & command, const string & arg) { Path program = getEnv("NIX_SETUID_HELPER", - nixLibexecDir + "/nix-setuid-helper"); - + settings.nixLibexecDir + "/nix-setuid-helper"); + /* Fork. */ Pid pid; pid = fork(); @@ -558,7 +562,7 @@ static void runSetuidHelper(const string & command, args.push_back(0); restoreSIGPIPE(); - + execve(program.c_str(), (char * *) &args[0], 0); throw SysError(format("executing `%1%'") % program); } @@ -594,12 +598,6 @@ bool amPrivileged() } -bool haveBuildUsers() -{ - return querySetting("build-users-group", "") != ""; -} - - void getOwnership(const Path & path) { runSetuidHelper("get-ownership", path); @@ -615,7 +613,7 @@ void deletePathWrapped(const Path & path, } catch (SysError & e) { /* If this failed due to a permission error, then try it with the setuid helper. */ - if (haveBuildUsers() && !amPrivileged()) { + if (settings.buildUsersGroup != "" && !amPrivileged()) { getOwnership(path); deletePath(path, bytesFreed, blocksFreed); } else @@ -644,7 +642,7 @@ struct HookInstance /* Pipe for the builder's standard output/error. */ Pipe builderOut; - + /* The process ID of the hook. */ Pid pid; @@ -657,12 +655,12 @@ struct HookInstance HookInstance::HookInstance() { debug("starting build hook"); - + Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); - + /* Create a pipe to get the output of the child. */ fromHook.create(); - + /* Create the communication pipes. */ toHook.create(); @@ -672,7 +670,7 @@ HookInstance::HookInstance() /* Fork the hook. */ pid = fork(); switch (pid) { - + case -1: throw SysError("unable to fork"); @@ -682,7 +680,7 @@ HookInstance::HookInstance() commonChildInit(fromHook); if (chdir("/") == -1) throw SysError("changing into `/"); - + /* Dup the communication pipes. */ toHook.writeSide.close(); if (dup2(toHook.readSide, STDIN_FILENO) == -1) @@ -693,20 +691,20 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide, 4) == -1) throw SysError("dupping builder's stdout/stderr"); - /* XXX: Pass `buildTimeout' to the hook? */ - execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), - (format("%1%") % maxSilentTime).str().c_str(), - (format("%1%") % printBuildTrace).str().c_str(), + /* XXX: Pass `buildTimeout' to the hook? */ + execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), + (format("%1%") % settings.maxSilentTime).str().c_str(), + (format("%1%") % settings.printBuildTrace).str().c_str(), NULL); - + throw SysError(format("executing `%1%'") % buildHook); - + } catch (std::exception & e) { std::cerr << format("build hook error: %1%") % e.what() << std::endl; } quickExit(1); } - + /* parent */ pid.setSeparatePG(true); pid.setKillSignal(SIGTERM); @@ -736,6 +734,8 @@ HookInstance::~HookInstance() typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; +class SubstitutionGoal; + class DerivationGoal : public Goal { private: @@ -744,7 +744,7 @@ private: /* The derivation stored at drvPath. */ Derivation drv; - + /* The remainder is state held during the build. */ /* Locks on the output paths. */ @@ -752,7 +752,7 @@ private: /* All input paths (that is, the union of FS closures of the immediate input paths). */ - PathSet inputPaths; + PathSet inputPaths; /* Referenceable paths (i.e., input and output paths). */ PathSet allPaths; @@ -776,10 +776,10 @@ private: /* The build hook. */ boost::shared_ptr<HookInstance> hook; - + /* Whether we're currently doing a chroot build. */ bool useChroot; - + Path chrootRootDir; /* RAII object to delete the chroot directory. */ @@ -790,10 +790,10 @@ private: /* Whether this is a fixed-output derivation. */ bool fixedOutput; - + typedef void (DerivationGoal::*GoalState)(); GoalState state; - + /* Stuff we need to pass to initChild(). */ PathSet dirsInChroot; typedef map<string, string> Environment; @@ -804,7 +804,7 @@ public: ~DerivationGoal(); void cancel(); - + void work(); Path getDrvPath() @@ -909,7 +909,7 @@ void DerivationGoal::killChild() pid.wait(true); } else pid.kill(); - + assert(pid == -1); } @@ -934,7 +934,7 @@ void DerivationGoal::init() { trace("init"); - if (readOnlyMode) + if (settings.readOnlyMode) throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath); /* The first thing to do is to make sure that the derivation @@ -960,7 +960,7 @@ void DerivationGoal::haveDerivation() side: if the user forgot to make it a root, we wouldn't want things being garbage collected while we're busy. */ worker.store.addTempRoot(drvPath); - + assert(worker.store.isValidPath(drvPath)); /* Get the derivation. */ @@ -982,16 +982,14 @@ void DerivationGoal::haveDerivation() don't bother. */ foreach (PathSet::iterator, i, invalidOutputs) if (pathFailed(*i)) return; - + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - foreach (PathSet::iterator, i, invalidOutputs) - /* Don't bother creating a substitution goal if there are no - substitutes. */ - if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i)) + if (settings.useSubstitutes) + foreach (PathSet::iterator, i, invalidOutputs) addWaitee(worker.makeSubstitutionGoal(*i)); - + if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); else @@ -1003,10 +1001,10 @@ void DerivationGoal::outputsSubstituted() { trace("all outputs substituted (maybe)"); - if (nrFailed > 0 && !tryFallback) + if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath); - nrFailed = 0; + nrFailed = nrNoSubstituters = 0; if (checkPathValidity(false).size() == 0) { amDone(ecSuccess); @@ -1045,7 +1043,7 @@ void DerivationGoal::inputsRealised() /* Gather information necessary for computing the closure and/or running the build hook. */ - + /* The outputs are referenceable paths. */ foreach (DerivationOutputs::iterator, i, drv.outputs) { debug(format("building path `%1%'") % i->second.path); @@ -1082,7 +1080,7 @@ void DerivationGoal::inputsRealised() fixedOutput = true; foreach (DerivationOutputs::iterator, i, drv.outputs) if (i->second.hash == "") fixedOutput = false; - + /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ @@ -1102,9 +1100,9 @@ PathSet outputPaths(const DerivationOutputs & outputs) static bool canBuildLocally(const string & platform) { - return platform == thisSystem + return platform == settings.thisSystem #ifdef CAN_DO_LINUX32_BUILDS - || (platform == "i686-linux" && thisSystem == "x86_64-linux") + || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") #endif ; } @@ -1126,7 +1124,7 @@ void DerivationGoal::tryToBuild() worker.waitForAnyGoal(shared_from_this()); return; } - + /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other @@ -1201,17 +1199,17 @@ void DerivationGoal::tryToBuild() break; } } - + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; } - + try { /* Okay, we have to build. */ @@ -1221,7 +1219,7 @@ void DerivationGoal::tryToBuild() printMsg(lvlError, e.msg()); outputLocks.unlock(); buildUser.release(); - if (printBuildTrace) + if (settings.printBuildTrace) printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % 0 % e.msg()); worker.permanentFailure = true; @@ -1259,7 +1257,7 @@ void DerivationGoal::buildDone() /* So the child is gone now. */ worker.childTerminated(savedPid); - + /* Close the read side of the logger pipe. */ if (hook) { hook->builderOut.readSide.close(); @@ -1289,13 +1287,13 @@ void DerivationGoal::buildDone() if (rename((chrootRootDir + path).c_str(), path.c_str()) == -1) throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path); } - + if (!pathExists(path)) continue; struct stat st; if (lstat(path.c_str(), &st) == -1) throw SysError(format("getting attributes of path `%1%'") % path); - + #ifndef __CYGWIN__ /* Check that the output is not group or world writable, as that means that someone else can have interfered @@ -1313,14 +1311,14 @@ void DerivationGoal::buildDone() if (buildUser.enabled() && !amPrivileged()) getOwnership(path); } - + /* Check the exit status. */ if (!statusOk(status)) { deleteTmpDir(false); throw BuildError(format("builder for `%1%' %2%") % drvPath % statusToString(status)); } - + deleteTmpDir(true); /* Delete the chroot (if we were using one). */ @@ -1330,7 +1328,7 @@ void DerivationGoal::buildDone() hard-linked inputs to be cleared. So set them again. */ foreach (PathSet::iterator, i, regularInputPaths) makeImmutable(*i); - + /* Compute the FS closure of the outputs and register them as being valid. */ computeClosure(); @@ -1352,8 +1350,8 @@ void DerivationGoal::buildDone() problem. */ bool hookError = hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100); - - if (printBuildTrace) { + + if (settings.printBuildTrace) { if (hook && hookError) printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % status % e.msg()); @@ -1369,10 +1367,10 @@ void DerivationGoal::buildDone() able to access the network). Hook errors (like communication problems with the remote machine) shouldn't be cached either. */ - if (worker.cacheFailure && !hookError && !fixedOutput) + if (settings.cacheFailure && !hookError && !fixedOutput) foreach (DerivationOutputs::iterator, i, drv.outputs) worker.store.registerFailedPath(i->second.path); - + worker.permanentFailure = !hookError && !fixedOutput; amDone(ecFailed); return; @@ -1381,18 +1379,18 @@ void DerivationGoal::buildDone() /* Release the build user, if applicable. */ buildUser.release(); - if (printBuildTrace) { + if (settings.printBuildTrace) { printMsg(lvlError, format("@ build-succeeded %1% %2%") % drvPath % drv.outputs["out"].path); } - + amDone(ecSuccess); } HookReply DerivationGoal::tryBuildHook() { - if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; + if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; if (!worker.hook) worker.hook = boost::shared_ptr<HookInstance>(new HookInstance); @@ -1405,7 +1403,7 @@ HookReply DerivationGoal::tryBuildHook() /* Send the request to the hook. */ writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") - % (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") + % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") % drv.platform % drvPath % concatStringsSep(",", features)).str()); /* Read the first line of input, which should be a word indicating @@ -1433,7 +1431,7 @@ HookReply DerivationGoal::tryBuildHook() hook = worker.hook; worker.hook.reset(); - + /* Tell the hook all the inputs that have to be copied to the remote system. This unfortunately has to contain the entire derivation closure to ensure that the validity invariant holds @@ -1442,18 +1440,18 @@ HookReply DerivationGoal::tryBuildHook() PathSet allInputs; allInputs.insert(inputPaths.begin(), inputPaths.end()); computeFSClosure(worker.store, drvPath, allInputs); - + string s; foreach (PathSet::iterator, i, allInputs) s += *i + " "; writeLine(hook->toHook.writeSide, s); - + /* Tell the hooks the outputs that have to be copied back from the remote system. */ s = ""; foreach (DerivationOutputs::iterator, i, drv.outputs) s += i->second.path + " "; writeLine(hook->toHook.writeSide, s); - + hook->toHook.writeSide.close(); /* Create the log file and pipe. */ @@ -1463,12 +1461,12 @@ HookReply DerivationGoal::tryBuildHook() fds.insert(hook->fromHook.readSide); fds.insert(hook->builderOut.readSide); worker.childStarted(shared_from_this(), hook->pid, fds, false, false); - - if (printBuildTrace) + + if (settings.printBuildTrace) printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % drv.platform % logFile); - - return rpAccept; + + return rpAccept; } @@ -1490,15 +1488,15 @@ void DerivationGoal::startBuilder() { startNest(nest, lvlInfo, format("building path(s) %1%") % showPaths(outputPaths(drv.outputs))) - + /* Right platform? */ if (!canBuildLocally(drv.platform)) throw Error( format("a `%1%' is required to build `%3%', but I am a `%2%'") - % drv.platform % thisSystem % drvPath); + % drv.platform % settings.thisSystem % drvPath); /* Construct the environment passed to the builder. */ - + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when PATH is not set. We don't want this, so we fill it in with some dummy value. */ @@ -1516,10 +1514,10 @@ void DerivationGoal::startBuilder() shouldn't care, but this is useful for purity checking (e.g., the compiler or linker might only want to accept paths to files in the store or in the build directory). */ - env["NIX_STORE"] = nixStore; + env["NIX_STORE"] = settings.nixStore; /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str(); + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); /* Add all bindings specified in the derivation. */ foreach (StringPairs::iterator, i, drv.env) @@ -1608,10 +1606,10 @@ void DerivationGoal::startBuilder() worker.store.makeValidityRegistration(paths, false, false)); } - + /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ - if (haveBuildUsers()) { + if (settings.buildUsersGroup != "") { buildUser.acquire(); assert(buildUser.getUID() != 0); assert(buildUser.getGID() != 0); @@ -1619,7 +1617,7 @@ void DerivationGoal::startBuilder() /* Make sure that no other processes are executing under this uid. */ buildUser.kill(); - + /* Change ownership of the temporary build directory, if we're root. If we're not root, then the setuid helper will do it just before it starts the builder. */ @@ -1633,15 +1631,15 @@ void DerivationGoal::startBuilder() the builder can create its output but not mess with the outputs of other processes). */ struct stat st; - if (stat(nixStore.c_str(), &st) == -1) - throw SysError(format("cannot stat `%1%'") % nixStore); + if (stat(settings.nixStore.c_str(), &st) == -1) + throw SysError(format("cannot stat `%1%'") % settings.nixStore); if (!(st.st_mode & S_ISVTX) || ((st.st_mode & S_IRWXG) != S_IRWXG) || (st.st_gid != buildUser.getGID())) throw Error(format( "builder does not have write permission to `%2%'; " "try `chgrp %1% %2%; chmod 1775 %2%'") - % buildUser.getGID() % nixStore); + % buildUser.getGID() % settings.nixStore); } @@ -1650,7 +1648,7 @@ void DerivationGoal::startBuilder() functions like fetchurl (which needs a proper /etc/resolv.conf) work properly. Purity checking for fixed-output derivations is somewhat pointless anyway. */ - useChroot = queryBoolSetting("build-use-chroot", false); + useChroot = settings.useChroot; if (fixedOutput) useChroot = false; @@ -1668,7 +1666,7 @@ void DerivationGoal::startBuilder() /* Clean up the chroot directory automatically. */ autoDelChroot = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); - + printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need @@ -1690,8 +1688,8 @@ void DerivationGoal::startBuilder() % (buildUser.enabled() ? buildUser.getUID() : getuid()) % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); - /* Declare the build user's group so that programs get a consistent - view of the system (e.g., "id -gn"). */ + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ writeFile(chrootRootDir + "/etc/group", (format("nixbld:!:%1%:\n") % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); @@ -1700,16 +1698,8 @@ void DerivationGoal::startBuilder() writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); /* Bind-mount a user-configurable set of directories from the - host file system. The `/dev/pts' directory must be mounted - separately so that newly-created pseudo-terminals show - up. */ - Paths defaultDirs; - defaultDirs.push_back("/dev"); - defaultDirs.push_back("/dev/pts"); - - Paths dirsInChroot_ = querySetting("build-chroot-dirs", defaultDirs); - dirsInChroot.insert(dirsInChroot_.begin(), dirsInChroot_.end()); - + host file system. */ + dirsInChroot = settings.dirsInChroot; dirsInChroot.insert(tmpDir); /* Make the closure of the inputs available in the chroot, @@ -1719,8 +1709,8 @@ void DerivationGoal::startBuilder() can be bind-mounted). !!! As an extra security precaution, make the fake Nix store only writable by the build user. */ - createDirs(chrootRootDir + nixStore); - chmod(chrootRootDir + nixStore, 01777); + createDirs(chrootRootDir + settings.nixStore); + chmod(chrootRootDir + settings.nixStore, 01777); foreach (PathSet::iterator, i, inputPaths) { struct stat st; @@ -1732,7 +1722,7 @@ void DerivationGoal::startBuilder() /* Creating a hard link to *i is impossible if its immutable bit is set. So clear it first. */ makeMutable(*i); - + Path p = chrootRootDir + *i; if (link(i->c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum @@ -1749,25 +1739,25 @@ void DerivationGoal::startBuilder() StringSource source(sink.s); restorePath(p, source); } - + makeImmutable(*i); regularInputPaths.insert(*i); } } - + #else throw Error("chroot builds are not supported on this platform"); #endif } - - + + /* Run the builder. */ printMsg(lvlChatty, format("executing builder `%1%'") % drv.builder); /* Create the log file. */ Path logFile = openLogFile(); - + /* Create a pipe to get the output of the builder. */ builderOut.create(); @@ -1787,7 +1777,7 @@ void DerivationGoal::startBuilder() - The private mount namespace ensures that all the bind mounts we do will only show up in this process and its children, and will disappear automatically when we're done. - + - The private network namespace ensures that the builder cannot talk to the outside world (or vice versa). It only has a private loopback interface. @@ -1820,7 +1810,7 @@ void DerivationGoal::startBuilder() worker.childStarted(shared_from_this(), pid, singleton<set<int> >(builderOut.readSide), true, true); - if (printBuildTrace) { + if (settings.printBuildTrace) { printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % drv.platform % logFile); } @@ -1839,7 +1829,7 @@ void DerivationGoal::initChild() /* Initialise the loopback interface. */ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); if (fd == -1) throw SysError("cannot open IP socket"); - + struct ifreq ifr; strcpy(ifr.ifr_name, "lo"); ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; @@ -1861,9 +1851,9 @@ void DerivationGoal::initChild() Path source = *i; Path target = chrootRootDir + source; debug(format("bind mounting `%1%' to `%2%'") % source % target); - + createDirs(target); - + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); } @@ -1878,7 +1868,7 @@ void DerivationGoal::initChild() if (pathExists("/dev/shm")) if (mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, 0) == -1) throw SysError("mounting /dev/shm"); - + /* Do the chroot(). Below we do a chdir() to the temporary build directory to make sure the current directory is in the chroot. (Actually the order @@ -1888,9 +1878,9 @@ void DerivationGoal::initChild() throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); } #endif - + commonChildInit(builderOut); - + if (chdir(tmpDir.c_str()) == -1) throw SysError(format("changing into `%1%'") % tmpDir); @@ -1900,16 +1890,14 @@ void DerivationGoal::initChild() #ifdef CAN_DO_LINUX32_BUILDS /* Change the personality to 32-bit if we're doing an i686-linux build on an x86_64-linux machine. */ - if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") { + if (drv.platform == "i686-linux" && settings.thisSystem == "x86_64-linux") { if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1) throw SysError("cannot set i686-linux personality"); } /* Impersonate a Linux 2.6 machine to get some determinism in builds that depend on the kernel version. */ - if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && - queryBoolSetting("build-impersonate-linux-26", true)) - { + if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { int cur = personality(0xffffffff); if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); } @@ -1924,7 +1912,7 @@ void DerivationGoal::initChild() Path program = drv.builder.c_str(); std::vector<const char *> args; /* careful with c_str()! */ string user; /* must be here for its c_str()! */ - + /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop all root privileges. Note that above we have closed all file @@ -1935,10 +1923,10 @@ void DerivationGoal::initChild() printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); if (amPrivileged()) { - + if (setgroups(0, 0) == -1) throw SysError("cannot clear the set of supplementary groups"); - + if (setgid(buildUser.getGID()) == -1 || getgid() != buildUser.getGID() || getegid() != buildUser.getGID()) @@ -1948,10 +1936,10 @@ void DerivationGoal::initChild() getuid() != buildUser.getUID() || geteuid() != buildUser.getUID()) throw SysError("setuid failed"); - + } else { /* Let the setuid helper take care of it. */ - program = nixLibexecDir + "/nix-setuid-helper"; + program = settings.nixLibexecDir + "/nix-setuid-helper"; args.push_back(program.c_str()); args.push_back("run-builder"); user = buildUser.getUser().c_str(); @@ -1959,7 +1947,7 @@ void DerivationGoal::initChild() args.push_back(drv.builder.c_str()); } } - + /* Fill in the arguments. */ string builderBasename = baseNameOf(drv.builder); args.push_back(builderBasename.c_str()); @@ -1974,7 +1962,7 @@ void DerivationGoal::initChild() throw SysError(format("executing `%1%'") % drv.builder); - + } catch (std::exception & e) { std::cerr << format("build error: %1%") % e.what() << std::endl; } @@ -2016,7 +2004,7 @@ void DerivationGoal::computeClosure() if (!worker.store.isValidPath(i->second.path)) allValid = false; if (allValid) return; } - + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ @@ -2031,18 +2019,18 @@ void DerivationGoal::computeClosure() struct stat st; if (lstat(path.c_str(), &st) == -1) throw SysError(format("getting attributes of path `%1%'") % path); - + startNest(nest, lvlTalkative, format("scanning for references inside `%1%'") % path); /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified - hash). */ + hash). */ if (i->second.hash != "") { bool recursive; HashType ht; Hash h; i->second.parseHashInfo(recursive, ht, h); - + if (!recursive) { /* The output path should be a regular file without execute permission. */ @@ -2061,12 +2049,12 @@ void DerivationGoal::computeClosure() } /* Get rid of all weird permissions. */ - canonicalisePathMetaData(path); + canonicalisePathMetaData(path); - /* For this output path, find the references to other paths - contained in it. Compute the SHA-256 NAR hash at the same - time. The hash is stored in the database so that we can - verify later on whether nobody has messed with the store. */ + /* For this output path, find the references to other paths + contained in it. Compute the SHA-256 NAR hash at the same + time. The hash is stored in the database so that we can + verify later on whether nobody has messed with the store. */ HashResult hash; PathSet references = scanForReferences(path, allPaths, hash); contentHashes[path] = hash; @@ -2119,13 +2107,13 @@ string drvsLogDir = "drvs"; Path DerivationGoal::openLogFile() { - if (!queryBoolSetting("build-keep-log", true)) return ""; - + if (!settings.keepLog) return ""; + /* Create a log file. */ - Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str(); + Path dir = (format("%1%/%2%") % settings.nixLogDir % drvsLogDir).str(); createDirs(dir); - if (queryBoolSetting("build-compress-log", true)) { + if (settings.compressLog) { Path logFileName = (format("%1%/%2%.bz2") % dir % baseNameOf(drvPath)).str(); AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); @@ -2172,9 +2160,9 @@ void DerivationGoal::closeLogFile() void DerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { - if (keepFailed && !force) { - printMsg(lvlError, - format("builder for `%1%' failed; keeping build directory `%2%'") + if (settings.keepFailed && !force) { + printMsg(lvlError, + format("builder for `%1%' failed; keeping build directory `%2%'") % drvPath % tmpDir); if (buildUser.enabled() && !amPrivileged()) getOwnership(tmpDir); @@ -2192,7 +2180,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) if ((hook && fd == hook->builderOut.readSide) || (!hook && fd == builderOut.readSide)) { - if (verbosity >= buildVerbosity) + if (verbosity >= settings.buildVerbosity) writeToStderr((unsigned char *) data.data(), data.size()); if (bzLogFile) { int err; @@ -2228,15 +2216,15 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid) bool DerivationGoal::pathFailed(const Path & path) { - if (!worker.cacheFailure) return false; - + if (!settings.cacheFailure) return false; + if (!worker.store.hasPathFailed(path)) return false; printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path); - - if (printBuildTrace) + + if (settings.printBuildTrace) printMsg(lvlError, format("@ build-failed %1% %2% cached") % drvPath % path); - + worker.permanentFailure = true; amDone(ecFailed); @@ -2250,7 +2238,7 @@ bool DerivationGoal::pathFailed(const Path & path) class SubstitutionGoal : public Goal { friend class Worker; - + private: /* The store path that should be realised through a substitute. */ Path storePath; @@ -2261,10 +2249,16 @@ private: /* The current substituter. */ Path sub; + /* Whether any substituter can realise this path */ + bool hasSubstitute; + /* Path info returned by the substituter's query info operation. */ SubstitutablePathInfo info; - /* Pipe for the substitute's standard output/error. */ + /* Pipe for the substituter's standard output. */ + Pipe outPipe; + + /* Pipe for the substituter's standard error. */ Pipe logPipe; /* The process ID of the builder. */ @@ -2272,7 +2266,7 @@ private: /* Lock on the store path. */ boost::shared_ptr<PathLocks> outputLock; - + typedef void (SubstitutionGoal::*GoalState)(); GoalState state; @@ -2281,7 +2275,7 @@ public: ~SubstitutionGoal(); void cancel(); - + void work(); /* The states. */ @@ -2302,6 +2296,7 @@ public: SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker) : Goal(worker) + , hasSubstitute(false) { this->storePath = storePath; state = &SubstitutionGoal::init; @@ -2341,18 +2336,18 @@ void SubstitutionGoal::init() trace("init"); worker.store.addTempRoot(storePath); - + /* If the path already exists we're done. */ if (worker.store.isValidPath(storePath)) { amDone(ecSuccess); return; } - if (readOnlyMode) + if (settings.readOnlyMode) throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath); - subs = substituters; - + subs = settings.substituters; + tryNext(); } @@ -2365,17 +2360,23 @@ void SubstitutionGoal::tryNext() /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); - amDone(ecFailed); + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(hasSubstitute ? ecFailed : ecNoSubstituters); return; } sub = subs.front(); subs.pop_front(); - if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) { - tryNext(); - return; - } + SubstitutablePathInfos infos; + PathSet dummy(singleton<PathSet>(storePath)); + worker.store.querySubstitutablePathInfos(sub, dummy, infos); + SubstitutablePathInfos::iterator k = infos.find(storePath); + if (k == infos.end()) { tryNext(); return; } + info = k->second; + hasSubstitute = true; /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ @@ -2417,7 +2418,7 @@ void SubstitutionGoal::tryToRun() is maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= (maxBuildJobs == 0 ? 1 : maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } @@ -2432,7 +2433,7 @@ void SubstitutionGoal::tryToRun() worker.waitForAnyGoal(shared_from_this()); return; /* restart in the tryToRun() state when another goal finishes */ } - + /* Acquire a lock on the output path. */ outputLock = boost::shared_ptr<PathLocks>(new PathLocks); if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { @@ -2449,7 +2450,8 @@ void SubstitutionGoal::tryToRun() } printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); - + + outPipe.create(); logPipe.create(); /* Remove the (stale) output path if it exists. */ @@ -2459,17 +2461,24 @@ void SubstitutionGoal::tryToRun() /* Fork the substitute program. */ pid = fork(); switch (pid) { - + case -1: throw SysError("unable to fork"); case 0: try { /* child */ - logPipe.readSide.close(); - commonChildInit(logPipe); + if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("cannot dup output pipe into stdout"); + outPipe.readSide.close(); + outPipe.writeSide.close(); + + /* Pass configuration options (including those overriden + with --option) to the substituter. */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + /* Fill in the arguments. */ Strings args; args.push_back(baseNameOf(sub)); @@ -2478,25 +2487,26 @@ void SubstitutionGoal::tryToRun() const char * * argArr = strings2CharPtrs(args); execv(sub.c_str(), (char * *) argArr); - + throw SysError(format("executing `%1%'") % sub); - + } catch (std::exception & e) { std::cerr << format("substitute error: %1%") % e.what() << std::endl; } quickExit(1); } - + /* parent */ pid.setSeparatePG(true); pid.setKillSignal(SIGTERM); + outPipe.writeSide.close(); logPipe.writeSide.close(); worker.childStarted(shared_from_this(), pid, singleton<set<int> >(logPipe.readSide), true, true); state = &SubstitutionGoal::finished; - if (printBuildTrace) { + if (settings.printBuildTrace) { printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); } @@ -2518,27 +2528,47 @@ void SubstitutionGoal::finished() /* Close the read side of the logger pipe. */ logPipe.readSide.close(); - debug(format("substitute for `%1%' finished") % storePath); + /* Get the hash info from stdout. */ + string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; + outPipe.readSide.close(); /* Check the exit status and the build result. */ + HashResult hash; try { - + if (!statusOk(status)) throw SubstError(format("fetching path `%1%' %2%") % storePath % statusToString(status)); if (!pathExists(storePath)) throw SubstError(format("substitute did not produce path `%1%'") % storePath); - + + hash = hashPath(htSHA256, storePath); + + /* Verify the expected hash we got from the substituer. */ + if (expectedHashStr != "") { + size_t n = expectedHashStr.find(':'); + if (n == string::npos) + throw Error(format("bad hash from substituter: %1%") % expectedHashStr); + HashType hashType = parseHashType(string(expectedHashStr, 0, n)); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr); + Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); + Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, storePath).first; + if (expectedHash != actualHash) + throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%") + % storePath % printHash(expectedHash) % printHash(actualHash)); + } + } catch (SubstError & e) { printMsg(lvlInfo, e.msg()); - - if (printBuildTrace) { + + if (settings.printBuildTrace) { printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") % storePath % status % e.msg()); } - + /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); @@ -2547,10 +2577,8 @@ void SubstitutionGoal::finished() canonicalisePathMetaData(storePath); - HashResult hash = hashPath(htSHA256, storePath); - worker.store.optimisePath(storePath); // FIXME: combine with hashPath() - + ValidPathInfo info2; info2.path = storePath; info2.hash = hash.first; @@ -2560,14 +2588,14 @@ void SubstitutionGoal::finished() worker.store.registerValidPath(info2); outputLock->setDeletion(true); - + printMsg(lvlChatty, format("substitution of path `%1%' succeeded") % storePath); - if (printBuildTrace) { + if (settings.printBuildTrace) { printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); } - + amDone(ecSuccess); } @@ -2575,7 +2603,7 @@ void SubstitutionGoal::finished() void SubstitutionGoal::handleChildOutput(int fd, const string & data) { assert(fd == logPipe.readSide); - if (verbosity >= buildVerbosity) + if (verbosity >= settings.buildVerbosity) writeToStderr((unsigned char *) data.data(), data.size()); /* Don't write substitution output to a log file for now. We probably should, though. */ @@ -2598,12 +2626,11 @@ static bool working = false; Worker::Worker(LocalStore & store) : store(store) { - /* Debugging: prevent recursive workers. */ + /* Debugging: prevent recursive workers. */ if (working) abort(); working = true; nrLocalBuilds = 0; lastWokenUp = 0; - cacheFailure = queryBoolSetting("build-cache-failure", false); permanentFailure = false; } @@ -2668,7 +2695,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !keepGoing) + if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } @@ -2713,7 +2740,7 @@ void Worker::childStarted(GoalPtr goal, void Worker::childTerminated(pid_t pid, bool wakeSleepers) { assert(pid != -1); /* common mistake */ - + Children::iterator i = children.find(pid); assert(i != children.end()); @@ -2725,7 +2752,7 @@ void Worker::childTerminated(pid_t pid, bool wakeSleepers) children.erase(pid); if (wakeSleepers) { - + /* Wake up goals waiting for a build slot. */ foreach (WeakGoals::iterator, i, wantingToBuild) { GoalPtr goal = i->lock(); @@ -2740,7 +2767,7 @@ void Worker::childTerminated(pid_t pid, bool wakeSleepers) void Worker::waitForBuildSlot(GoalPtr goal) { debug("wait for build slot"); - if (getNrLocalBuilds() < maxBuildJobs) + if (getNrLocalBuilds() < settings.maxBuildJobs) wakeUp(goal); /* we can do it right away */ else wantingToBuild.insert(goal); @@ -2764,7 +2791,7 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i); - + startNest(nest, lvlDebug, format("entered goal loop")); while (1) { @@ -2789,7 +2816,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && maxBuildJobs == 0) throw Error( + if (awake.empty() && settings.maxBuildJobs == 0) throw Error( "unable to start any build; either increase `--max-jobs' " "or enable distributed builds"); assert(!awake.empty()); @@ -2799,9 +2826,9 @@ void Worker::run(const Goals & _topGoals) /* If --keep-going is not set, it's possible that the main goal exited while some of its subgoals were still active. But if --keep-going *is* set, then they must all be finished now. */ - assert(!keepGoing || awake.empty()); - assert(!keepGoing || wantingToBuild.empty()); - assert(!keepGoing || children.empty()); + assert(!settings.keepGoing || awake.empty()); + assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || children.empty()); } @@ -2821,15 +2848,15 @@ void Worker::waitForInput() time_t before = time(0); /* If a global timeout has been set, sleep until it's done. */ - if (buildTimeout != 0) { - useTimeout = true; + if (settings.buildTimeout != 0) { + useTimeout = true; if (lastWait == 0 || lastWait > before) lastWait = before; - timeout.tv_sec = std::max((time_t) 0, lastWait + buildTimeout - before); + timeout.tv_sec = std::max((time_t) 0, lastWait + settings.buildTimeout - before); } /* If we're monitoring for silence on stdout/stderr, sleep until the first deadline for any child. */ - if (maxSilentTime != 0) { + if (settings.maxSilentTime != 0) { time_t oldest = 0; foreach (Children::iterator, i, children) { if (i->second.monitorForSilence) { @@ -2838,10 +2865,10 @@ void Worker::waitForInput() } } if (oldest) { - time_t silenceTimeout = std::max((time_t) 0, oldest + maxSilentTime - before); + time_t silenceTimeout = std::max((time_t) 0, oldest + settings.maxSilentTime - before); timeout.tv_sec = useTimeout - ? std::min(silenceTimeout, timeout.tv_sec) - : silenceTimeout; + ? std::min(silenceTimeout, timeout.tv_sec) + : silenceTimeout; useTimeout = true; printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); } @@ -2849,14 +2876,12 @@ void Worker::waitForInput() /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ - int wakeUpInterval = queryIntSetting("build-poll-interval", 5); - if (!waitingForAWhile.empty()) { useTimeout = true; if (lastWokenUp == 0) printMsg(lvlError, "waiting for locks or build slots..."); if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before); + timeout.tv_sec = std::max((time_t) 0, lastWokenUp + settings.pollInterval - before); } else lastWokenUp = 0; using namespace std; @@ -2891,7 +2916,7 @@ void Worker::waitForInput() cancel(). */ set<pid_t> pids; foreach (Children::iterator, i, children) pids.insert(i->first); - + foreach (set<pid_t>::iterator, i, pids) { checkInterrupt(); Children::iterator j = children.find(*i); @@ -2922,27 +2947,27 @@ void Worker::waitForInput() } } - if (maxSilentTime != 0 && + if (settings.maxSilentTime != 0 && j->second.monitorForSilence && - after - j->second.lastOutput >= (time_t) maxSilentTime) + after - j->second.lastOutput >= (time_t) settings.maxSilentTime) { printMsg(lvlError, format("%1% timed out after %2% seconds of silence") - % goal->getName() % maxSilentTime); + % goal->getName() % settings.maxSilentTime); goal->cancel(); } - if (buildTimeout != 0 && - after - before >= (time_t) buildTimeout) + if (settings.buildTimeout != 0 && + after - before >= (time_t) settings.buildTimeout) { printMsg(lvlError, format("%1% timed out after %2% seconds of activity") - % goal->getName() % buildTimeout); + % goal->getName() % settings.buildTimeout); goal->cancel(); } } - if (!waitingForAWhile.empty() && lastWokenUp + wakeUpInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { lastWokenUp = after; foreach (WeakGoals::iterator, i, waitingForAWhile) { GoalPtr goal = i->lock(); @@ -2985,7 +3010,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths) if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath()); } - + if (!failed.empty()) throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); } @@ -3006,5 +3031,5 @@ void LocalStore::ensurePath(const Path & path) throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus()); } - + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 97343d57d4..73047c7538 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -12,7 +12,7 @@ void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash { recursive = false; string algo = hashAlgo; - + if (string(algo, 0, 2) == "r:") { recursive = true; algo = string(algo, 2); @@ -21,7 +21,7 @@ void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash hashType = parseHashType(algo); if (hashType == htUnknown) throw Error(format("unknown hash algorithm `%1%'") % algo); - + hash = parseHash(hashType, this->hash); } @@ -38,7 +38,7 @@ Path writeDerivation(StoreAPI & store, held during a garbage collection). */ string suffix = name + drvExtension; string contents = unparseDerivation(drv); - return readOnlyMode + return settings.readOnlyMode ? computeStorePathForText(suffix, contents, references) : store.addTextToStore(suffix, contents, references); } @@ -51,7 +51,7 @@ static Path parsePath(std::istream & str) throw Error(format("bad path `%1%' in derivation") % s); return s; } - + static StringSet parseStrings(std::istream & str, bool arePaths) { @@ -60,7 +60,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) res.insert(arePaths ? parsePath(str) : parseString(str)); return res; } - + Derivation parseDerivation(const string & s) { @@ -106,7 +106,7 @@ Derivation parseDerivation(const string & s) expect(str, ")"); drv.env[name] = value; } - + expect(str, ")"); return drv; } @@ -165,7 +165,7 @@ string unparseDerivation(const Derivation & drv) s += "],"; printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end()); - + s += ','; printString(s, drv.platform); s += ','; printString(s, drv.builder); s += ','; printStrings(s, drv.args.begin(), drv.args.end()); @@ -178,9 +178,9 @@ string unparseDerivation(const Derivation & drv) s += ','; printString(s, i->second); s += ')'; } - + s += "])"; - + return s; } @@ -190,7 +190,7 @@ bool isDerivation(const string & fileName) return hasSuffix(fileName, drvExtension); } - + bool isFixedOutputDrv(const Derivation & drv) { return drv.outputs.size() == 1 && @@ -247,7 +247,7 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv) inputs2[printHash(h)] = i->second; } drv.inputDrvs = inputs2; - + return hashString(htSHA256, unparseDerivation(drv)); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 874efe4d32..1355702f87 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -34,10 +34,10 @@ static const int defaultGcLevel = 1000; int LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") - % nixStateDir % gcLockName).str(); - + % settings.nixStateDir % gcLockName).str(); + debug(format("acquiring global GC lock `%1%'") % fnGCLock); - + AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600); if (fdGCLock == -1) throw SysError(format("opening global GC lock `%1%'") % fnGCLock); @@ -51,7 +51,7 @@ int LocalStore::openGCLock(LockType lockType) /* !!! Restrict read permission on the GC root. Otherwise any process that can open the file for reading can DoS the collector. */ - + return fdGCLock.borrow(); } @@ -85,7 +85,7 @@ void LocalStore::addIndirectRoot(const Path & path) { string hash = printHash32(hashString(htSHA1, path)); Path realRoot = canonPath((format("%1%/%2%/auto/%3%") - % nixStateDir % gcRootsDir % hash).str()); + % settings.nixStateDir % gcRootsDir % hash).str()); createSymlink(realRoot, path); } @@ -113,15 +113,15 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, else { if (!allowOutsideRootsDir) { - Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str()); - + Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str()); + if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") throw Error(format( "path `%1%' is not a valid garbage collector root; " "it's not in the directory `%2%'") % gcRoot % rootsDir); } - + createSymlink(gcRoot, storePath); } @@ -130,10 +130,10 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, Instead of reading all the roots, it would be more efficient to check if the root is in a directory in or linked from the gcroots directory. */ - if (queryBoolSetting("gc-check-reachability", false)) { + if (settings.checkRootReachability) { Roots roots = store.findRoots(); if (roots.find(gcRoot) == roots.end()) - printMsg(lvlError, + printMsg(lvlError, format( "warning: `%1%' is not in a directory where the garbage collector looks for roots; " "therefore, `%2%' might be removed by the garbage collector") @@ -144,7 +144,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, progress. This prevents the set of permanent roots from increasing while a GC is in progress. */ store.syncWithGC(); - + return gcRoot; } @@ -160,23 +160,23 @@ void LocalStore::addTempRoot(const Path & path) if (fdTempRoots == -1) { while (1) { - Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str(); + Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); createDirs(dir); - + fnTempRoots = (format("%1%/%2%") % dir % getpid()).str(); AutoCloseFD fdGCLock = openGCLock(ltRead); - + if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ unlink(fnTempRoots.c_str()); - fdTempRoots = openLockFile(fnTempRoots, true); + fdTempRoots = openLockFile(fnTempRoots, true); fdGCLock.close(); - + debug(format("acquiring read lock on `%1%'") % fnTempRoots); lockFile(fdTempRoots, ltRead, true); @@ -186,7 +186,7 @@ void LocalStore::addTempRoot(const Path & path) if (fstat(fdTempRoots, &st) == -1) throw SysError(format("statting `%1%'") % fnTempRoots); if (st.st_size == 0) break; - + /* The garbage collector deleted this file before we could get a lock. (It won't delete the file after we get a lock.) Try again. */ @@ -218,7 +218,7 @@ void removeTempRoots() /* Automatically clean up the temporary roots file when we exit. */ -struct RemoveTempRoots +struct RemoveTempRoots { ~RemoveTempRoots() { @@ -238,10 +238,10 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds) /* Read the `temproots' directory for per-process temporary root files. */ Strings tempRootFiles = readDirectory( - (format("%1%/%2%") % nixStateDir % tempRootsDir).str()); + (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str()); foreach (Strings::iterator, i, tempRootFiles) { - Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str(); + Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str(); debug(format("reading temporary root file `%1%'") % path); FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666))); @@ -295,7 +295,7 @@ static void findRoots(StoreAPI & store, const Path & path, bool recurseSymlinks, bool deleteStale, Roots & roots) { try { - + struct stat st; if (lstat(path.c_str(), &st) == -1) throw SysError(format("statting `%1%'") % path); @@ -315,7 +315,7 @@ static void findRoots(StoreAPI & store, const Path & path, debug(format("found root `%1%' in `%2%'") % target % path); Path storePath = toStorePath(target); - if (store.isValidPath(storePath)) + if (store.isValidPath(storePath)) roots[path] = storePath; else printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") @@ -350,7 +350,7 @@ static void findRoots(StoreAPI & store, const Path & path, static Roots findRoots(StoreAPI & store, bool deleteStale) { Roots roots; - Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str()); + Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str()); findRoots(store, rootsDir, true, deleteStale, roots); return roots; } @@ -365,16 +365,16 @@ Roots LocalStore::findRoots() static void addAdditionalRoots(StoreAPI & store, PathSet & roots) { Path rootFinder = getEnv("NIX_ROOT_FINDER", - nixLibexecDir + "/nix/find-runtime-roots.pl"); + settings.nixLibexecDir + "/nix/find-runtime-roots.pl"); if (rootFinder.empty()) return; - + debug(format("executing `%1%' to find additional roots") % rootFinder); string result = runProgram(rootFinder); Strings paths = tokenizeString(result, "\n"); - + foreach (Strings::iterator, i, paths) { if (isInStore(*i)) { Path path = toStorePath(*i); @@ -557,7 +557,7 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } else printMsg(lvlTalkative, format("would delete `%1%'") % path); - + state.deleted.insert(path); if (state.options.action != GCOptions::gcReturnLive) state.results.paths.insert(path); @@ -605,10 +605,10 @@ void LocalStore::removeUnusedLinks() void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { GCState state(results); - state.options = options; - - state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false); - state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true); + state.options = options; + + state.gcKeepOutputs = settings.gcKeepOutputs; + state.gcKeepDerivations = settings.gcKeepDerivations; /* Using `--ignore-liveness' with `--delete' can have unintended consequences if `gc-keep-outputs' or `gc-keep-derivations' are @@ -618,7 +618,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) state.gcKeepOutputs = false; state.gcKeepDerivations = false; } - + /* Acquire the global GC root. This prevents a) New roots from being added. b) Processes from creating new temporary root files. */ @@ -659,18 +659,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (!tryToDelete(state, *i)) throw Error(format("cannot delete path `%1%' since it is still alive") % *i); } - + } else { - + if (shouldDelete(state.options.action)) printMsg(lvlError, format("deleting garbage...")); else printMsg(lvlError, format("determining live/dead paths...")); - + try { - AutoCloseDir dir = opendir(nixStore.c_str()); - if (!dir) throw SysError(format("opening directory `%1%'") % nixStore); + AutoCloseDir dir = opendir(settings.nixStore.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore); /* Read the store and immediately delete all paths that aren't valid. When using --max-freed etc., deleting @@ -684,14 +684,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; - Path path = nixStore + "/" + name; + Path path = settings.nixStore + "/" + name; if (isValidPath(path)) entries.push_back(path); else tryToDelete(state, path); } - dir.close(); + dir.close(); /* Now delete the unreachable valid paths. Randomise the order in which we delete entries to make the collector diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5c22f14066..c75ebdd0e3 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -10,36 +10,63 @@ namespace nix { -string nixStore = "/UNINIT"; -string nixDataDir = "/UNINIT"; -string nixLogDir = "/UNINIT"; -string nixStateDir = "/UNINIT"; -string nixDBPath = "/UNINIT"; -string nixConfDir = "/UNINIT"; -string nixLibexecDir = "/UNINIT"; -string nixBinDir = "/UNINIT"; - -bool keepFailed = false; -bool keepGoing = false; -bool tryFallback = false; -Verbosity buildVerbosity = lvlError; -unsigned int maxBuildJobs = 1; -unsigned int buildCores = 1; -bool readOnlyMode = false; -string thisSystem = "unset"; -time_t maxSilentTime = 0; -time_t buildTimeout = 0; -Paths substituters; -bool useBuildHook = true; -bool printBuildTrace = false; - - -static bool settingsRead = false; - -static std::map<string, Strings> settings; - -/* Overriden settings. */ -std::map<string, Strings> settingsCmdline; +Settings settings; + + +Settings::Settings() +{ + keepFailed = false; + keepGoing = false; + tryFallback = false; + buildVerbosity = lvlError; + maxBuildJobs = 1; + buildCores = 1; + readOnlyMode = false; + thisSystem = SYSTEM; + maxSilentTime = 0; + buildTimeout = 0; + useBuildHook = true; + printBuildTrace = false; + reservedSize = 1024 * 1024; + fsyncMetadata = true; + useSQLiteWAL = true; + syncBeforeRegistering = false; + useSubstitutes = true; + useChroot = false; + dirsInChroot.insert("/dev"); + dirsInChroot.insert("/dev/pts"); + impersonateLinux26 = false; + keepLog = true; + compressLog = true; + cacheFailure = false; + pollInterval = 5; + checkRootReachability = false; + gcKeepOutputs = false; + gcKeepDerivations = true; + autoOptimiseStore = true; + envKeepDerivations = false; +} + + +void Settings::processEnvironment() +{ + nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); + nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); + nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); + nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); + nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); + nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); + nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); + nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); + + string subs = getEnv("NIX_SUBSTITUTERS", "default"); + if (subs == "default") { + substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); + } else + substituters = tokenizeString(subs, ":"); +} string & at(Strings & ss, unsigned int n) @@ -50,7 +77,7 @@ string & at(Strings & ss, unsigned int n) } -static void readSettings() +void Settings::loadConfFile() { Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); if (!pathExists(settingsFile)) return; @@ -78,95 +105,103 @@ static void readSettings() Strings::iterator i = tokens.begin(); advance(i, 2); - settings[name] = Strings(i, tokens.end()); + settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow }; - - settings.insert(settingsCmdline.begin(), settingsCmdline.end()); - - settingsRead = true; } -Strings querySetting(const string & name, const Strings & def) +void Settings::set(const string & name, const string & value) { - if (!settingsRead) readSettings(); - std::map<string, Strings>::iterator i = settings.find(name); - return i == settings.end() ? def : i->second; + settings[name] = value; + overrides[name] = value; } -string querySetting(const string & name, const string & def) +void Settings::update() { - Strings defs; - defs.push_back(def); + get(tryFallback, "build-fallback"); + get(maxBuildJobs, "build-max-jobs"); + get(buildCores, "build-cores"); + get(thisSystem, "system"); + get(maxSilentTime, "build-max-silent-time"); + get(buildTimeout, "build-timeout"); + get(reservedSize, "gc-reserved-space"); + get(fsyncMetadata, "fsync-metadata"); + get(useSQLiteWAL, "use-sqlite-wal"); + get(syncBeforeRegistering, "sync-before-registering"); + get(useSubstitutes, "build-use-substitutes"); + get(buildUsersGroup, "build-users-group"); + get(useChroot, "build-use-chroot"); + get(dirsInChroot, "build-chroot-dirs"); + get(impersonateLinux26, "build-impersonate-linux-26"); + get(keepLog, "build-keep-log"); + get(compressLog, "build-compress-log"); + get(cacheFailure, "build-cache-failure"); + get(pollInterval, "build-poll-interval"); + get(checkRootReachability, "gc-check-reachability"); + get(gcKeepOutputs, "gc-keep-outputs"); + get(gcKeepDerivations, "gc-keep-derivations"); + get(autoOptimiseStore, "auto-optimise-store"); + get(envKeepDerivations, "env-keep-derivations"); +} - Strings value = querySetting(name, defs); - if (value.size() != 1) - throw Error(format("configuration option `%1%' should not be a list") % name); - return value.front(); +void Settings::get(string & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res = i->second; } -bool queryBoolSetting(const string & name, bool def) +void Settings::get(bool & res, const string & name) { - string v = querySetting(name, def ? "true" : "false"); - if (v == "true") return true; - else if (v == "false") return false; + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + if (i->second == "true") res = true; + else if (i->second == "false") res = false; else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'") - % name % v); + % name % i->second); } -unsigned int queryIntSetting(const string & name, unsigned int def) +void Settings::get(PathSet & res, const string & name) { - int n; - if (!string2Int(querySetting(name, int2String(def)), n) || n < 0) - throw Error(format("configuration setting `%1%' should have an integer value") % name); - return n; + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res.clear(); + Strings ss = tokenizeString(i->second); + res.insert(ss.begin(), ss.end()); } -void overrideSetting(const string & name, const Strings & value) +template<class N> void Settings::get(N & res, const string & name) { - if (settingsRead) settings[name] = value; - settingsCmdline[name] = value; + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + if (!string2Int(i->second, res)) + throw Error(format("configuration setting `%1%' should have an integer value") % name); } -void reloadSettings() +string Settings::pack() { - settingsRead = false; - settings.clear(); + string s; + foreach (SettingsMap::iterator, i, settings) { + if (i->first.find('\n') != string::npos || + i->first.find('=') != string::npos || + i->second.find('\n') != string::npos) + throw Error("illegal option name/value"); + s += i->first; s += '='; s += i->second; s += '\n'; + } + return s; } -void setDefaultsFromEnvironment() +Settings::SettingsMap Settings::getOverrides() { - /* Setup Nix paths. */ - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); - nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); - nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); - nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); - nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); - nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); - nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); - - string subs = getEnv("NIX_SUBSTITUTERS", "default"); - if (subs == "default") { - substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); - } else - substituters = tokenizeString(subs, ":"); - - /* Get some settings from the configuration file. */ - thisSystem = querySetting("system", SYSTEM); - maxBuildJobs = queryIntSetting("build-max-jobs", 1); - buildCores = queryIntSetting("build-cores", 1); - maxSilentTime = queryIntSetting("build-max-silent-time", 0); - buildTimeout = queryIntSetting("build-timeout", 0); + return overrides; } - + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1c0877a5e1..1fb196db29 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -2,118 +2,191 @@ #include "types.hh" +#include <map> + namespace nix { -/* Path names. */ +struct Settings { + + typedef std::map<string, string> SettingsMap; + + Settings(); + + void processEnvironment(); + + void loadConfFile(); + + void set(const string & name, const string & value); + + void update(); + + string pack(); + + SettingsMap getOverrides(); + + /* The directory where we store sources and derived files. */ + Path nixStore; + + Path nixDataDir; /* !!! fix */ + + /* The directory where we log various operations. */ + Path nixLogDir; + + /* The directory where state is stored. */ + Path nixStateDir; + + /* The directory where we keep the SQLite database. */ + Path nixDBPath; + + /* The directory where configuration files are stored. */ + Path nixConfDir; + + /* The directory where internal helper programs are stored. */ + Path nixLibexecDir; + + /* The directory where the main programs are stored. */ + Path nixBinDir; + + /* Whether to keep temporary directories of failed builds. */ + bool keepFailed; + + /* Whether to keep building subgoals when a sibling (another + subgoal of the same goal) fails. */ + bool keepGoing; + + /* Whether, if we cannot realise the known closure corresponding + to a derivation, we should try to normalise the derivation + instead. */ + bool tryFallback; + + /* Verbosity level for build output. */ + Verbosity buildVerbosity; -/* nixStore is the directory where we generally store atomic and - derived files. */ -extern string nixStore; + /* Maximum number of parallel build jobs. 0 means unlimited. */ + unsigned int maxBuildJobs; -extern string nixDataDir; /* !!! fix */ + /* Number of CPU cores to utilize in parallel within a build, + i.e. by passing this number to Make via '-j'. 0 means that the + number of actual CPU cores on the local host ought to be + auto-detected. */ + unsigned int buildCores; -/* nixLogDir is the directory where we log various operations. */ -extern string nixLogDir; + /* Read-only mode. Don't copy stuff to the store, don't change + the database. */ + bool readOnlyMode; -/* nixStateDir is the directory where state is stored. */ -extern string nixStateDir; + /* The canonical system name, as returned by config.guess. */ + string thisSystem; -/* nixDBPath is the path name of our Berkeley DB environment. */ -extern string nixDBPath; + /* The maximum time in seconds that a builer can go without + producing any output on stdout/stderr before it is killed. 0 + means infinity. */ + time_t maxSilentTime; -/* nixConfDir is the directory where configuration files are - stored. */ -extern string nixConfDir; + /* The maximum duration in seconds that a builder can run. 0 + means infinity. */ + time_t buildTimeout; -/* nixLibexecDir is the directory where internal helper programs are - stored. */ -extern string nixLibexecDir; + /* The substituters. There are programs that can somehow realise + a store path without building, e.g., by downloading it or + copying it from a CD. */ + Paths substituters; -/* nixBinDir is the directory where the main programs are stored. */ -extern string nixBinDir; + /* Whether to use build hooks (for distributed builds). Sometimes + users want to disable this from the command-line. */ + bool useBuildHook; + /* Whether buildDerivations() should print out lines on stderr in + a fixed format to allow its progress to be monitored. Each + line starts with a "@". The following are defined: -/* Misc. global flags. */ + @ build-started <drvpath> <outpath> <system> <logfile> + @ build-failed <drvpath> <outpath> <exitcode> <error text> + @ build-succeeded <drvpath> <outpath> + @ substituter-started <outpath> <substituter> + @ substituter-failed <outpath> <exitcode> <error text> + @ substituter-succeeded <outpath> -/* Whether to keep temporary directories of failed builds. */ -extern bool keepFailed; + Best combined with --no-build-output, otherwise stderr might + conceivably contain lines in this format printed by the + builders. */ + bool printBuildTrace; -/* Whether to keep building subgoals when a sibling (another subgoal - of the same goal) fails. */ -extern bool keepGoing; + /* Amount of reserved space for the garbage collector + (/nix/var/nix/db/reserved). */ + off_t reservedSize; -/* Whether, if we cannot realise the known closure corresponding to a - derivation, we should try to normalise the derivation instead. */ -extern bool tryFallback; + /* Whether SQLite should use fsync. */ + bool fsyncMetadata; -/* Verbosity level for build output. */ -extern Verbosity buildVerbosity; + /* Whether SQLite should use WAL mode. */ + bool useSQLiteWAL; -/* Maximum number of parallel build jobs. 0 means unlimited. */ -extern unsigned int maxBuildJobs; + /* Whether to call sync() before registering a path as valid. */ + bool syncBeforeRegistering; -/* Number of CPU cores to utilize in parallel within a build, i.e. by passing - this number to Make via '-j'. 0 means that the number of actual CPU cores on - the local host ought to be auto-detected. */ -extern unsigned int buildCores; + /* Whether to use substitutes. */ + bool useSubstitutes; -/* Read-only mode. Don't copy stuff to the store, don't change the - database. */ -extern bool readOnlyMode; + /* The Unix group that contains the build users. */ + string buildUsersGroup; -/* The canonical system name, as returned by config.guess. */ -extern string thisSystem; + /* Whether to build in chroot. */ + bool useChroot; -/* The maximum time in seconds that a builer can go without producing - any output on stdout/stderr before it is killed. 0 means - infinity. */ -extern time_t maxSilentTime; + /* The directories from the host filesystem to be included in the + chroot. */ + PathSet dirsInChroot; -/* The maximum duration in seconds that a builder can run. 0 means - infinity. */ -extern time_t buildTimeout; + /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ + bool impersonateLinux26; -/* The substituters. There are programs that can somehow realise a - store path without building, e.g., by downloading it or copying it - from a CD. */ -extern Paths substituters; + /* Whether to store build logs. */ + bool keepLog; -/* Whether to use build hooks (for distributed builds). Sometimes - users want to disable this from the command-line. */ -extern bool useBuildHook; + /* Whether to compress logs. */ + bool compressLog; -/* Whether buildDerivations() should print out lines on stderr in a - fixed format to allow its progress to be monitored. Each line - starts with a "@". The following are defined: + /* Whether to cache build failures. */ + bool cacheFailure; - @ build-started <drvpath> <outpath> <system> <logfile> - @ build-failed <drvpath> <outpath> <exitcode> <error text> - @ build-succeeded <drvpath> <outpath> - @ substituter-started <outpath> <substituter> - @ substituter-failed <outpath> <exitcode> <error text> - @ substituter-succeeded <outpath> + /* How often (in seconds) to poll for locks. */ + unsigned int pollInterval; - Best combined with --no-build-output, otherwise stderr might - conceivably contain lines in this format printed by the builders. -*/ -extern bool printBuildTrace; + /* Whether to check if new GC roots can in fact be found by the + garbage collector. */ + bool checkRootReachability; + /* Whether the garbage collector should keep outputs of live + derivations. */ + bool gcKeepOutputs; -Strings querySetting(const string & name, const Strings & def); + /* Whether the garbage collector should keep derivers of live + paths. */ + bool gcKeepDerivations; -string querySetting(const string & name, const string & def); + /* Whether to automatically replace files with identical contents + with hard links. */ + bool autoOptimiseStore; -bool queryBoolSetting(const string & name, bool def); + /* Whether to add derivations as a dependency of user environments + (to prevent them from being GCed). */ + bool envKeepDerivations; -unsigned int queryIntSetting(const string & name, unsigned int def); +private: + SettingsMap settings, overrides; -void overrideSetting(const string & name, const Strings & value); + void get(string & res, const string & name); + void get(bool & res, const string & name); + void get(PathSet & res, const string & name); + template<class N> void get(N & res, const string & name); +}; -void reloadSettings(); -void setDefaultsFromEnvironment(); +// FIXME: don't use a global variable. +extern Settings settings; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 05b2b9c6e5..bd63ce55d0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,7 +6,7 @@ #include "worker-protocol.hh" #include "derivations.hh" #include "immutable.hh" - + #include <iostream> #include <algorithm> @@ -147,11 +147,11 @@ struct SQLiteStmtUse }; -struct SQLiteTxn +struct SQLiteTxn { bool active; sqlite3 * db; - + SQLiteTxn(sqlite3 * db) : active(false) { this->db = db; if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) @@ -159,14 +159,14 @@ struct SQLiteTxn active = true; } - void commit() + void commit() { if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "committing transaction"); active = false; } - - ~SQLiteTxn() + + ~SQLiteTxn() { try { if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) @@ -181,7 +181,7 @@ struct SQLiteTxn void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; - Path path = nixStore; + Path path = settings.nixStore; struct stat st; while (path != "/") { if (lstat(path.c_str(), &st)) @@ -198,29 +198,27 @@ void checkStoreNotSymlink() LocalStore::LocalStore(bool reserveSpace) { - substitutablePathsLoaded = false; - - schemaPath = nixDBPath + "/schema"; - - if (readOnlyMode) { + schemaPath = settings.nixDBPath + "/schema"; + + if (settings.readOnlyMode) { openDB(false); return; } /* Create missing state directories if they don't already exist. */ - createDirs(nixStore); - createDirs(linksDir = nixStore + "/.links"); - Path profilesDir = nixStateDir + "/profiles"; - createDirs(nixStateDir + "/profiles"); - createDirs(nixStateDir + "/temproots"); - createDirs(nixDBPath); - Path gcRootsDir = nixStateDir + "/gcroots"; + createDirs(settings.nixStore); + createDirs(linksDir = settings.nixStore + "/.links"); + Path profilesDir = settings.nixStateDir + "/profiles"; + createDirs(settings.nixStateDir + "/profiles"); + createDirs(settings.nixStateDir + "/temproots"); + createDirs(settings.nixDBPath); + Path gcRootsDir = settings.nixStateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1) throw SysError(format("creating symlink to `%1%'") % profilesDir); } - + checkStoreNotSymlink(); /* We can't open a SQLite database if the disk is full. Since @@ -228,13 +226,12 @@ LocalStore::LocalStore(bool reserveSpace) needed, we reserve some dummy space that we can free just before doing a garbage collection. */ try { - Path reservedPath = nixDBPath + "/reserved"; + Path reservedPath = settings.nixDBPath + "/reserved"; if (reserveSpace) { - int reservedSize = queryIntSetting("gc-reserved-space", 1024 * 1024); struct stat st; if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != reservedSize) - writeFile(reservedPath, string(reservedSize, 'X')); + st.st_size != settings.reservedSize) + writeFile(reservedPath, string(settings.reservedSize, 'X')); } else deletePath(reservedPath); @@ -244,15 +241,15 @@ LocalStore::LocalStore(bool reserveSpace) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ try { - Path globalLockPath = nixDBPath + "/big-lock"; + Path globalLockPath = settings.nixDBPath + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); } catch (SysError & e) { if (e.errNo != EACCES) throw; - readOnlyMode = true; + settings.readOnlyMode = true; openDB(false); return; } - + if (!lockFile(globalLock, ltRead, false)) { printMsg(lvlError, "waiting for the big Nix store lock..."); lockFile(globalLock, ltRead, true); @@ -264,20 +261,20 @@ LocalStore::LocalStore(bool reserveSpace) if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); - + else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } - + else if (curSchema < nixSchemaVersion) { if (curSchema < 5) throw Error( "Your Nix store has a database in Berkeley DB format,\n" "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); - + if (!lockFile(globalLock, ltWrite, false)) { printMsg(lvlError, "waiting for exclusive access to the Nix store..."); lockFile(globalLock, ltWrite, true); @@ -293,7 +290,7 @@ LocalStore::LocalStore(bool reserveSpace) lockFile(globalLock, ltRead, true); } - + else openDB(false); } @@ -327,7 +324,7 @@ int LocalStore::getSchema() void LocalStore::openDB(bool create) { /* Open the Nix database. */ - if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, + if (sqlite3_open_v2((settings.nixDBPath + "/db.sqlite").c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error("cannot open SQLite database"); @@ -339,18 +336,18 @@ void LocalStore::openDB(bool create) /* !!! check whether sqlite has been built with foreign key support */ - + /* Whether SQLite should fsync(). "Normal" synchronous mode should be safe enough. If the user asks for it, don't sync at all. This can cause database corruption if the system crashes. */ - string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; + string syncMode = settings.fsyncMetadata ? "normal" : "off"; if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "setting synchronous mode"); /* Set the SQLite journal mode. WAL mode is fastest, so it's the default. */ - string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; + string mode = settings.useSQLiteWAL ? "wal" : "truncate"; string prevMode; { SQLiteStmt stmt; @@ -368,7 +365,7 @@ void LocalStore::openDB(bool create) derivation is done in a single fsync(). */ if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "setting autocheckpoint interval"); - + /* Initialise the database schema, if necessary. */ if (create) { #include "schema.sql.hh" @@ -423,7 +420,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + throw SysError(format("getting attributes of path `%1%'") % path); /* Really make sure that the path is of a supported type. This has already been checked in dumpPath(). */ @@ -451,7 +448,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) /* Mask out all type related bits. */ mode_t mode = st.st_mode & ~S_IFMT; - + if (mode != 0444 && mode != 0555) { mode = (st.st_mode & S_IFMT) | 0444 @@ -461,7 +458,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) } } - + if (st.st_mtime != mtimeStore) { struct timeval times[2]; times[0].tv_sec = st.st_atime; @@ -472,14 +469,14 @@ void canonicalisePathMetaData(const Path & path, bool recurse) if (lutimes(path.c_str(), times) == -1) #else if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif +#endif throw SysError(format("changing modification time of `%1%'") % path); } if (recurse && S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - canonicalisePathMetaData(path + "/" + *i, true); + foreach (Strings::iterator, i, names) + canonicalisePathMetaData(path + "/" + *i, true); } makeImmutable(path); @@ -494,7 +491,7 @@ void canonicalisePathMetaData(const Path & path) be a symlink, since we can't change its ownership. */ struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + throw SysError(format("getting attributes of path `%1%'") % path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); @@ -508,7 +505,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & string drvName = storePathToName(drvPath); assert(isDerivation(drvName)); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - + if (isFixedOutputDrv(drv)) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) @@ -532,7 +529,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } Hash h = hashDerivationModulo(*this, drvCopy); - + foreach (DerivationOutputs::const_iterator, i, drv.outputs) { Path outPath = makeOutputPath(i->first, h, drvName); StringPairs::const_iterator j = drv.env.find(i->first); @@ -568,14 +565,14 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che derivation. */ if (isDerivation(info.path)) { Derivation drv = parseDerivation(readFile(info.path)); - + /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - + foreach (DerivationOutputs::iterator, i, drv.outputs) { SQLiteStmtUse use(stmtAddDerivationOutput); stmtAddDerivationOutput.bind(id); @@ -681,7 +678,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) SQLiteStmtUse use1(stmtQueryPathInfo); stmtQueryPathInfo.bind(path); - + int r = sqlite3_step(stmtQueryPathInfo); if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); @@ -691,7 +688,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); assert(s); info.hash = parseHashField(path, s); - + info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); @@ -756,13 +753,22 @@ bool LocalStore::isValidPath(const Path & path) } -PathSet LocalStore::queryValidPaths() +PathSet LocalStore::queryValidPaths(const PathSet & paths) +{ + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath(*i)) res.insert(*i); + return res; +} + + +PathSet LocalStore::queryAllValidPaths() { SQLiteStmt stmt; stmt.create(db, "select path from ValidPaths"); - + PathSet res; - + int r; while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { const char * s = (const char *) sqlite3_column_text(stmt, 0); @@ -825,10 +831,10 @@ PathSet LocalStore::queryValidDerivers(const Path & path) assert(s); derivers.insert(s); } - + if (r != SQLITE_DONE) throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); - + return derivers; } @@ -836,10 +842,10 @@ PathSet LocalStore::queryValidDerivers(const Path & path) PathSet LocalStore::queryDerivationOutputs(const Path & path) { SQLiteTxn txn(db); - + SQLiteStmtUse use(stmtQueryDerivationOutputs); stmtQueryDerivationOutputs.bind(queryValidPathId(path)); - + PathSet outputs; int r; while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { @@ -847,7 +853,7 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) assert(s); outputs.insert(s); } - + if (r != SQLITE_DONE) throwSQLiteError(db, format("error getting outputs of `%1%'") % path); @@ -858,10 +864,10 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) StringSet LocalStore::queryDerivationOutputNames(const Path & path) { SQLiteTxn txn(db); - + SQLiteStmtUse use(stmtQueryDerivationOutputs); stmtQueryDerivationOutputs.bind(queryValidPathId(path)); - + StringSet outputNames; int r; while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { @@ -869,7 +875,7 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) assert(s); outputNames.insert(s); } - + if (r != SQLITE_DONE) throwSQLiteError(db, format("error getting output names of `%1%'") % path); @@ -880,11 +886,11 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) Path LocalStore::queryPathFromHashPart(const string & hashPart) { if (hashPart.size() != 32) throw Error("invalid hash part"); - + SQLiteTxn txn(db); - Path prefix = nixStore + "/" + hashPart; - + Path prefix = settings.nixStore + "/" + hashPart; + SQLiteStmtUse use(stmtQueryPathFromHashPart); stmtQueryPathFromHashPart.bind(prefix); @@ -900,16 +906,17 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) { if (run.pid != -1) return; - + debug(format("starting substituter program `%1%'") % substituter); - Pipe toPipe, fromPipe; - + Pipe toPipe, fromPipe, errorPipe; + toPipe.create(); fromPipe.create(); + errorPipe.create(); run.pid = fork(); - + switch (run.pid) { case -1: @@ -923,13 +930,19 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & library named libutil. As a result, substituters written in Perl (i.e. all of them) fail. */ unsetenv("DYLD_LIBRARY_PATH"); - + + /* Pass configuration options (including those overriden + with --option) to the substituter. */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + fromPipe.readSide.close(); toPipe.writeSide.close(); if (dup2(toPipe.readSide, STDIN_FILENO) == -1) throw SysError("dupping stdin"); if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("dupping stderr"); closeMostFDs(set<int>()); execl(substituter.c_str(), substituter.c_str(), "--query", NULL); throw SysError(format("executing `%1%'") % substituter); @@ -940,9 +953,10 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & } /* Parent. */ - + run.to = toPipe.writeSide.borrow(); run.from = fromPipe.readSide.borrow(); + run.error = errorPipe.readSide.borrow(); } @@ -955,50 +969,79 @@ template<class T> T getIntLine(int fd) } -bool LocalStore::hasSubstitutes(const Path & path) +PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { - foreach (Paths::iterator, i, substituters) { + PathSet res; + foreach (Paths::iterator, i, settings.substituters) { + if (res.size() == paths.size()) break; RunningSubstituter & run(runningSubstituters[*i]); startSubstituter(*i, run); - writeLine(run.to, "have\n" + path); - if (getIntLine<int>(run.from)) return true; + string s = "have "; + foreach (PathSet::const_iterator, j, paths) + if (res.find(*j) == res.end()) { s += *j; s += " "; } + writeLine(run.to, s); + while (true) { + /* FIXME: we only read stderr when an error occurs, so + substituters should only write (short) messages to + stderr when they fail. I.e. they shouldn't write debug + output. */ + try { + Path path = readLine(run.from); + if (path == "") break; + res.insert(path); + } catch (EndOfFile e) { + throw Error(format("substituter `%1%' failed: %2%") % *i % chomp(drainFD(run.error))); + } + } } - - return false; + return res; } -bool LocalStore::querySubstitutablePathInfo(const Path & substituter, - const Path & path, SubstitutablePathInfo & info) +void LocalStore::querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos) { RunningSubstituter & run(runningSubstituters[substituter]); startSubstituter(substituter, run); - writeLine(run.to, "info\n" + path); - - if (!getIntLine<int>(run.from)) return false; - - info.deriver = readLine(run.from); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLine<int>(run.from); - while (nrRefs--) { - Path p = readLine(run.from); - assertStorePath(p); - info.references.insert(p); + string s = "info "; + foreach (PathSet::const_iterator, i, paths) + if (infos.find(*i) == infos.end()) { s += *i; s += " "; } + writeLine(run.to, s); + + while (true) { + try { + Path path = readLine(run.from); + if (path == "") break; + if (paths.find(path) == paths.end()) + throw Error(format("got unexpected path `%1%' from substituter") % path); + paths.erase(path); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readLine(run.from); + if (info.deriver != "") assertStorePath(info.deriver); + int nrRefs = getIntLine<int>(run.from); + while (nrRefs--) { + Path p = readLine(run.from); + assertStorePath(p); + info.references.insert(p); + } + info.downloadSize = getIntLine<long long>(run.from); + info.narSize = getIntLine<long long>(run.from); + } catch (EndOfFile e) { + throw Error(format("substituter `%1%' failed: %2%") % substituter % chomp(drainFD(run.error))); + } } - info.downloadSize = getIntLine<long long>(run.from); - info.narSize = getIntLine<long long>(run.from); - - return true; } -bool LocalStore::querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info) +void LocalStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) { - foreach (Paths::iterator, i, substituters) - if (querySubstitutablePathInfo(*i, path, info)) return true; - return false; + PathSet todo = paths; + foreach (Paths::iterator, i, settings.substituters) { + if (todo.empty()) break; + querySubstitutablePathInfos(*i, todo, infos); + } } @@ -1018,17 +1061,16 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* sqlite will fsync by default, but the new valid paths may not be fsync-ed. + /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. * So some may want to fsync them before registering the validity, at the * expense of some speed of the path registering operation. */ - if (queryBoolSetting("sync-before-registering", false)) - sync(); + if (settings.syncBeforeRegistering) sync(); while (1) { try { SQLiteTxn txn(db); PathSet paths; - + foreach (ValidPathInfos::const_iterator, i, infos) { assert(i->hash.type == htSHA256); /* !!! Maybe the registration info should be updated if the @@ -1119,7 +1161,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, hash = hashPath(htSHA256, dstPath); optimisePath(dstPath); // FIXME: combine with hashPath() - + ValidPathInfo info; info.path = dstPath; info.hash = hash.first; @@ -1144,7 +1186,7 @@ Path LocalStore::addToStore(const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (recursive) dumpPath(srcPath, sink, filter); else sink.s = readFile(srcPath); @@ -1157,7 +1199,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s, const PathSet & references) { Path dstPath = computeStorePathForText(name, s, references); - + addTempRoot(dstPath); if (!isValidPath(dstPath)) { @@ -1175,7 +1217,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s, HashResult hash = hashPath(htSHA256, dstPath); optimisePath(dstPath); - + ValidPathInfo info; info.path = dstPath; info.hash = hash.first; @@ -1233,7 +1275,7 @@ void LocalStore::exportPath(const Path & path, bool sign, throw Error(format("path `%1%' is not valid") % path); HashAndWriteSink hashAndWriteSink(sink); - + dumpPath(path, hashAndWriteSink); /* Refuse to export paths that have changed. This prevents @@ -1248,7 +1290,7 @@ void LocalStore::exportPath(const Path & path, bool sign, writeInt(EXPORT_MAGIC, hashAndWriteSink); writeString(path, hashAndWriteSink); - + PathSet references; queryReferences(path, references); writeStrings(references, hashAndWriteSink); @@ -1258,15 +1300,15 @@ void LocalStore::exportPath(const Path & path, bool sign, if (sign) { Hash hash = hashAndWriteSink.currentHash(); - + writeInt(1, hashAndWriteSink); - + Path tmpDir = createTempDir(); AutoDelete delTmp(tmpDir); Path hashFile = tmpDir + "/hash"; writeFile(hashFile, printHash(hash)); - Path secretKey = nixConfDir + "/signing-key.sec"; + Path secretKey = settings.nixConfDir + "/signing-key.sec"; checkSecrecy(secretKey); Strings args; @@ -1279,7 +1321,7 @@ void LocalStore::exportPath(const Path & path, bool sign, string signature = runProgram(OPENSSL_PATH, true, args); writeString(signature, hashAndWriteSink); - + } else writeInt(0, hashAndWriteSink); } @@ -1312,7 +1354,7 @@ Path LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and addTempRoot(), so repeat until `tmpDir' exists. */ - tmpDir = createTempDir(nixStore); + tmpDir = createTempDir(settings.nixStore); addTempRoot(tmpDir); } while (!pathExists(tmpDir)); return tmpDir; @@ -1322,7 +1364,7 @@ Path LocalStore::createTempDirInStore() Path LocalStore::importPath(bool requireSignature, Source & source) { HashAndReadSource hashAndReadSource(source); - + /* We don't yet know what store path this archive contains (the store path follows the archive data proper), and besides, we don't know yet whether the signature is valid. */ @@ -1352,7 +1394,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) if (requireSignature && !haveSignature) throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); - + if (haveSignature) { string signature = readString(hashAndReadSource); @@ -1364,7 +1406,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) args.push_back("rsautl"); args.push_back("-verify"); args.push_back("-inkey"); - args.push_back(nixConfDir + "/signing-key.pub"); + args.push_back(settings.nixConfDir + "/signing-key.pub"); args.push_back("-pubin"); args.push_back("-in"); args.push_back(sigFile); @@ -1406,13 +1448,13 @@ Path LocalStore::importPath(bool requireSignature, Source & source) % unpacked % dstPath); canonicalisePathMetaData(dstPath); - + /* !!! if we were clever, we could prevent the hashPath() here. */ HashResult hash = hashPath(htSHA256, dstPath); optimisePath(dstPath); // FIXME: combine with hashPath() - + ValidPathInfo info; info.path = dstPath; info.hash = hash.first; @@ -1421,10 +1463,10 @@ Path LocalStore::importPath(bool requireSignature, Source & source) info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; registerValidPath(info); } - + outputLock.setDeletion(true); } - + return dstPath; } @@ -1472,14 +1514,14 @@ void LocalStore::verifyStore(bool checkContents) /* Acquire the global GC lock to prevent a garbage collection. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); - - Paths entries = readDirectory(nixStore); + + Paths entries = readDirectory(settings.nixStore); PathSet store(entries.begin(), entries.end()); /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence..."); - PathSet validPaths2 = queryValidPaths(), validPaths, done; + PathSet validPaths2 = queryAllValidPaths(), validPaths, done; foreach (PathSet::iterator, i, validPaths2) verifyPath(*i, store, done, validPaths); @@ -1501,7 +1543,7 @@ void LocalStore::verifyStore(bool checkContents) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); HashResult current = hashPath(info.hash.type, *i); - + if (info.hash != nullHash && info.hash != current.first) { printMsg(lvlError, format("path `%1%' was modified! " "expected hash `%2%', got `%3%'") @@ -1516,18 +1558,18 @@ void LocalStore::verifyStore(bool checkContents) info.hash = current.first; update = true; } - + /* Fill in missing narSize fields (from old stores). */ if (info.narSize == 0) { printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); info.narSize = current.second; - update = true; + update = true; } if (update) updatePathInfo(info); } - + } catch (Error & e) { /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ @@ -1543,7 +1585,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths) { checkInterrupt(); - + if (done.find(path) != done.end()) return; done.insert(path); @@ -1570,10 +1612,10 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, invalidatePath(path); } else printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path); - + return; } - + validPaths.insert(path); } @@ -1583,9 +1625,9 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, PathSet LocalStore::queryValidPathsOld() { PathSet paths; - Strings entries = readDirectory(nixDBPath + "/info"); + Strings entries = readDirectory(settings.nixDBPath + "/info"); foreach (Strings::iterator, i, entries) - if (i->at(0) != '.') paths.insert(nixStore + "/" + *i); + if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i); return paths; } @@ -1597,7 +1639,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) /* Read the info file. */ string baseName = baseNameOf(path); - Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str(); + Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); if (!pathExists(infoFile)) throw Error(format("path `%1%' is not valid") % path); string info = readFile(infoFile); @@ -1639,14 +1681,14 @@ void LocalStore::upgradeStore6() PathSet validPaths = queryValidPathsOld(); SQLiteTxn txn(db); - + foreach (PathSet::iterator, i, validPaths) { addValidPath(queryPathInfoOld(*i), false); std::cerr << "."; } std::cerr << "|"; - + foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); unsigned long long referrer = queryValidPathId(*i); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 50910f353a..3cb016e9ca 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -45,7 +45,7 @@ struct OptimiseStats struct RunningSubstituter { Pid pid; - AutoCloseFD to, from; + AutoCloseFD to, from, error; }; @@ -75,19 +75,16 @@ struct SQLiteStmt void bind64(long long value); void bind(); }; - + class LocalStore : public StoreAPI { private: - bool substitutablePathsLoaded; - PathSet substitutablePaths; - typedef std::map<Path, RunningSubstituter> RunningSubstituters; RunningSubstituters runningSubstituters; Path linksDir; - + public: /* Initialise the local store, upgrading the schema if @@ -95,13 +92,15 @@ public: LocalStore(bool reserveSpace = true); ~LocalStore(); - + /* Implementations of abstract store API methods. */ - + bool isValidPath(const Path & path); - PathSet queryValidPaths(); - + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); Hash queryPathHash(const Path & path); @@ -121,19 +120,17 @@ public: PathSet queryDerivationOutputs(const Path & path); StringSet queryDerivationOutputNames(const Path & path); - + Path queryPathFromHashPart(const string & hashPart); - - PathSet querySubstitutablePaths(); - - bool hasSubstitutes(const Path & path); - - bool querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info); - - bool querySubstitutablePathInfo(const Path & substituter, - const Path & path, SubstitutablePathInfo & info); - + + PathSet querySubstitutablePaths(const PathSet & paths); + + void querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos); + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); + Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter); @@ -152,7 +149,7 @@ public: Sink & sink); Paths importPaths(bool requireSignature, Source & source); - + void buildPaths(const PathSet & paths); void ensurePath(const Path & path); @@ -160,7 +157,7 @@ public: void addTempRoot(const Path & path); void addIndirectRoot(const Path & path); - + void syncWithGC(); Roots findRoots(); @@ -173,7 +170,7 @@ public: /* Optimise a single store path. */ void optimisePath(const Path & path); - + /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); @@ -232,18 +229,18 @@ private: unsigned long long queryValidPathId(const Path & path); unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); - + void addReference(unsigned long long referrer, unsigned long long reference); - + void appendReferrer(const Path & from, const Path & to, bool lock); - + void rewriteReferrers(const Path & path, bool purge, PathSet referrers); void invalidatePath(const Path & path); /* Delete a path from the Nix store. */ void invalidatePathChecked(const Path & path); - + void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths); @@ -256,14 +253,14 @@ private: struct GCState; void deleteGarbage(GCState & state, const Path & path); - + bool tryToDelete(GCState & state, const Path & path); - + bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - + int openGCLock(LockType lockType); - + void removeUnusedLinks(); void startSubstituter(const Path & substituter, @@ -272,7 +269,7 @@ private: Path createTempDirInStore(); Path importPath(bool requireSignature, Source & source); - + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); void optimisePath_(OptimiseStats & stats, const Path & path); @@ -293,9 +290,6 @@ void canonicalisePathMetaData(const Path & path, bool recurse); MakeError(PathInUse, Error); -/* Whether we are in build users mode. */ -bool haveBuildUsers(); - /* Whether we are root. */ bool amPrivileged(); @@ -308,5 +302,5 @@ void deletePathWrapped(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed); void deletePathWrapped(const Path & path); - + } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0934999363..3ce300e306 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -52,68 +52,118 @@ void queryMissing(StoreAPI & store, const PathSet & targets, unsigned long long & downloadSize, unsigned long long & narSize) { downloadSize = narSize = 0; - + PathSet todo(targets.begin(), targets.end()), done; + /* Getting substitute info has high latency when using the binary + cache substituter. Thus it's essential to do substitute + queries in parallel as much as possible. To accomplish this + we do the following: + + - For all paths still to be processed (‘todo’), we add all + paths for which we need info to the set ‘query’. For an + unbuilt derivation this is the output paths; otherwise, it's + the path itself. + + - We get info about all paths in ‘query’ in parallel. + + - We process the results and add new items to ‘todo’ if + necessary. E.g. if a path is substitutable, then we need to + get info on its references. + + - Repeat until ‘todo’ is empty. + */ + while (!todo.empty()) { - Path p = *(todo.begin()); - todo.erase(p); - if (done.find(p) != done.end()) continue; - done.insert(p); - - if (isDerivation(p)) { - if (!store.isValidPath(p)) { - unknown.insert(p); - continue; + + PathSet query, todoDrv, todoNonDrv; + + foreach (PathSet::iterator, i, todo) { + if (done.find(*i) != done.end()) continue; + done.insert(*i); + + if (isDerivation(*i)) { + if (!store.isValidPath(*i)) { + // FIXME: we could try to substitute p. + unknown.insert(*i); + continue; + } + Derivation drv = derivationFromPath(store, *i); + + PathSet invalid; + foreach (DerivationOutputs::iterator, j, drv.outputs) + if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path); + if (invalid.empty()) continue; + + todoDrv.insert(*i); + if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end()); } - Derivation drv = derivationFromPath(store, p); + + else { + if (store.isValidPath(*i)) continue; + query.insert(*i); + todoNonDrv.insert(*i); + } + } + + todo.clear(); + + SubstitutablePathInfos infos; + store.querySubstitutablePathInfos(query, infos); + + foreach (PathSet::iterator, i, todoDrv) { + // FIXME: cache this + Derivation drv = derivationFromPath(store, *i); bool mustBuild = false; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!store.isValidPath(i->second.path) && - !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path))) - mustBuild = true; + if (settings.useSubstitutes) { + foreach (DerivationOutputs::iterator, j, drv.outputs) + if (!store.isValidPath(j->second.path) && + infos.find(j->second.path) == infos.end()) + mustBuild = true; + } else + mustBuild = true; if (mustBuild) { - willBuild.insert(p); + willBuild.insert(*i); todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); foreach (DerivationInputs::iterator, i, drv.inputDrvs) todo.insert(i->first); - } else + } else foreach (DerivationOutputs::iterator, i, drv.outputs) - todo.insert(i->second.path); + todoNonDrv.insert(i->second.path); } - else { - if (store.isValidPath(p)) continue; - SubstitutablePathInfo info; - if (store.querySubstitutablePathInfo(p, info)) { - willSubstitute.insert(p); - downloadSize += info.downloadSize; - narSize += info.narSize; - todo.insert(info.references.begin(), info.references.end()); + foreach (PathSet::iterator, i, todoNonDrv) { + done.insert(*i); + SubstitutablePathInfos::iterator info = infos.find(*i); + if (info != infos.end()) { + willSubstitute.insert(*i); + downloadSize += info->second.downloadSize; + narSize += info->second.narSize; + todo.insert(info->second.references.begin(), info->second.references.end()); } else - unknown.insert(p); + unknown.insert(*i); } } } - + static void dfsVisit(StoreAPI & store, const PathSet & paths, const Path & path, PathSet & visited, Paths & sorted, PathSet & parents) { if (parents.find(path) != parents.end()) throw BuildError(format("cycle detected in the references of `%1%'") % path); - + if (visited.find(path) != visited.end()) return; visited.insert(path); parents.insert(path); - + PathSet references; if (store.isValidPath(path)) store.queryReferences(path, references); - + foreach (PathSet::iterator, i, references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c05447f4a2..334f4f355f 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -19,7 +19,7 @@ static void makeWritable(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + throw SysError(format("getting attributes of path `%1%'") % path); if (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)) makeMutable(path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError(format("changing writability of `%1%'") % path); @@ -57,22 +57,22 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + throw SysError(format("getting attributes of path `%1%'") % path); if (S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - optimisePath_(stats, path + "/" + *i); + foreach (Strings::iterator, i, names) + optimisePath_(stats, path + "/" + *i); return; } - + /* We can hard link regular files and maybe symlinks. */ if (!S_ISREG(st.st_mode) #if CAN_LINK_SYMLINK && !S_ISLNK(st.st_mode) #endif ) return; - + /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under NixOS (example: $fontconfig/var/cache being modified). Skip @@ -113,25 +113,25 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) current file with a hard link to that file. */ struct stat stLink; if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path `%1%'") % linkPath); - + throw SysError(format("getting attributes of path `%1%'") % linkPath); + stats.sameContents++; if (st.st_ino == stLink.st_ino) { printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); return; } - + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % nixStore % getpid() % rand()).str(); + % settings.nixStore % getpid() % rand()).str(); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its permissions). */ bool mustToggle = !isStorePath(path); if (mustToggle) makeWritable(dirOf(path)); - + /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); @@ -182,7 +182,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) void LocalStore::optimiseStore(OptimiseStats & stats) { - PathSet paths = queryValidPaths(); + PathSet paths = queryAllValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); @@ -195,10 +195,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats) void LocalStore::optimisePath(const Path & path) { - if (queryBoolSetting("auto-optimise-store", true)) { - OptimiseStats stats; - optimisePath_(stats, path); - } + OptimiseStats stats; + if (settings.autoOptimiseStore) optimisePath_(stats, path); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cbb70b2fd7..56396541ad 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -60,7 +60,7 @@ void RemoteStore::openConnection(bool reserveSpace) else throw Error(format("invalid setting for NIX_REMOTE, `%1%'") % remoteMode); - + from.fd = fdSocket; to.fd = fdSocket; @@ -100,18 +100,18 @@ void RemoteStore::forkSlave() /* Start the worker. */ Path worker = getEnv("NIX_WORKER"); if (worker == "") - worker = nixBinDir + "/nix-worker"; + worker = settings.nixBinDir + "/nix-worker"; child = fork(); - + switch (child) { - + case -1: throw SysError("unable to fork"); case 0: try { /* child */ - + if (dup2(fdChild, STDOUT_FILENO) == -1) throw SysError("dupping write side"); @@ -124,7 +124,7 @@ void RemoteStore::forkSlave() execlp(worker.c_str(), worker.c_str(), "--slave", NULL); throw SysError(format("executing `%1%'") % worker); - + } catch (std::exception & e) { std::cerr << format("child error: %1%\n") % e.what(); } @@ -142,7 +142,7 @@ void RemoteStore::connectToDaemon() if (fdSocket == -1) throw SysError("cannot create Unix domain socket"); - string socketPath = nixStateDir + DEFAULT_SOCKET_PATH; + string socketPath = settings.nixStateDir + DEFAULT_SOCKET_PATH; /* Urgh, sockaddr_un allows path names of only 108 characters. So chdir to the socket directory so that we can pass a relative @@ -150,16 +150,16 @@ void RemoteStore::connectToDaemon() applications... */ AutoCloseFD fdPrevDir = open(".", O_RDONLY); if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - chdir(dirOf(socketPath).c_str()); + chdir(dirOf(socketPath).c_str()); Path socketPathRel = "./" + baseNameOf(socketPath); - + struct sockaddr_un addr; addr.sun_family = AF_UNIX; if (socketPathRel.size() >= sizeof(addr.sun_path)) throw Error(format("socket path `%1%' is too long") % socketPathRel); using namespace std; strcpy(addr.sun_path, socketPathRel.c_str()); - + if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at `%1%'") % socketPath); @@ -184,24 +184,34 @@ RemoteStore::~RemoteStore() void RemoteStore::setOptions() { writeInt(wopSetOptions, to); - writeInt(keepFailed, to); - writeInt(keepGoing, to); - writeInt(tryFallback, to); + + writeInt(settings.keepFailed, to); + writeInt(settings.keepGoing, to); + writeInt(settings.tryFallback, to); writeInt(verbosity, to); - writeInt(maxBuildJobs, to); - writeInt(maxSilentTime, to); + writeInt(settings.maxBuildJobs, to); + writeInt(settings.maxSilentTime, to); if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) - writeInt(useBuildHook, to); + writeInt(settings.useBuildHook, to); if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) { - writeInt(buildVerbosity, to); + writeInt(settings.buildVerbosity, to); writeInt(logType, to); - writeInt(printBuildTrace, to); + writeInt(settings.printBuildTrace, to); } if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) - writeInt(buildCores, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) - writeInt(queryBoolSetting("build-use-substitutes", true), to); - + writeInt(settings.buildCores, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) + writeInt(settings.useSubstitutes, to); + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) { + Settings::SettingsMap overrides = settings.getOverrides(); + writeInt(overrides.size(), to); + foreach (Settings::SettingsMap::iterator, i, overrides) { + writeString(i->first, to); + writeString(i->second, to); + } + } + processStderr(); } @@ -217,42 +227,96 @@ bool RemoteStore::isValidPath(const Path & path) } -PathSet RemoteStore::queryValidPaths() +PathSet RemoteStore::queryValidPaths(const PathSet & paths) { openConnection(); - writeInt(wopQueryValidPaths, to); + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath(*i)) res.insert(*i); + return res; + } else { + writeInt(wopQueryValidPaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } +} + + +PathSet RemoteStore::queryAllValidPaths() +{ + openConnection(); + writeInt(wopQueryAllValidPaths, to); processStderr(); return readStorePaths<PathSet>(from); } -bool RemoteStore::hasSubstitutes(const Path & path) +PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { openConnection(); - writeInt(wopHasSubstitutes, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); - return reply != 0; + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) { + writeInt(wopHasSubstitutes, to); + writeString(*i, to); + processStderr(); + if (readInt(from)) res.insert(*i); + } + return res; + } else { + writeInt(wopQuerySubstitutablePaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } } -bool RemoteStore::querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info) +void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) { + if (paths.empty()) return; + openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false; - writeInt(wopQuerySubstitutablePathInfo, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); - if (reply == 0) return false; - info.deriver = readString(from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; - return true; + + if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; + + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + + foreach (PathSet::const_iterator, i, paths) { + SubstitutablePathInfo info; + writeInt(wopQuerySubstitutablePathInfo, to); + writeString(*i, to); + processStderr(); + unsigned int reply = readInt(from); + if (reply == 0) continue; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; + infos[*i] = info; + } + + } else { + + writeInt(wopQuerySubstitutablePathInfos, to); + writeStrings(paths, to); + processStderr(); + unsigned int count = readInt(from); + for (unsigned int n = 0; n < count; n++) { + Path path = readStorePath(from); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = readLongLong(from); + } + + } } @@ -357,7 +421,7 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { openConnection(); - + Path srcPath(absPath(_srcPath)); writeInt(wopAddToStore, to); @@ -380,7 +444,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, writeString(name, to); writeString(s, to); writeStrings(references, to); - + processStderr(); return readStorePath(from); } @@ -477,7 +541,7 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { openConnection(false); - + writeInt(wopCollectGarbage, to); writeInt(options.action, to); writeStrings(options.pathsToDelete, to); @@ -489,9 +553,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) writeInt(0, to); writeInt(0, to); } - + processStderr(); - + results.paths = readStrings<PathSet>(from); results.bytesFreed = readLongLong(from); results.blocksFreed = readLongLong(from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index c57b49ce15..ae4c48dad6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -26,7 +26,9 @@ public: bool isValidPath(const Path & path); - PathSet queryValidPaths(); + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); ValidPathInfo queryPathInfo(const Path & path); @@ -44,10 +46,10 @@ public: Path queryPathFromHashPart(const string & hashPart); - bool hasSubstitutes(const Path & path); + PathSet querySubstitutablePaths(const PathSet & paths); - bool querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info); + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b64988268c..6f81a9aab0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,16 +19,16 @@ GCOptions::GCOptions() bool isInStore(const Path & path) { return path[0] == '/' - && string(path, 0, nixStore.size()) == nixStore - && path.size() >= nixStore.size() + 2 - && path[nixStore.size()] == '/'; + && string(path, 0, settings.nixStore.size()) == settings.nixStore + && path.size() >= settings.nixStore.size() + 2 + && path[settings.nixStore.size()] == '/'; } bool isStorePath(const Path & path) { return isInStore(path) - && path.find('/', nixStore.size() + 1) == Path::npos; + && path.find('/', settings.nixStore.size() + 1) == Path::npos; } @@ -43,7 +43,7 @@ Path toStorePath(const Path & path) { if (!isInStore(path)) throw Error(format("path `%1%' is not in the Nix store") % path); - Path::size_type slash = path.find('/', nixStore.size() + 1); + Path::size_type slash = path.find('/', settings.nixStore.size() + 1); if (slash == Path::npos) return path; else @@ -74,7 +74,7 @@ Path followLinksToStorePath(const Path & path) string storePathToName(const Path & path) { assertStorePath(path); - return string(path, nixStore.size() + 34); + return string(path, settings.nixStore.size() + 34); } @@ -173,11 +173,11 @@ Path makeStorePath(const string & type, { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ string s = type + ":sha256:" + printHash(hash) + ":" - + nixStore + ":" + name; + + settings.nixStore + ":" + name; checkStoreName(name); - return nixStore + "/" + return settings.nixStore + "/" + printHash32(compressHash(hashString(htSHA256, s), 20)) + "-" + name; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9ba67852ef..324d802dc4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -85,6 +85,8 @@ struct SubstitutablePathInfo unsigned long long narSize; /* 0 = unknown */ }; +typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; + struct ValidPathInfo { @@ -107,20 +109,23 @@ public: virtual ~StoreAPI() { } - /* Checks whether a path is valid. */ + /* Check whether a path is valid. */ virtual bool isValidPath(const Path & path) = 0; - /* Query the set of valid paths. */ - virtual PathSet queryValidPaths() = 0; + /* Query which of the given paths is valid. */ + virtual PathSet queryValidPaths(const PathSet & paths) = 0; + + /* Query the set of all valid paths. */ + virtual PathSet queryAllValidPaths() = 0; /* Query information about a valid path. */ virtual ValidPathInfo queryPathInfo(const Path & path) = 0; - /* Queries the hash of a valid path. */ + /* Query the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; - /* Queries the set of outgoing FS references for a store path. - The result is not cleared. */ + /* Query the set of outgoing FS references for a store path. The + result is not cleared. */ virtual void queryReferences(const Path & path, PathSet & references) = 0; @@ -143,13 +148,14 @@ public: path, or "" if the path doesn't exist. */ virtual Path queryPathFromHashPart(const string & hashPart) = 0; - /* Query whether a path has substitutes. */ - virtual bool hasSubstitutes(const Path & path) = 0; - - /* Query the references, deriver and download size of a - substitutable path. */ - virtual bool querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info) = 0; + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 501c0b3db5..7e4c3ec5fb 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10b +#define PROTOCOL_VERSION 0x10c #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -32,13 +32,16 @@ typedef enum { wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, wopQueryDerivationOutputs = 22, - wopQueryValidPaths = 23, + wopQueryAllValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, wopQueryDerivationOutputNames = 28, wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, } WorkerOp; @@ -60,5 +63,5 @@ typedef enum { Path readStorePath(Source & from); template<class T> T readStorePaths(Source & from); - + } |