diff options
Diffstat (limited to 'src/libstore')
-rw-r--r-- | src/libstore/build.cc | 3258 | ||||
-rw-r--r-- | src/libstore/derivations.cc | 278 | ||||
-rw-r--r-- | src/libstore/derivations.hh | 93 | ||||
-rw-r--r-- | src/libstore/gc.cc | 752 | ||||
-rw-r--r-- | src/libstore/globals.cc | 218 | ||||
-rw-r--r-- | src/libstore/globals.hh | 209 | ||||
-rw-r--r-- | src/libstore/local-store.cc | 1996 | ||||
-rw-r--r-- | src/libstore/local-store.hh | 333 | ||||
-rw-r--r-- | src/libstore/local.mk | 28 | ||||
-rw-r--r-- | src/libstore/misc.cc | 220 | ||||
-rw-r--r-- | src/libstore/misc.hh | 38 | ||||
-rw-r--r-- | src/libstore/optimise-store.cc | 180 | ||||
-rw-r--r-- | src/libstore/pathlocks.cc | 199 | ||||
-rw-r--r-- | src/libstore/pathlocks.hh | 45 | ||||
-rw-r--r-- | src/libstore/references.cc | 122 | ||||
-rw-r--r-- | src/libstore/references.hh | 11 | ||||
-rw-r--r-- | src/libstore/remote-store.cc | 602 | ||||
-rw-r--r-- | src/libstore/remote-store.hh | 104 | ||||
-rw-r--r-- | src/libstore/schema.sql | 44 | ||||
-rw-r--r-- | src/libstore/store-api.cc | 331 | ||||
-rw-r--r-- | src/libstore/store-api.hh | 364 | ||||
-rw-r--r-- | src/libstore/worker-protocol.hh | 60 |
22 files changed, 9485 insertions, 0 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc new file mode 100644 index 000000000000..1b379752fa0e --- /dev/null +++ b/src/libstore/build.cc @@ -0,0 +1,3258 @@ +#include "config.h" + +#include "references.hh" +#include "pathlocks.hh" +#include "misc.hh" +#include "globals.hh" +#include "local-store.hh" +#include "util.hh" +#include "archive.hh" +#include "affinity.hh" + +#include <map> +#include <sstream> +#include <algorithm> +#include <boost/shared_ptr.hpp> +#include <boost/weak_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> + +#include <time.h> +#include <sys/time.h> +#include <sys/wait.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/utsname.h> +#include <fcntl.h> +#include <unistd.h> +#include <errno.h> +#include <stdio.h> +#include <cstring> + +#include <pwd.h> +#include <grp.h> + +#include <bzlib.h> + + +/* Includes required for chroot support. */ +#if HAVE_SYS_PARAM_H +#include <sys/param.h> +#endif +#if HAVE_SYS_MOUNT_H +#include <sys/mount.h> +#endif +#if HAVE_SCHED_H +#include <sched.h> +#endif + +/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but + <linux/fs.h> does. */ +#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H +#include <linux/fs.h> +#endif + +#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) + +#if CHROOT_ENABLED +#include <sys/socket.h> +#include <sys/ioctl.h> +#include <net/if.h> +#include <netinet/ip.h> +#endif + + +#if HAVE_SYS_PERSONALITY_H +#include <sys/personality.h> +#define CAN_DO_LINUX32_BUILDS +#endif + + +namespace nix { + +using std::map; + + +static string pathNullDevice = "/dev/null"; + + +/* Forward definition. */ +class Worker; +struct HookInstance; + + +/* A pointer to a goal. */ +class Goal; +typedef boost::shared_ptr<Goal> GoalPtr; +typedef boost::weak_ptr<Goal> WeakGoalPtr; + +/* Set of goals. */ +typedef set<GoalPtr> Goals; +typedef set<WeakGoalPtr> WeakGoals; + +/* A map of paths to goals (and the other way around). */ +typedef map<Path, WeakGoalPtr> WeakGoalMap; + + + +class Goal : public boost::enable_shared_from_this<Goal> +{ +public: + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; + +protected: + + /* Backlink to the worker. */ + Worker & worker; + + /* Goals that this goal is waiting for. */ + Goals waitees; + + /* Goals waiting for this one to finish. Must use weak pointers + here to prevent cycles. */ + WeakGoals waiters; + + /* Number of goals we are/were waiting for that have failed. */ + unsigned int nrFailed; + + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; + + /* Number of substitution goals we are/were waiting for that + failed because othey had unsubstitutable references. */ + unsigned int nrIncompleteClosure; + + /* Name of this goal for debugging purposes. */ + string name; + + /* Whether the goal is finished. */ + ExitCode exitCode; + + Goal(Worker & worker) : worker(worker) + { + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + exitCode = ecBusy; + } + + virtual ~Goal() + { + trace("goal destroyed"); + } + +public: + virtual void work() = 0; + + void addWaitee(GoalPtr waitee); + + virtual void waiteeDone(GoalPtr waitee, ExitCode result); + + virtual void handleChildOutput(int fd, const string & data) + { + abort(); + } + + virtual void handleEOF(int fd) + { + abort(); + } + + void trace(const format & f); + + string getName() + { + return name; + } + + ExitCode getExitCode() + { + return exitCode; + } + + /* Cancel the goal. It should wake up its waiters, get rid of any + running child processes that are being monitored by the worker + (important!), etc. */ + virtual void cancel(bool timeout) = 0; + +protected: + void amDone(ExitCode result); +}; + + +/* A mapping used to remember for each child process to what goal it + belongs, and file descriptors for receiving log data and output + path creation commands. */ +struct Child +{ + WeakGoalPtr goal; + set<int> fds; + bool respectTimeouts; + bool inBuildSlot; + time_t lastOutput; /* time we last got output on stdout/stderr */ + time_t timeStarted; +}; + +typedef map<pid_t, Child> Children; + + +/* The worker class. */ +class Worker +{ +private: + + /* Note: the worker should only have strong pointers to the + top-level goals. */ + + /* The top-level goals of the worker. */ + Goals topGoals; + + /* Goals that are ready to do some work. */ + WeakGoals awake; + + /* Goals waiting for a build slot. */ + WeakGoals wantingToBuild; + + /* Child processes currently running. */ + Children children; + + /* Number of build slots occupied. This includes local builds and + substitutions but not remote builds via the build hook. */ + unsigned int nrLocalBuilds; + + /* Maps used to prevent multiple instantiations of a goal for the + same derivation / path. */ + WeakGoalMap derivationGoals; + WeakGoalMap substitutionGoals; + + /* Goals waiting for busy paths to be unlocked. */ + WeakGoals waitingForAnyGoal; + + /* Goals sleeping for a few seconds (polling a lock). */ + WeakGoals waitingForAWhile; + + /* Last time the goals in `waitingForAWhile' where woken up. */ + time_t lastWokenUp; + +public: + + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; + + LocalStore & store; + + boost::shared_ptr<HookInstance> hook; + + Worker(LocalStore & store); + ~Worker(); + + /* Make a goal (with caching). */ + GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false); + GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); + + /* Remove a dead goal. */ + void removeGoal(GoalPtr goal); + + /* Wake up a goal (i.e., there is something for it to do). */ + void wakeUp(GoalPtr goal); + + /* Return the number of local build and substitution processes + currently running (but not remote builds via the build + hook). */ + unsigned int getNrLocalBuilds(); + + /* Registers a running child process. `inBuildSlot' means that + the process counts towards the jobs limit. */ + void childStarted(GoalPtr goal, pid_t pid, + const set<int> & fds, bool inBuildSlot, bool respectTimeouts); + + /* Unregisters a running child process. `wakeSleepers' should be + false if there is no sense in waking up goals that are sleeping + because they can't run yet (e.g., there is no free build slot, + or the hook would still say `postpone'). */ + void childTerminated(pid_t pid, bool wakeSleepers = true); + + /* Put `goal' to sleep until a build slot becomes available (which + might be right away). */ + void waitForBuildSlot(GoalPtr goal); + + /* Wait for any goal to finish. Pretty indiscriminate way to + wait for some resource that some other goal is holding. */ + void waitForAnyGoal(GoalPtr goal); + + /* Wait for a few seconds and then retry this goal. Used when + waiting for a lock held by another process. This kind of + polling is inefficient, but POSIX doesn't really provide a way + to wait for multiple locks in the main select() loop. */ + void waitForAWhile(GoalPtr goal); + + /* Loop until the specified top-level goals have finished. */ + void run(const Goals & topGoals); + + /* Wait for input to become available. */ + void waitForInput(); + + unsigned int exitStatus(); +}; + + +////////////////////////////////////////////////////////////////////// + + +void Goal::addWaitee(GoalPtr waitee) +{ + waitees.insert(waitee); + waitee->waiters.insert(shared_from_this()); +} + + +void Goal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + assert(waitees.find(waitee) != waitees.end()); + waitees.erase(waitee); + + trace(format("waitee `%1%' done; %2% left") % + waitee->name % waitees.size()); + + if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; + + if (result == ecNoSubstituters) ++nrNoSubstituters; + + if (result == ecIncompleteClosure) ++nrIncompleteClosure; + + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + foreach (Goals::iterator, i, waitees) { + GoalPtr goal = *i; + WeakGoals waiters2; + foreach (WeakGoals::iterator, j, goal->waiters) + if (j->lock() != shared_from_this()) waiters2.insert(*j); + goal->waiters = waiters2; + } + waitees.clear(); + + worker.wakeUp(shared_from_this()); + } +} + + +void Goal::amDone(ExitCode result) +{ + trace("done"); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); + exitCode = result; + foreach (WeakGoals::iterator, i, waiters) { + GoalPtr goal = i->lock(); + if (goal) goal->waiteeDone(shared_from_this(), result); + } + waiters.clear(); + worker.removeGoal(shared_from_this()); +} + + +void Goal::trace(const format & f) +{ + debug(format("%1%: %2%") % name % f); +} + + + +////////////////////////////////////////////////////////////////////// + + +/* Common initialisation performed in child processes. */ +static void commonChildInit(Pipe & logPipe) +{ + restoreAffinity(); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError(format("cannot open `%1%'") % pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + + +/* Convert a string list to an array of char pointers. Careful: the + string list should outlive the array. */ +const char * * strings2CharPtrs(const Strings & ss) +{ + const char * * arr = new const char * [ss.size() + 1]; + const char * * p = arr; + foreach (Strings::const_iterator, i, ss) *p++ = i->c_str(); + *p = 0; + return arr; +} + + +/* Restore default handling of SIGPIPE, otherwise some programs will + randomly say "Broken pipe". */ +static void restoreSIGPIPE() +{ + struct sigaction act, oact; + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGPIPE, &act, &oact)) throw SysError("resetting SIGPIPE"); +} + + +////////////////////////////////////////////////////////////////////// + + +class UserLock +{ +private: + /* POSIX locks suck. If we have a lock on a file, and we open and + close that file again (without closing the original file + descriptor), we lose the lock. So we have to be *very* careful + not to open a lock file on which we are holding a lock. */ + static PathSet lockedPaths; /* !!! not thread-safe */ + + Path fnUserLock; + AutoCloseFD fdUserLock; + + string user; + uid_t uid; + gid_t gid; + +public: + UserLock(); + ~UserLock(); + + void acquire(); + void release(); + + void kill(); + + string getUser() { return user; } + uid_t getUID() { return uid; } + uid_t getGID() { return gid; } + + bool enabled() { return uid != 0; } + +}; + + +PathSet UserLock::lockedPaths; + + +UserLock::UserLock() +{ + uid = gid = 0; +} + + +UserLock::~UserLock() +{ + release(); +} + + +void UserLock::acquire() +{ + assert(uid == 0); + + assert(settings.buildUsersGroup != ""); + + /* Get the members of the build-users-group. */ + struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + if (!gr) + throw Error(format("the group `%1%' specified in `build-users-group' does not exist") + % settings.buildUsersGroup); + gid = gr->gr_gid; + + /* Copy the result of getgrnam. */ + Strings users; + for (char * * p = gr->gr_mem; *p; ++p) { + debug(format("found build user `%1%'") % *p); + users.push_back(*p); + } + + if (users.empty()) + throw Error(format("the build users group `%1%' has no members") + % settings.buildUsersGroup); + + /* Find a user account that isn't currently in use for another + build. */ + foreach (Strings::iterator, i, users) { + debug(format("trying user `%1%'") % *i); + + struct passwd * pw = getpwnam(i->c_str()); + if (!pw) + throw Error(format("the user `%1%' in the group `%2%' does not exist") + % *i % settings.buildUsersGroup); + + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); + + if (lockedPaths.find(fnUserLock) != lockedPaths.end()) + /* We already have a lock on this one. */ + continue; + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT, 0600); + if (fd == -1) + throw SysError(format("opening user lock `%1%'") % fnUserLock); + closeOnExec(fd); + + if (lockFile(fd, ltWrite, false)) { + fdUserLock = fd.borrow(); + lockedPaths.insert(fnUserLock); + user = *i; + uid = pw->pw_uid; + + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of `%1%'") + % settings.buildUsersGroup); + + return; + } + } + + throw Error(format("all build users are currently in use; " + "consider creating additional users and adding them to the `%1%' group") + % settings.buildUsersGroup); +} + + +void UserLock::release() +{ + if (uid == 0) return; + fdUserLock.close(); /* releases lock */ + assert(lockedPaths.find(fnUserLock) != lockedPaths.end()); + lockedPaths.erase(fnUserLock); + fnUserLock = ""; + uid = 0; +} + + +void UserLock::kill() +{ + assert(enabled()); + killUser(uid); +} + + +////////////////////////////////////////////////////////////////////// + + +struct HookInstance +{ + /* Pipes for talking to the build hook. */ + Pipe toHook; + + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* The process ID of the hook. */ + Pid pid; + + HookInstance(); + + ~HookInstance(); +}; + + +HookInstance::HookInstance() +{ + debug("starting build hook"); + + Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); + + /* Create a pipe to get the output of the child. */ + fromHook.create(); + + /* Create the communication pipes. */ + toHook.create(); + + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + + /* Fork the hook. */ + pid = maybeVfork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + commonChildInit(fromHook); + + if (chdir("/") == -1) throw SysError("changing into `/"); + + /* Dup the communication pipes. */ + if (dup2(toHook.readSide, STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); + + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide, 4) == -1) + throw SysError("dupping builder's stdout/stderr"); + + execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), + (format("%1%") % settings.maxSilentTime).str().c_str(), + (format("%1%") % settings.printBuildTrace).str().c_str(), + (format("%1%") % settings.buildTimeout).str().c_str(), + NULL); + + throw SysError(format("executing `%1%'") % buildHook); + + } catch (std::exception & e) { + writeToStderr("build hook error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* parent */ + pid.setSeparatePG(true); + pid.setKillSignal(SIGTERM); + fromHook.writeSide.close(); + toHook.readSide.close(); +} + + +HookInstance::~HookInstance() +{ + try { + pid.kill(); + } catch (...) { + ignoreException(); + } +} + + +////////////////////////////////////////////////////////////////////// + + +typedef map<string, string> HashRewrites; + + +string rewriteHashes(string s, const HashRewrites & rewrites) +{ + foreach (HashRewrites::const_iterator, i, rewrites) { + assert(i->first.size() == i->second.size()); + size_t j = 0; + while ((j = s.find(i->first, j)) != string::npos) { + debug(format("rewriting @ %1%") % j); + s.replace(j, i->second.size(), i->second); + } + } + return s; +} + + +////////////////////////////////////////////////////////////////////// + + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + +class SubstitutionGoal; + +class DerivationGoal : public Goal +{ +private: + /* The path of the derivation. */ + Path drvPath; + + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart; + + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; + + /* The derivation stored at drvPath. */ + Derivation drv; + + /* The remainder is state held during the build. */ + + /* Locks on the output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + PathSet inputPaths; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* Outputs that are already valid. */ + PathSet validPaths; + + /* Outputs that are corrupt or not valid. */ + PathSet missingPaths; + + /* Paths that have been subject to hash rewriting. */ + PathSet rewrittenPaths; + + /* User selected for running the builder. */ + UserLock buildUser; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* File descriptor for the log file. */ + FILE * fLogFile; + BZFILE * bzLogFile; + AutoCloseFD fdLogFile; + + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* The build hook. */ + boost::shared_ptr<HookInstance> hook; + + /* Whether we're currently doing a chroot build. */ + bool useChroot; + + Path chrootRootDir; + + /* RAII object to delete the chroot directory. */ + boost::shared_ptr<AutoDelete> autoDelChroot; + + /* All inputs that are regular files. */ + PathSet regularInputPaths; + + /* Whether this is a fixed-output derivation. */ + bool fixedOutput; + + typedef void (DerivationGoal::*GoalState)(); + GoalState state; + + /* Stuff we need to pass to initChild(). */ + typedef map<Path, Path> DirsInChroot; // maps target path to source path + DirsInChroot dirsInChroot; + typedef map<string, string> Environment; + Environment env; + + /* Hash rewriting. */ + HashRewrites rewritesToTmp, rewritesFromTmp; + PathSet redirectedOutputs; + + /* Whether we're repairing. If set, a derivation will be rebuilt + if its outputs are valid but corrupt or missing. */ + bool repair; + map<Path, Path> redirectedBadOutputs; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + /* Magic exit code denoting that setting up the child environment + failed. (It's possible that the child actually returns the + exit code, but ah well.) */ + const static int childSetupFailed = 189; + +public: + DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false); + ~DerivationGoal(); + + void cancel(bool timeout); + + void work(); + + Path getDrvPath() + { + return drvPath; + } + + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet & outputs); + +private: + /* The states. */ + void init(); + void haveDerivation(); + void outputsSubstituted(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void buildDone(); + + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Initialise the builder's process. */ + void initChild(); + + friend int childEntry(void *); + + /* Must be called after the output paths have become valid (either + due to a successful build or hook, or because they already + were). */ + void computeClosure(); + + /* Open a log file and a pipe to it. */ + Path openLogFile(); + + /* Close the log file. */ + void closeLogFile(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data); + void handleEOF(int fd); + + /* Return the set of (in)valid paths. */ + PathSet checkPathValidity(bool returnValid, bool checkHash); + + /* Abort the goal if `path' failed to build. */ + bool pathFailed(const Path & path); + + /* Forcibly kill the child process, if any. */ + void killChild(); + + Path addHashRewrite(const Path & path); + + void repairClosure(); +}; + + +DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair) + : Goal(worker) + , wantedOutputs(wantedOutputs) + , needRestart(false) + , retrySubstitution(false) + , fLogFile(0) + , bzLogFile(0) + , useChroot(false) + , repair(repair) +{ + this->drvPath = drvPath; + state = &DerivationGoal::init; + name = (format("building of `%1%'") % drvPath).str(); + trace("created"); +} + + +DerivationGoal::~DerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { + killChild(); + deleteTmpDir(false); + closeLogFile(); + } catch (...) { + ignoreException(); + } +} + + +void DerivationGoal::killChild() +{ + if (pid != -1) { + worker.childTerminated(pid); + + if (buildUser.enabled()) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + buildUser.kill(); + pid.wait(true); + } else + pid.kill(); + + assert(pid == -1); + } + + hook.reset(); +} + + +void DerivationGoal::cancel(bool timeout) +{ + if (settings.printBuildTrace && timeout) + printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); + killChild(); + amDone(ecFailed); +} + + +void DerivationGoal::work() +{ + (this->*state)(); +} + + +void DerivationGoal::addWantedOutputs(const StringSet & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + foreach (StringSet::const_iterator, i, outputs) + if (wantedOutputs.find(*i) == wantedOutputs.end()) { + wantedOutputs.insert(*i); + needRestart = true; + } +} + + +void DerivationGoal::init() +{ + trace("init"); + + if (settings.readOnlyMode) + throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + addWaitee(worker.makeSubstitutionGoal(drvPath)); + + state = &DerivationGoal::haveDerivation; +} + + +void DerivationGoal::haveDerivation() +{ + trace("loading derivation"); + + if (nrFailed != 0) { + printMsg(lvlError, format("cannot build missing derivation `%1%'") % drvPath); + amDone(ecFailed); + return; + } + + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); + + assert(worker.store.isValidPath(drvPath)); + + /* Get the derivation. */ + drv = derivationFromPath(worker.store, drvPath); + + foreach (DerivationOutputs::iterator, i, drv.outputs) + worker.store.addTempRoot(i->second.path); + + /* Check what outputs paths are not already valid. */ + PathSet invalidOutputs = checkPathValidity(false, repair); + + /* If they are all valid, then we're done. */ + if (invalidOutputs.size() == 0 && !repair) { + amDone(ecSuccess); + return; + } + + /* Check whether any output previously failed to build. If so, + don't bother. */ + foreach (PathSet::iterator, i, invalidOutputs) + if (pathFailed(*i)) return; + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && !willBuildLocally(drv)) + foreach (PathSet::iterator, i, invalidOutputs) + addWaitee(worker.makeSubstitutionGoal(*i, repair)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstituted(); + else + state = &DerivationGoal::outputsSubstituted; +} + + +void DerivationGoal::outputsSubstituted() +{ + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) + throw Error(format("some substitutes for the outputs of derivation `%1%' failed (usually happens due to networking issues); try `--fallback' to build derivation from source ") % drvPath); + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. */ + if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + if (checkPathValidity(false, repair).size() == 0) { + if (repair) repairClosure(); else amDone(ecSuccess); + return; + } + + /* Otherwise, at least one of the output paths could not be + produced using a substitute. So we have to build instead. */ + + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = PathSet(); + + /* The inputs must be built before we can build this goal. */ + foreach (DerivationInputs::iterator, i, drv.inputDrvs) + addWaitee(worker.makeDerivationGoal(i->first, i->second, repair)); + + foreach (PathSet::iterator, i, drv.inputSrcs) + addWaitee(worker.makeSubstitutionGoal(*i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; +} + + +void DerivationGoal::repairClosure() +{ + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + PathSet outputClosure; + foreach (DerivationOutputs::iterator, i, drv.outputs) + computeFSClosure(worker.store, i->second.path, outputClosure); + + /* Filter out our own outputs (which we have already checked). */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + outputClosure.erase(i->second.path); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + PathSet inputClosure; + computeFSClosure(worker.store, drvPath, inputClosure); + std::map<Path, Path> outputsToDrv; + foreach (PathSet::iterator, i, inputClosure) + if (isDerivation(*i)) { + Derivation drv = derivationFromPath(worker.store, *i); + foreach (DerivationOutputs::iterator, j, drv.outputs) + outputsToDrv[j->second.path] = *i; + } + + /* Check each path (slow!). */ + PathSet broken; + foreach (PathSet::iterator, i, outputClosure) { + if (worker.store.pathContentsGood(*i)) continue; + printMsg(lvlError, format("found corrupted or missing path `%1%' in the output closure of `%2%'") % *i % drvPath); + Path drvPath2 = outputsToDrv[*i]; + if (drvPath2 == "") + addWaitee(worker.makeSubstitutionGoal(*i, true)); + else + addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true)); + } + + if (waitees.empty()) { + amDone(ecSuccess); + return; + } + + state = &DerivationGoal::closureRepaired; +} + + +void DerivationGoal::closureRepaired() +{ + trace("closure repaired"); + if (nrFailed > 0) + throw Error(format("some paths in the output closure of derivation `%1%' could not be repaired") % drvPath); + amDone(ecSuccess); +} + + +void DerivationGoal::inputsRealised() +{ + trace("all inputs realised"); + + if (nrFailed != 0) { + printMsg(lvlError, + format("cannot build derivation `%1%': " + "%2% dependencies couldn't be built") + % drvPath % nrFailed); + amDone(ecFailed); + return; + } + + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* The outputs are referenceable paths. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) { + debug(format("building path `%1%'") % i->second.path); + allPaths.insert(i->second.path); + } + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + foreach (DerivationInputs::iterator, i, drv.inputDrvs) { + /* Add the relevant output closures of the input derivation + `*i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i->first)); + Derivation inDrv = derivationFromPath(worker.store, i->first); + foreach (StringSet::iterator, j, i->second) + if (inDrv.outputs.find(*j) != inDrv.outputs.end()) + computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); + else + throw Error( + format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'") + % drvPath % *j % i->first); + } + + /* Second, the input sources. */ + foreach (PathSet::iterator, i, drv.inputSrcs) + computeFSClosure(worker.store, *i, inputPaths); + + debug(format("added input paths %1%") % showPaths(inputPaths)); + + allPaths.insert(inputPaths.begin(), inputPaths.end()); + + /* Is this a fixed-output derivation? */ + fixedOutput = true; + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (i->second.hash == "") fixedOutput = false; + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); +} + + +PathSet outputPaths(const DerivationOutputs & outputs) +{ + PathSet paths; + foreach (DerivationOutputs::const_iterator, i, outputs) + paths.insert(i->second.path); + return paths; +} + + +static string get(const StringPairs & map, const string & key) +{ + StringPairs::const_iterator i = map.find(key); + return i == map.end() ? (string) "" : i->second; +} + + +static bool canBuildLocally(const string & platform) +{ + return platform == settings.thisSystem +#ifdef CAN_DO_LINUX32_BUILDS + || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") +#endif + ; +} + + +bool willBuildLocally(const Derivation & drv) +{ + return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); +} + + +void DerivationGoal::tryToBuild() +{ + trace("trying to build"); + + /* Check for the possibility that some other goal in this process + has locked the output since we checked in haveDerivation(). + (It can't happen between here and the lockPaths() call below + because we're not allowing multi-threading.) If so, put this + goal to sleep until another goal finishes, then try again. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (pathIsLockedByMe(i->second.path)) { + debug(format("putting derivation `%1%' to sleep because `%2%' is locked by another goal") + % drvPath % i->second.path); + worker.waitForAnyGoal(shared_from_this()); + return; + } + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. If we + can't acquire the lock, then continue; hopefully some other + goal can start a build, and if not, the main loop will sleep a + few seconds and then retry this goal. */ + if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + validPaths = checkPathValidity(true, repair); + if (validPaths.size() == drv.outputs.size()) { + debug(format("skipping build of derivation `%1%', someone beat us to it") + % drvPath); + outputLocks.setDeletion(true); + amDone(ecSuccess); + return; + } + + missingPaths = outputPaths(drv.outputs); + foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i); + + /* If any of the outputs already exist but are not valid, delete + them. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) { + Path path = i->second.path; + if (worker.store.isValidPath(path)) continue; + if (!pathExists(path)) continue; + debug(format("removing unregistered path `%1%'") % path); + deletePath(path); + } + + /* Check again whether any output previously failed to build, + because some other process may have tried and failed before we + acquired the lock. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (pathFailed(i->second.path)) return; + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. */ + bool buildLocally = willBuildLocally(drv); + + /* Is the build hook willing to accept this job? */ + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + state = &DerivationGoal::buildDone; + return; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + try { + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + printMsg(lvlError, e.msg()); + outputLocks.unlock(); + buildUser.release(); + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-failed %1% - %2% %3%") + % drvPath % 0 % e.msg()); + worker.permanentFailure = true; + amDone(ecFailed); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; +} + + +void replaceValidPath(const Path & storePath, const Path tmpPath) +{ + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str(); + if (pathExists(storePath)) + rename(storePath.c_str(), oldPath.c_str()); + if (rename(tmpPath.c_str(), storePath.c_str()) == -1) + throw SysError(format("moving `%1%' to `%2%'") % tmpPath % storePath); + if (pathExists(oldPath)) + deletePath(oldPath); +} + + +void DerivationGoal::buildDone() +{ + trace("build done"); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe --- just don't do that + :-) */ + int status; + pid_t savedPid; + if (hook) { + savedPid = hook->pid; + status = hook->pid.wait(true); + } else { + /* !!! this could block! security problem! solution: kill the + child */ + savedPid = pid; + status = pid.wait(true); + } + + debug(format("builder process for `%1%' finished") % drvPath); + + /* So the child is gone now. */ + worker.childTerminated(savedPid); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); + } + else builderOut.readSide.close(); + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser.enabled()) buildUser.kill(); + + try { + + /* Some cleanup per path. We do this here and not in + computeClosure() for convenience when the build has + failed. */ + foreach (PathSet::iterator, i, missingPaths) { + Path path = *i; + + /* If the output was already valid, just skip (discard) it. */ + if (validPaths.find(path) != validPaths.end()) continue; + + if (useChroot && pathExists(chrootRootDir + path)) { + /* Move output paths from the chroot to the Nix store. */ + if (repair) + replaceValidPath(path, chrootRootDir + path); + else + if (rename((chrootRootDir + path).c_str(), path.c_str()) == -1) + throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path); + } + + Path redirected; + if (repair && (redirected = redirectedBadOutputs[path]) != "" && pathExists(redirected)) + replaceValidPath(path, redirected); + + if (!pathExists(path)) continue; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("getting attributes of path `%1%'") % path); + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, + as that means that someone else can have interfered + with the build. Also, the output should be owned by + the build user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser.enabled() && st.st_uid != buildUser.getUID())) + throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path); +#endif + + /* Apply hash rewriting if necessary. */ + if (!rewritesFromTmp.empty()) { + printMsg(lvlError, format("warning: rewriting hashes in `%1%'; cross fingers") % path); + + /* Canonicalise first. This ensures that the path + we're rewriting doesn't contain a hard link to + /etc/shadow or something like that. */ + canonicalisePathMetaData(path, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(path, sink); + deletePath(path); + sink.s = rewriteHashes(sink.s, rewritesFromTmp); + StringSource source(sink.s); + restorePath(path, source); + + rewrittenPaths.insert(path); + } + } + + /* Check the exit status. */ + if (!statusOk(status)) { + deleteTmpDir(false); + if (WIFEXITED(status) && WEXITSTATUS(status) == childSetupFailed) + throw Error(format("failed to set up the build environment for `%1%'") % drvPath); + throw BuildError(format("builder for `%1%' %2%") + % drvPath % statusToString(status)); + } + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + /* Delete redirected outputs (when doing hash rewriting). */ + foreach (PathSet::iterator, i, redirectedOutputs) + deletePath(*i); + + /* Compute the FS closure of the outputs and register them as + being valid. */ + computeClosure(); + + deleteTmpDir(true); + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + } catch (BuildError & e) { + printMsg(lvlError, e.msg()); + outputLocks.unlock(); + buildUser.release(); + + /* When using a build hook, the hook will return a remote + build failure using exit code 100. Anything else is a hook + problem. */ + bool hookError = hook && + (!WIFEXITED(status) || WEXITSTATUS(status) != 100); + + if (settings.printBuildTrace) { + if (hook && hookError) + printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") + % drvPath % status % e.msg()); + else + printMsg(lvlError, format("@ build-failed %1% - %2% %3%") + % drvPath % 1 % e.msg()); + } + + /* Register the outputs of this build as "failed" so we won't + try to build them again (negative caching). However, don't + do this for fixed-output derivations, since they're likely + to fail for transient reasons (e.g., fetchurl not being + able to access the network). Hook errors (like + communication problems with the remote machine) shouldn't + be cached either. */ + if (settings.cacheFailure && !hookError && !fixedOutput) + foreach (DerivationOutputs::iterator, i, drv.outputs) + worker.store.registerFailedPath(i->second.path); + + worker.permanentFailure = !hookError && !fixedOutput; + amDone(ecFailed); + return; + } + + /* Release the build user, if applicable. */ + buildUser.release(); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); + + amDone(ecSuccess); +} + + +HookReply DerivationGoal::tryBuildHook() +{ + if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; + + if (!worker.hook) + worker.hook = boost::shared_ptr<HookInstance>(new HookInstance); + + /* Tell the hook about system features (beyond the system type) + required from the build machine. (The hook could parse the + drv file itself, but this is easier.) */ + Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures")); + foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ + + /* Send the request to the hook. */ + writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") + % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") + % drv.platform % drvPath % concatStringsSep(",", features)).str()); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + string s = readLine(worker.hook->fromHook.readSide); + if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } + s += "\n"; + writeToStderr(s); + } + + debug(format("hook reply is `%1%'") % reply); + + if (reply == "decline" || reply == "postpone") + return reply == "decline" ? rpDecline : rpPostpone; + else if (reply != "accept") + throw Error(format("bad hook reply `%1%'") % reply); + + printMsg(lvlTalkative, format("using hook to build path(s) %1%") + % showPaths(outputPaths(drv.outputs))); + + hook = worker.hook; + worker.hook.reset(); + + /* Tell the hook all the inputs that have to be copied to the + remote system. This unfortunately has to contain the entire + derivation closure to ensure that the validity invariant holds + on the remote system. (I.e., it's unfortunate that we have to + list it since the remote system *probably* already has it.) */ + PathSet allInputs; + allInputs.insert(inputPaths.begin(), inputPaths.end()); + computeFSClosure(worker.store, drvPath, allInputs); + + string s; + foreach (PathSet::iterator, i, allInputs) s += *i + " "; + writeLine(hook->toHook.writeSide, s); + + /* Tell the hooks the outputs that have to be copied back from the + remote system. */ + s = ""; + foreach (DerivationOutputs::iterator, i, drv.outputs) + s += i->second.path + " "; + writeLine(hook->toHook.writeSide, s); + + hook->toHook.writeSide.close(); + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + set<int> fds; + fds.insert(hook->fromHook.readSide); + fds.insert(hook->builderOut.readSide); + worker.childStarted(shared_from_this(), hook->pid, fds, false, false); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-started %1% - %2% %3%") + % drvPath % drv.platform % logFile); + + return rpAccept; +} + + +void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("setting permissions on `%1%'") % path); +} + + +int childEntry(void * arg) +{ + ((DerivationGoal *) arg)->initChild(); + return 1; +} + + +void DerivationGoal::startBuilder() +{ + startNest(nest, lvlInfo, format(repair ? "repairing path(s) %1%" : "building path(s) %1%") % showPaths(missingPaths)); + + /* Right platform? */ + if (!canBuildLocally(drv.platform)) { + if (settings.printBuildTrace) + printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform); + throw Error( + format("a `%1%' is required to build `%3%', but I am a `%2%'") + % drv.platform % settings.thisSystem % drvPath); + } + + /* Construct the environment passed to the builder. */ + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + Path homeDir = "/homeless-shelter"; + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = settings.nixStore; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + /* Add all bindings specified in the derivation. */ + foreach (StringPairs::iterator, i, drv.env) + env[i->first] = i->second; + + /* Create a temporary directory where the build will take + place. */ + tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700); + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDir; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDir; + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars")); + foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); + } + + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv.env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + + /* Check that the store path is valid. */ + Path storePath = *i++; + if (!isInStore(storePath)) + throw BuildError(format("`exportReferencesGraph' contains a non-store path `%1%'") + % storePath); + storePath = toStorePath(storePath); + if (!worker.store.isValidPath(storePath)) + throw BuildError(format("`exportReferencesGraph' contains an invalid path `%1%'") + % storePath); + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths, paths2; + computeFSClosure(worker.store, storePath, paths); + paths2 = paths; + + foreach (PathSet::iterator, j, paths2) { + if (isDerivation(*j)) { + Derivation drv = derivationFromPath(worker.store, *j); + foreach (DerivationOutputs::iterator, k, drv.outputs) + computeFSClosure(worker.store, k->second.path, paths); + } + } + + /* Write closure info to `fileName'. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration(paths, false, false)); + } + + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "") { + buildUser.acquire(); + assert(buildUser.getUID() != 0); + assert(buildUser.getGID() != 0); + + /* Make sure that no other processes are executing under this + uid. */ + buildUser.kill(); + + /* Change ownership of the temporary build directory. */ + if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1) + throw SysError(format("cannot change ownership of `%1%'") % tmpDir); + + /* Check that the Nix store has the appropriate permissions, + i.e., owned by root and mode 1775 (sticky bit on so that + the builder can create its output but not mess with the + outputs of other processes). */ + struct stat st; + if (stat(settings.nixStore.c_str(), &st) == -1) + throw SysError(format("cannot stat `%1%'") % settings.nixStore); + if (!(st.st_mode & S_ISVTX) || + ((st.st_mode & S_IRWXG) != S_IRWXG) || + (st.st_gid != buildUser.getGID())) + throw Error(format( + "builder does not have write permission to `%2%'; " + "try `chgrp %1% %2%; chmod 1775 %2%'") + % buildUser.getGID() % settings.nixStore); + } + + + /* Are we doing a chroot build? Note that fixed-output + derivations are never done in a chroot, mainly so that + functions like fetchurl (which needs a proper /etc/resolv.conf) + work properly. Purity checking for fixed-output derivations + is somewhat pointless anyway. */ + useChroot = settings.useChroot; + + if (fixedOutput) useChroot = false; + + /* Hack to allow derivations to disable chroot builds. */ + if (get(drv.env, "__noChroot") == "1") useChroot = false; + + if (useChroot) { +#if CHROOT_ENABLED + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = drvPath + ".chroot"; + if (pathExists(chrootRootDir)) deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); + + printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + writeFile(chrootRootDir + "/etc/passwd", + (format( + "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n") + % (buildUser.enabled() ? buildUser.getUID() : getuid()) + % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + (format("nixbld:!:%1%:\n") + % (buildUser.enabled() ? buildUser.getGID() : getgid())).str()); + + /* Create /etc/hosts with localhost entry. */ + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); + + /* Bind-mount a user-configurable set of directories from the + host file system. */ + foreach (StringSet::iterator, i, settings.dirsInChroot) { + size_t p = i->find('='); + if (p == string::npos) + dirsInChroot[*i] = *i; + else + dirsInChroot[string(*i, 0, p)] = string(*i, p + 1); + } + dirsInChroot[tmpDir] = tmpDir; + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + createDirs(chrootRootDir + settings.nixStore); + chmod_(chrootRootDir + settings.nixStore, 01777); + + foreach (PathSet::iterator, i, inputPaths) { + struct stat st; + if (lstat(i->c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % *i); + if (S_ISDIR(st.st_mode)) + dirsInChroot[*i] = *i; + else { + Path p = chrootRootDir + *i; + if (link(i->c_str(), p.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum + link count on a file (e.g. 32000 of ext3), + which is quite possible after a `nix-store + --optimise'. */ + if (errno != EMLINK) + throw SysError(format("linking `%1%' to `%2%'") % p % *i); + StringSink sink; + dumpPath(*i, sink); + StringSource source(sink.s); + restorePath(p, source); + } + + regularInputPaths.insert(*i); + } + } + + /* If we're repairing, it's possible that we're rebuilding a + path that is in settings.dirsInChroot (typically the + dependencies of /bin/sh). Throw them out. */ + if (repair) + foreach (DerivationOutputs::iterator, i, drv.outputs) + dirsInChroot.erase(i->second.path); + +#else + throw Error("chroot builds are not supported on this platform"); +#endif + } + + else { + + if (pathExists(homeDir)) + throw Error(format("directory `%1%' exists; please remove it") % homeDir); + + /* We're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + if (validPaths.size() > 0) + //throw Error(format("derivation `%1%' is blocked by its output path(s) %2%") % drvPath % showPaths(validPaths)); + foreach (PathSet::iterator, i, validPaths) + redirectedOutputs.insert(addHashRewrite(*i)); + + /* If we're repairing, then we don't want to delete the + corrupt outputs in advance. So rewrite them as well. */ + if (repair) + foreach (PathSet::iterator, i, missingPaths) + if (worker.store.isValidPath(*i) && pathExists(*i)) + redirectedBadOutputs[*i] = addHashRewrite(*i); + } + + + /* Run the builder. */ + printMsg(lvlChatty, format("executing builder `%1%'") % + drv.builder); + + /* Create the log file. */ + Path logFile = openLogFile(); + + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + + /* Fork a child to build the package. Note that while we + currently use forks to run and wait for the children, it + shouldn't be hard to use threads for this on systems where + fork() is unavailable or inefficient. + + If we're building in a chroot, then also set up private + namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those on + the inside, but processes inside the chroot are visible from + the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind mounts + we do will only show up in this process and its children, and + will disappear automatically when we're done. + + - The private network namespace ensures that the builder cannot + talk to the outside world (or vice versa). It only has a + private loopback interface. + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures that + all IPC objects are destroyed when the builder exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + */ +#if CHROOT_ENABLED + if (useChroot) { + char stack[32 * 1024]; + pid = clone(childEntry, stack + sizeof(stack) - 8, + CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD, this); + } else +#endif + { + pid = fork(); + if (pid == 0) initChild(); + } + + if (pid == -1) throw SysError("unable to fork"); + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide.close(); + worker.childStarted(shared_from_this(), pid, + singleton<set<int> >(builderOut.readSide), true, true); + + if (settings.printBuildTrace) { + printMsg(lvlError, format("@ build-started %1% - %2% %3%") + % drvPath % drv.platform % logFile); + } +} + + +void DerivationGoal::initChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + bool inSetup = true; + + try { /* child */ + +#if CHROOT_ENABLED + if (useChroot) { + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (fd == -1) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd, SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + + fd.close(); + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + sethostname(hostname, sizeof(hostname)); + char domainname[] = "(none)"; // kernel default + setdomainname(domainname, sizeof(domainname)); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); + foreach (Strings::iterator, i, mounts) { + vector<string> fields = tokenizeString<vector<string> >(*i, " "); + string fs = decodeOctalEscaped(fields.at(4)); + if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) + throw SysError(format("unable to make filesystem `%1%' private") % fs); + } + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + foreach (DirsInChroot::iterator, i, dirsInChroot) { + struct stat st; + Path source = i->second; + Path target = chrootRootDir + i->first; + if (source == "/proc") continue; // backwards compatibility + debug(format("bind mounting `%1%' to `%2%'") % source % target); + if (stat(source.c_str(), &st) == -1) + throw SysError(format("getting attributes of path `%1%'") % source); + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) + throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); + } + + /* Bind a new instance of procfs on /proc to reflect our + private PID namespace. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm")) + if (mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, 0) == -1) + throw SysError("mounting /dev/shm"); + + /* Do the chroot(). Below we do a chdir() to the + temporary build directory to make sure the current + directory is in the chroot. (Actually the order + doesn't matter, since due to the bind mount tmpDir and + tmpRootDit/tmpDir are the same directories.) */ + if (chroot(chrootRootDir.c_str()) == -1) + throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); + } +#endif + + commonChildInit(builderOut); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into `%1%'") % tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs(set<int>()); + +#ifdef CAN_DO_LINUX32_BUILDS + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv.platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if (personality(0x0008 | 0x8000000 /* == PER_LINUX32_3GB */) == -1) + throw SysError("cannot set i686-linux personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } +#endif + + /* Fill in the environment. */ + Strings envStrs; + foreach (Environment::const_iterator, i, env) + envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); + const char * * envArr = strings2CharPtrs(envStrs); + + Path program = drv.builder.c_str(); + std::vector<const char *> args; /* careful with c_str()! */ + string user; /* must be here for its c_str()! */ + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (buildUser.enabled()) { + printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); + + if (setgroups(0, 0) == -1) + throw SysError("cannot clear the set of supplementary groups"); + + if (setgid(buildUser.getGID()) == -1 || + getgid() != buildUser.getGID() || + getegid() != buildUser.getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser.getUID()) == -1 || + getuid() != buildUser.getUID() || + geteuid() != buildUser.getUID()) + throw SysError("setuid failed"); + } + + /* Fill in the arguments. */ + string builderBasename = baseNameOf(drv.builder); + args.push_back(builderBasename.c_str()); + foreach (Strings::iterator, i, drv.args) + args.push_back(rewriteHashes(*i, rewritesToTmp).c_str()); + args.push_back(0); + + restoreSIGPIPE(); + + /* Execute the program. This should not return. */ + inSetup = false; + execve(program.c_str(), (char * *) &args[0], (char * *) envArr); + + throw SysError(format("executing `%1%'") % drv.builder); + + } catch (std::exception & e) { + writeToStderr("build error: " + string(e.what()) + "\n"); + _exit(inSetup ? childSetupFailed : 1); + } + + abort(); /* never reached */ +} + + +/* Parse a list of reference specifiers. Each element must either be + a store path, or the symbolic name of the output of the derivation + (such as `out'). */ +PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) +{ + PathSet result; + Paths paths = tokenizeString<Paths>(attr); + foreach (Strings::iterator, i, paths) { + if (isStorePath(*i)) + result.insert(*i); + else if (drv.outputs.find(*i) != drv.outputs.end()) + result.insert(drv.outputs.find(*i)->second.path); + else throw BuildError( + format("derivation contains an illegal reference specifier `%1%'") + % *i); + } + return result; +} + + +void DerivationGoal::computeClosure() +{ + map<Path, PathSet> allReferences; + map<Path, HashResult> contentHashes; + + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. */ + if (hook) { + bool allValid = true; + foreach (DerivationOutputs::iterator, i, drv.outputs) + if (!worker.store.isValidPath(i->second.path)) allValid = false; + if (allValid) return; + } + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + foreach (DerivationOutputs::iterator, i, drv.outputs) { + Path path = i->second.path; + if (missingPaths.find(path) == missingPaths.end()) continue; + + if (!pathExists(path)) { + throw BuildError( + format("builder for `%1%' failed to produce output path `%2%'") + % drvPath % path); + } + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("getting attributes of path `%1%'") % path); + + startNest(nest, lvlTalkative, + format("scanning for references inside `%1%'") % path); + + /* Check that fixed-output derivations produced the right + outputs (i.e., the content hash should match the specified + hash). */ + if (i->second.hash != "") { + + bool recursive; HashType ht; Hash h; + i->second.parseHashInfo(recursive, ht, h); + + if (!recursive) { + /* The output path should be a regular file without + execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw BuildError( + format("output path `%1% should be a non-executable regular file") + % path); + } + + /* Check the hash. */ + Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path); + if (h != h2) + throw BuildError( + format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") + % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); + } + + /* Get rid of all weird permissions. This also checks that + all files are owned by the build user, if applicable. */ + canonicalisePathMetaData(path, + buildUser.enabled() && rewrittenPaths.find(path) == rewrittenPaths.end() ? buildUser.getUID() : -1, inodesSeen); + + /* For this output path, find the references to other paths + contained in it. Compute the SHA-256 NAR hash at the same + time. The hash is stored in the database so that we can + verify later on whether nobody has messed with the store. */ + HashResult hash; + PathSet references = scanForReferences(path, allPaths, hash); + contentHashes[path] = hash; + + /* For debugging, print out the referenced and unreferenced + paths. */ + foreach (PathSet::iterator, i, inputPaths) { + PathSet::iterator j = references.find(*i); + if (j == references.end()) + debug(format("unreferenced input: `%1%'") % *i); + else + debug(format("referenced input: `%1%'") % *i); + } + + allReferences[path] = references; + + /* If the derivation specifies an `allowedReferences' + attribute (containing a list of paths that the output may + refer to), check that all references are in that list. !!! + allowedReferences should really be per-output. */ + if (drv.env.find("allowedReferences") != drv.env.end()) { + PathSet allowed = parseReferenceSpecifiers(drv, get(drv.env, "allowedReferences")); + foreach (PathSet::iterator, i, references) + if (allowed.find(*i) == allowed.end()) + throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); + } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + + worker.store.markContentsGood(path); + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + ValidPathInfos infos; + foreach (PathSet::iterator, i, missingPaths) { + ValidPathInfo info; + info.path = *i; + info.hash = contentHashes[*i].first; + info.narSize = contentHashes[*i].second; + info.references = allReferences[*i]; + info.deriver = drvPath; + infos.push_back(info); + } + worker.store.registerValidPaths(infos); +} + + +string drvsLogDir = "drvs"; + + +Path DerivationGoal::openLogFile() +{ + logSize = 0; + + if (!settings.keepLog) return ""; + + string baseName = baseNameOf(drvPath); + + /* Create a log file. */ + Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); + createDirs(dir); + + if (settings.compressLog) { + + Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); + AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fd); + + if (!(fLogFile = fdopen(fd.borrow(), "w"))) + throw SysError(format("opening file `%1%'") % logFileName); + + int err; + if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) + throw Error(format("cannot open compressed log file `%1%'") % logFileName); + + return logFileName; + + } else { + Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); + closeOnExec(fdLogFile); + return logFileName; + } +} + + +void DerivationGoal::closeLogFile() +{ + if (bzLogFile) { + int err; + BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0); + bzLogFile = 0; + if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err); + } + + if (fLogFile) { + fclose(fLogFile); + fLogFile = 0; + } + + fdLogFile.close(); +} + + +void DerivationGoal::deleteTmpDir(bool force) +{ + if (tmpDir != "") { + if (settings.keepFailed && !force) { + printMsg(lvlError, + format("note: keeping build directory `%2%'") + % drvPath % tmpDir); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(tmpDir); + tmpDir = ""; + } +} + + +void DerivationGoal::handleChildOutput(int fd, const string & data) +{ + if ((hook && fd == hook->builderOut.readSide) || + (!hook && fd == builderOut.readSide)) + { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + printMsg(lvlError, + format("%1% killed after writing more than %2% bytes of log output") + % getName() % settings.maxLogSize); + cancel(true); // not really a timeout, but close enough + return; + } + if (verbosity >= settings.buildVerbosity) + writeToStderr(data); + if (bzLogFile) { + int err; + BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); + if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); + } else if (fdLogFile != -1) + writeFull(fdLogFile, (unsigned char *) data.data(), data.size()); + } + + if (hook && fd == hook->fromHook.readSide) + writeToStderr(data); +} + + +void DerivationGoal::handleEOF(int fd) +{ + worker.wakeUp(shared_from_this()); +} + + +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +{ + PathSet result; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + if (!wantOutput(i->first, wantedOutputs)) continue; + bool good = + worker.store.isValidPath(i->second.path) && + (!checkHash || worker.store.pathContentsGood(i->second.path)); + if (good == returnValid) result.insert(i->second.path); + } + return result; +} + + +bool DerivationGoal::pathFailed(const Path & path) +{ + if (!settings.cacheFailure) return false; + + if (!worker.store.hasPathFailed(path)) return false; + + printMsg(lvlError, format("builder for `%1%' failed previously (cached)") % path); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); + + worker.permanentFailure = true; + amDone(ecFailed); + + return true; +} + + +Path DerivationGoal::addHashRewrite(const Path & path) +{ + string h1 = string(path, settings.nixStore.size() + 1, 32); + string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); + Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33); + if (pathExists(p)) deletePath(p); + assert(path.size() == p.size()); + rewritesToTmp[h1] = h2; + rewritesFromTmp[h2] = h1; + return p; +} + + +////////////////////////////////////////////////////////////////////// + + +class SubstitutionGoal : public Goal +{ + friend class Worker; + +private: + /* The store path that should be realised through a substitute. */ + Path storePath; + + /* The remaining substituters. */ + Paths subs; + + /* The current substituter. */ + Path sub; + + /* Whether any substituter can realise this path */ + bool hasSubstitute; + + /* Path info returned by the substituter's query info operation. */ + SubstitutablePathInfo info; + + /* Pipe for the substituter's standard output. */ + Pipe outPipe; + + /* Pipe for the substituter's standard error. */ + Pipe logPipe; + + /* The process ID of the builder. */ + Pid pid; + + /* Lock on the store path. */ + boost::shared_ptr<PathLocks> outputLock; + + /* Whether to try to repair a valid path. */ + bool repair; + + /* Location where we're downloading the substitute. Differs from + storePath when doing a repair. */ + Path destPath; + + typedef void (SubstitutionGoal::*GoalState)(); + GoalState state; + +public: + SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); + ~SubstitutionGoal(); + + void cancel(bool timeout); + + void work(); + + /* The states. */ + void init(); + void tryNext(); + void gotInfo(); + void referencesValid(); + void tryToRun(); + void finished(); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data); + void handleEOF(int fd); + + Path getStorePath() { return storePath; } +}; + + +SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair) + : Goal(worker) + , hasSubstitute(false) + , repair(repair) +{ + this->storePath = storePath; + state = &SubstitutionGoal::init; + name = (format("substitution of `%1%'") % storePath).str(); + trace("created"); +} + + +SubstitutionGoal::~SubstitutionGoal() +{ + if (pid != -1) worker.childTerminated(pid); +} + + +void SubstitutionGoal::cancel(bool timeout) +{ + if (settings.printBuildTrace && timeout) + printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); + if (pid != -1) { + pid_t savedPid = pid; + pid.kill(); + worker.childTerminated(savedPid); + } + amDone(ecFailed); +} + + +void SubstitutionGoal::work() +{ + (this->*state)(); +} + + +void SubstitutionGoal::init() +{ + trace("init"); + + worker.store.addTempRoot(storePath); + + /* If the path already exists we're done. */ + if (!repair && worker.store.isValidPath(storePath)) { + amDone(ecSuccess); + return; + } + + if (settings.readOnlyMode) + throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath); + + subs = settings.substituters; + + tryNext(); +} + + +void SubstitutionGoal::tryNext() +{ + trace("trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(hasSubstitute ? ecFailed : ecNoSubstituters); + return; + } + + sub = subs.front(); + subs.pop_front(); + + SubstitutablePathInfos infos; + PathSet dummy(singleton<PathSet>(storePath)); + worker.store.querySubstitutablePathInfos(sub, dummy, infos); + SubstitutablePathInfos::iterator k = infos.find(storePath); + if (k == infos.end()) { tryNext(); return; } + info = k->second; + hasSubstitute = true; + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + foreach (PathSet::iterator, i, info.references) + if (*i != storePath) /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(*i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + referencesValid(); + else + state = &SubstitutionGoal::referencesValid; +} + + +void SubstitutionGoal::referencesValid() +{ + trace("all references realised"); + + if (nrFailed > 0) { + debug(format("some references of path `%1%' could not be realised") % storePath); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + return; + } + + foreach (PathSet::iterator, i, info.references) + if (*i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(*i)); + + state = &SubstitutionGoal::tryToRun; + worker.wakeUp(shared_from_this()); +} + + +void SubstitutionGoal::tryToRun() +{ + trace("trying to run"); + + /* Make sure that we are allowed to start a build. Note that even + is maxBuildJobs == 0 (no local builds allowed), we still allow + a substituter to run. This is because substitutions cannot be + distributed to another machine via the build hook. */ + if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { + worker.waitForBuildSlot(shared_from_this()); + return; + } + + /* Maybe a derivation goal has already locked this path + (exceedingly unlikely, since it should have used a substitute + first, but let's be defensive). */ + outputLock.reset(); // make sure this goal's lock is gone + if (pathIsLockedByMe(storePath)) { + debug(format("restarting substitution of `%1%' because it's locked by another goal") + % storePath); + worker.waitForAnyGoal(shared_from_this()); + return; /* restart in the tryToRun() state when another goal finishes */ + } + + /* Acquire a lock on the output path. */ + outputLock = boost::shared_ptr<PathLocks>(new PathLocks); + if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Check again whether the path is invalid. */ + if (!repair && worker.store.isValidPath(storePath)) { + debug(format("store path `%1%' has become valid") % storePath); + outputLock->setDeletion(true); + amDone(ecSuccess); + return; + } + + printMsg(lvlInfo, format("fetching path `%1%'...") % storePath); + + outPipe.create(); + logPipe.create(); + + destPath = repair ? storePath + ".tmp" : storePath; + + /* Remove the (stale) output path if it exists. */ + if (pathExists(destPath)) + deletePath(destPath); + + worker.store.setSubstituterEnv(); + + /* Fill in the arguments. */ + Strings args; + args.push_back(baseNameOf(sub)); + args.push_back("--substitute"); + args.push_back(storePath); + args.push_back(destPath); + const char * * argArr = strings2CharPtrs(args); + + /* Fork the substitute program. */ + pid = maybeVfork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + commonChildInit(logPipe); + + if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("cannot dup output pipe into stdout"); + + execv(sub.c_str(), (char * *) argArr); + + throw SysError(format("executing `%1%'") % sub); + + } catch (std::exception & e) { + writeToStderr("substitute error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* parent */ + pid.setSeparatePG(true); + pid.setKillSignal(SIGTERM); + outPipe.writeSide.close(); + logPipe.writeSide.close(); + worker.childStarted(shared_from_this(), + pid, singleton<set<int> >(logPipe.readSide), true, true); + + state = &SubstitutionGoal::finished; + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); +} + + +void SubstitutionGoal::finished() +{ + trace("substitute finished"); + + /* Since we got an EOF on the logger pipe, the substitute is + presumed to have terminated. */ + pid_t savedPid = pid; + int status = pid.wait(true); + + /* So the child is gone now. */ + worker.childTerminated(savedPid); + + /* Close the read side of the logger pipe. */ + logPipe.readSide.close(); + + /* Get the hash info from stdout. */ + string dummy = readLine(outPipe.readSide); + string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; + outPipe.readSide.close(); + + /* Check the exit status and the build result. */ + HashResult hash; + try { + + if (!statusOk(status)) + throw SubstError(format("fetching path `%1%' %2%") + % storePath % statusToString(status)); + + if (!pathExists(destPath)) + throw SubstError(format("substitute did not produce path `%1%'") % destPath); + + hash = hashPath(htSHA256, destPath); + + /* Verify the expected hash we got from the substituer. */ + if (expectedHashStr != "") { + size_t n = expectedHashStr.find(':'); + if (n == string::npos) + throw Error(format("bad hash from substituter: %1%") % expectedHashStr); + HashType hashType = parseHashType(string(expectedHashStr, 0, n)); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr); + Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); + Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; + if (expectedHash != actualHash) + throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%") + % storePath % printHash(expectedHash) % printHash(actualHash)); + } + + } catch (SubstError & e) { + + printMsg(lvlInfo, e.msg()); + + if (settings.printBuildTrace) { + printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") + % storePath % status % e.msg()); + } + + /* Try the next substitute. */ + state = &SubstitutionGoal::tryNext; + worker.wakeUp(shared_from_this()); + return; + } + + canonicalisePathMetaData(destPath, -1); + + worker.store.optimisePath(destPath); // FIXME: combine with hashPath() + + if (repair) replaceValidPath(storePath, destPath); + + ValidPathInfo info2; + info2.path = storePath; + info2.hash = hash.first; + info2.narSize = hash.second; + info2.references = info.references; + info2.deriver = info.deriver; + worker.store.registerValidPath(info2); + + outputLock->setDeletion(true); + + worker.store.markContentsGood(storePath); + + printMsg(lvlChatty, + format("substitution of path `%1%' succeeded") % storePath); + + if (settings.printBuildTrace) + printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); + + amDone(ecSuccess); +} + + +void SubstitutionGoal::handleChildOutput(int fd, const string & data) +{ + assert(fd == logPipe.readSide); + if (verbosity >= settings.buildVerbosity) writeToStderr(data); + /* Don't write substitution output to a log file for now. We + probably should, though. */ +} + + +void SubstitutionGoal::handleEOF(int fd) +{ + if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); +} + + + +////////////////////////////////////////////////////////////////////// + + +static bool working = false; + + +Worker::Worker(LocalStore & store) + : store(store) +{ + /* Debugging: prevent recursive workers. */ + if (working) abort(); + working = true; + nrLocalBuilds = 0; + lastWokenUp = 0; + permanentFailure = false; +} + + +Worker::~Worker() +{ + working = false; + + /* Explicitly get rid of all strong pointers now. After this all + goals that refer to this worker should be gone. (Otherwise we + are in trouble, since goals may call childTerminated() etc. in + their destructors). */ + topGoals.clear(); +} + + +GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair) +{ + GoalPtr goal = derivationGoals[path].lock(); + if (!goal) { + goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair)); + derivationGoals[path] = goal; + wakeUp(goal); + } else + (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); + return goal; +} + + +GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) +{ + GoalPtr goal = substitutionGoals[path].lock(); + if (!goal) { + goal = GoalPtr(new SubstitutionGoal(path, *this, repair)); + substitutionGoals[path] = goal; + wakeUp(goal); + } + return goal; +} + + +static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +{ + /* !!! inefficient */ + for (WeakGoalMap::iterator i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + WeakGoalMap::iterator j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; +} + + +void Worker::removeGoal(GoalPtr goal) +{ + nix::removeGoal(goal, derivationGoals); + nix::removeGoal(goal, substitutionGoals); + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + topGoals.clear(); + } + + /* Wake up goals waiting for any goal to finish. */ + foreach (WeakGoals::iterator, i, waitingForAnyGoal) { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + + waitingForAnyGoal.clear(); +} + + +void Worker::wakeUp(GoalPtr goal) +{ + goal->trace("woken up"); + awake.insert(goal); +} + + +unsigned Worker::getNrLocalBuilds() +{ + return nrLocalBuilds; +} + + +void Worker::childStarted(GoalPtr goal, + pid_t pid, const set<int> & fds, bool inBuildSlot, + bool respectTimeouts) +{ + Child child; + child.goal = goal; + child.fds = fds; + child.timeStarted = child.lastOutput = time(0); + child.inBuildSlot = inBuildSlot; + child.respectTimeouts = respectTimeouts; + children[pid] = child; + if (inBuildSlot) nrLocalBuilds++; +} + + +void Worker::childTerminated(pid_t pid, bool wakeSleepers) +{ + assert(pid != -1); /* common mistake */ + + Children::iterator i = children.find(pid); + assert(i != children.end()); + + if (i->second.inBuildSlot) { + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + } + + children.erase(pid); + + if (wakeSleepers) { + + /* Wake up goals waiting for a build slot. */ + foreach (WeakGoals::iterator, i, wantingToBuild) { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + + wantingToBuild.clear(); + } +} + + +void Worker::waitForBuildSlot(GoalPtr goal) +{ + debug("wait for build slot"); + if (getNrLocalBuilds() < settings.maxBuildJobs) + wakeUp(goal); /* we can do it right away */ + else + wantingToBuild.insert(goal); +} + + +void Worker::waitForAnyGoal(GoalPtr goal) +{ + debug("wait for any goal"); + waitingForAnyGoal.insert(goal); +} + + +void Worker::waitForAWhile(GoalPtr goal) +{ + debug("wait for a while"); + waitingForAWhile.insert(goal); +} + + +void Worker::run(const Goals & _topGoals) +{ + foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i); + + startNest(nest, lvlDebug, format("entered goal loop")); + + while (1) { + + checkInterrupt(); + + /* Call every wake goal. */ + while (!awake.empty() && !topGoals.empty()) { + WeakGoals awake2(awake); + awake.clear(); + foreach (WeakGoals::iterator, i, awake2) { + checkInterrupt(); + GoalPtr goal = i->lock(); + if (goal) goal->work(); + if (topGoals.empty()) break; + } + } + + if (topGoals.empty()) break; + + /* Wait for input. */ + if (!children.empty() || !waitingForAWhile.empty()) + waitForInput(); + else { + if (awake.empty() && settings.maxBuildJobs == 0) throw Error( + "unable to start any build; either increase `--max-jobs' " + "or enable distributed builds"); + assert(!awake.empty()); + } + } + + /* If --keep-going is not set, it's possible that the main goal + exited while some of its subgoals were still active. But if + --keep-going *is* set, then they must all be finished now. */ + assert(!settings.keepGoing || awake.empty()); + assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || children.empty()); +} + + +void Worker::waitForInput() +{ + printMsg(lvlVomit, "waiting for children"); + + /* Process output from the file descriptors attached to the + children, namely log output and output path creation commands. + We also use this to detect child termination: if we get EOF on + the logger pipe of a build, we assume that the builder has + terminated. */ + + bool useTimeout = false; + struct timeval timeout; + timeout.tv_usec = 0; + time_t before = time(0); + + /* If we're monitoring for silence on stdout/stderr, or if there + is a build timeout, then wait for input until the first + deadline for any child. */ + assert(sizeof(time_t) >= sizeof(long)); + time_t nearest = LONG_MAX; // nearest deadline + foreach (Children::iterator, i, children) { + if (!i->second.respectTimeouts) continue; + if (settings.maxSilentTime != 0) + nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime); + if (settings.buildTimeout != 0) + nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout); + } + if (nearest != LONG_MAX) { + timeout.tv_sec = std::max((time_t) 1, nearest - before); + useTimeout = true; + printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); + } + + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + if (!waitingForAWhile.empty()) { + useTimeout = true; + if (lastWokenUp == 0) + printMsg(lvlError, "waiting for locks or build slots..."); + if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; + timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); + } else lastWokenUp = 0; + + using namespace std; + /* Use select() to wait for the input side of any logger pipe to + become `available'. Note that `available' (i.e., non-blocking) + includes EOF. */ + fd_set fds; + FD_ZERO(&fds); + int fdMax = 0; + foreach (Children::iterator, i, children) { + foreach (set<int>::iterator, j, i->second.fds) { + FD_SET(*j, &fds); + if (*j >= fdMax) fdMax = *j + 1; + } + } + + if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (errno == EINTR) return; + throw SysError("waiting for input"); + } + + time_t after = time(0); + + /* Process all available file descriptors. */ + + /* Since goals may be canceled from inside the loop below (causing + them go be erased from the `children' map), we have to be + careful that we don't keep iterators alive across calls to + cancel(). */ + set<pid_t> pids; + foreach (Children::iterator, i, children) pids.insert(i->first); + + foreach (set<pid_t>::iterator, i, pids) { + checkInterrupt(); + Children::iterator j = children.find(*i); + if (j == children.end()) continue; // child destroyed + GoalPtr goal = j->second.goal.lock(); + assert(goal); + + set<int> fds2(j->second.fds); + foreach (set<int>::iterator, k, fds2) { + if (FD_ISSET(*k, &fds)) { + unsigned char buffer[4096]; + ssize_t rd = read(*k, buffer, sizeof(buffer)); + if (rd == -1) { + if (errno != EINTR) + throw SysError(format("reading from %1%") + % goal->getName()); + } else if (rd == 0) { + debug(format("%1%: got EOF") % goal->getName()); + goal->handleEOF(*k); + j->second.fds.erase(*k); + } else { + printMsg(lvlVomit, format("%1%: read %2% bytes") + % goal->getName() % rd); + string data((char *) buffer, rd); + goal->handleChildOutput(*k, data); + j->second.lastOutput = after; + } + } + } + + if (settings.maxSilentTime != 0 && + j->second.respectTimeouts && + after - j->second.lastOutput >= (time_t) settings.maxSilentTime) + { + printMsg(lvlError, + format("%1% timed out after %2% seconds of silence") + % goal->getName() % settings.maxSilentTime); + goal->cancel(true); + } + + if (settings.buildTimeout != 0 && + j->second.respectTimeouts && + after - j->second.timeStarted >= (time_t) settings.buildTimeout) + { + printMsg(lvlError, + format("%1% timed out after %2% seconds") + % goal->getName() % settings.buildTimeout); + goal->cancel(true); + } + } + + if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { + lastWokenUp = after; + foreach (WeakGoals::iterator, i, waitingForAWhile) { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + waitingForAWhile.clear(); + } +} + + +unsigned int Worker::exitStatus() +{ + return permanentFailure ? 100 : 1; +} + + +////////////////////////////////////////////////////////////////////// + + +void LocalStore::buildPaths(const PathSet & drvPaths, bool repair) +{ + startNest(nest, lvlDebug, + format("building %1%") % showPaths(drvPaths)); + + Worker worker(*this); + + Goals goals; + foreach (PathSet::const_iterator, i, drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + if (isDerivation(i2.first)) + goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair)); + else + goals.insert(worker.makeSubstitutionGoal(*i, repair)); + } + + worker.run(goals); + + PathSet failed; + foreach (Goals::iterator, i, goals) + if ((*i)->getExitCode() == Goal::ecFailed) { + DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get()); + if (i2) failed.insert(i2->getDrvPath()); + else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath()); + } + + if (!failed.empty()) + throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); +} + + +void LocalStore::ensurePath(const Path & path) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path); + Goals goals = singleton<Goals>(goal); + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) + throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus()); +} + + +void LocalStore::repairPath(const Path & path) +{ + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path, true); + Goals goals = singleton<Goals>(goal); + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) + throw Error(format("cannot repair path `%1%'") % path, worker.exitStatus()); +} + + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc new file mode 100644 index 000000000000..d91e42784ca5 --- /dev/null +++ b/src/libstore/derivations.cc @@ -0,0 +1,278 @@ +#include "derivations.hh" +#include "store-api.hh" +#include "globals.hh" +#include "util.hh" +#include "misc.hh" + + +namespace nix { + + +void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const +{ + recursive = false; + string algo = hashAlgo; + + if (string(algo, 0, 2) == "r:") { + recursive = true; + algo = string(algo, 2); + } + + hashType = parseHashType(algo); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm `%1%'") % algo); + + hash = parseHash(hashType, this->hash); +} + + +Path writeDerivation(StoreAPI & store, + const Derivation & drv, const string & name, bool repair) +{ + PathSet references; + references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) + references.insert(i->first); + /* Note that the outputs of a derivation are *not* references + (that can be missing (of course) and should not necessarily be + held during a garbage collection). */ + string suffix = name + drvExtension; + string contents = unparseDerivation(drv); + return settings.readOnlyMode + ? computeStorePathForText(suffix, contents, references) + : store.addTextToStore(suffix, contents, references, repair); +} + + +static Path parsePath(std::istream & str) +{ + string s = parseString(str); + if (s.size() == 0 || s[0] != '/') + throw Error(format("bad path `%1%' in derivation") % s); + return s; +} + + +static StringSet parseStrings(std::istream & str, bool arePaths) +{ + StringSet res; + while (!endOfList(str)) + res.insert(arePaths ? parsePath(str) : parseString(str)); + return res; +} + + +Derivation parseDerivation(const string & s) +{ + Derivation drv; + std::istringstream str(s); + expect(str, "Derive(["); + + /* Parse the list of outputs. */ + while (!endOfList(str)) { + DerivationOutput out; + expect(str, "("); string id = parseString(str); + expect(str, ","); out.path = parsePath(str); + expect(str, ","); out.hashAlgo = parseString(str); + expect(str, ","); out.hash = parseString(str); + expect(str, ")"); + drv.outputs[id] = out; + } + + /* Parse the list of input derivations. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + Path drvPath = parsePath(str); + expect(str, ",["); + drv.inputDrvs[drvPath] = parseStrings(str, false); + expect(str, ")"); + } + + expect(str, ",["); drv.inputSrcs = parseStrings(str, true); + expect(str, ","); drv.platform = parseString(str); + expect(str, ","); drv.builder = parseString(str); + + /* Parse the builder arguments. */ + expect(str, ",["); + while (!endOfList(str)) + drv.args.push_back(parseString(str)); + + /* Parse the environment variables. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); string name = parseString(str); + expect(str, ","); string value = parseString(str); + expect(str, ")"); + drv.env[name] = value; + } + + expect(str, ")"); + return drv; +} + + +static void printString(string & res, const string & s) +{ + res += '"'; + for (const char * i = s.c_str(); *i; i++) + if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } + else if (*i == '\n') res += "\\n"; + else if (*i == '\r') res += "\\r"; + else if (*i == '\t') res += "\\t"; + else res += *i; + res += '"'; +} + + +template<class ForwardIterator> +static void printStrings(string & res, ForwardIterator i, ForwardIterator j) +{ + res += '['; + bool first = true; + for ( ; i != j; ++i) { + if (first) first = false; else res += ','; + printString(res, *i); + } + res += ']'; +} + + +string unparseDerivation(const Derivation & drv) +{ + string s; + s.reserve(65536); + s += "Derive(["; + + bool first = true; + foreach (DerivationOutputs::const_iterator, i, drv.outputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i->first); + s += ','; printString(s, i->second.path); + s += ','; printString(s, i->second.hashAlgo); + s += ','; printString(s, i->second.hash); + s += ')'; + } + + s += "],["; + first = true; + foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i->first); + s += ','; printStrings(s, i->second.begin(), i->second.end()); + s += ')'; + } + + s += "],"; + printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end()); + + s += ','; printString(s, drv.platform); + s += ','; printString(s, drv.builder); + s += ','; printStrings(s, drv.args.begin(), drv.args.end()); + + s += ",["; + first = true; + foreach (StringPairs::const_iterator, i, drv.env) { + if (first) first = false; else s += ','; + s += '('; printString(s, i->first); + s += ','; printString(s, i->second); + s += ')'; + } + + s += "])"; + + return s; +} + + +bool isDerivation(const string & fileName) +{ + return hasSuffix(fileName, drvExtension); +} + + +bool isFixedOutputDrv(const Derivation & drv) +{ + return drv.outputs.size() == 1 && + drv.outputs.begin()->first == "out" && + drv.outputs.begin()->second.hash != ""; +} + + +DrvHashes drvHashes; + + +/* Returns the hash of a derivation modulo fixed-output + subderivations. A fixed-output derivation is a derivation with one + output (`out') for which an expected hash and hash algorithm are + specified (using the `outputHash' and `outputHashAlgo' + attributes). We don't want changes to such derivations to + propagate upwards through the dependency graph, changing output + paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it + (after all, (the hash of) the file being downloaded is unchanged). + So the *output paths* should not change. On the other hand, the + *derivation paths* should change to reflect the new dependency + graph. + + That's what this function does: it returns a hash which is just the + hash of the derivation ATerm, except that any input derivation + paths have been replaced by the result of a recursive call to this + function, and that for fixed-output derivations we return a hash of + its output path. */ +Hash hashDerivationModulo(StoreAPI & store, Derivation drv) +{ + /* Return a fixed hash for fixed-output derivations. */ + if (isFixedOutputDrv(drv)) { + DerivationOutputs::const_iterator i = drv.outputs.begin(); + return hashString(htSHA256, "fixed:out:" + + i->second.hashAlgo + ":" + + i->second.hash + ":" + + i->second.path); + } + + /* For other derivations, replace the inputs paths with recursive + calls to this function.*/ + DerivationInputs inputs2; + foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { + Hash h = drvHashes[i->first]; + if (h.type == htUnknown) { + assert(store.isValidPath(i->first)); + Derivation drv2 = parseDerivation(readFile(i->first)); + h = hashDerivationModulo(store, drv2); + drvHashes[i->first] = h; + } + inputs2[printHash(h)] = i->second; + } + drv.inputDrvs = inputs2; + + return hashString(htSHA256, unparseDerivation(drv)); +} + + +DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) +{ + size_t n = s.find("!"); + return n == s.npos + ? DrvPathWithOutputs(s, std::set<string>()) + : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); +} + + +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) +{ + return outputs.empty() + ? drvPath + : drvPath + "!" + concatStringsSep(",", outputs); +} + + +bool wantOutput(const string & output, const std::set<string> & wanted) +{ + return wanted.empty() || wanted.find(output) != wanted.end(); +} + + +} diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh new file mode 100644 index 000000000000..703410b92552 --- /dev/null +++ b/src/libstore/derivations.hh @@ -0,0 +1,93 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +#include <map> + + +namespace nix { + + +/* Extension of derivations in the Nix store. */ +const string drvExtension = ".drv"; + + +/* Abstract syntax of derivations. */ + +struct DerivationOutput +{ + Path path; + string hashAlgo; /* hash used for expected hash computation */ + string hash; /* expected hash, may be null */ + DerivationOutput() + { + } + DerivationOutput(Path path, string hashAlgo, string hash) + { + this->path = path; + this->hashAlgo = hashAlgo; + this->hash = hash; + } + void parseHashInfo(bool & recursive, HashType & hashType, Hash & hash) const; +}; + +typedef std::map<string, DerivationOutput> DerivationOutputs; + +/* For inputs that are sub-derivations, we specify exactly which + output IDs we are interested in. */ +typedef std::map<Path, StringSet> DerivationInputs; + +typedef std::map<string, string> StringPairs; + +struct Derivation +{ + DerivationOutputs outputs; /* keyed on symbolic IDs */ + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + PathSet inputSrcs; /* inputs that are sources */ + string platform; + Path builder; + Strings args; + StringPairs env; +}; + + +class StoreAPI; + + +/* Write a derivation to the Nix store, and return its path. */ +Path writeDerivation(StoreAPI & store, + const Derivation & drv, const string & name, bool repair = false); + +/* Parse a derivation. */ +Derivation parseDerivation(const string & s); + +/* Print a derivation. */ +string unparseDerivation(const Derivation & drv); + +/* Check whether a file name ends with the extensions for + derivations. */ +bool isDerivation(const string & fileName); + +/* Return true iff this is a fixed-output derivation. */ +bool isFixedOutputDrv(const Derivation & drv); + +Hash hashDerivationModulo(StoreAPI & store, Derivation drv); + +/* Memoisation of hashDerivationModulo(). */ +typedef std::map<Path, Hash> DrvHashes; + +extern DrvHashes drvHashes; + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path and + the outputs. */ +typedef std::pair<string, std::set<string> > DrvPathWithOutputs; +DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); + +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); + +bool wantOutput(const string & output, const std::set<string> & wanted); + + +} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc new file mode 100644 index 000000000000..79bd7d56b3fe --- /dev/null +++ b/src/libstore/gc.cc @@ -0,0 +1,752 @@ +#include "globals.hh" +#include "misc.hh" +#include "local-store.hh" + +#include <boost/shared_ptr.hpp> + +#include <functional> +#include <queue> +#include <algorithm> + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> + + +namespace nix { + + +static string gcLockName = "gc.lock"; +static string tempRootsDir = "temproots"; +static string gcRootsDir = "gcroots"; + + +/* Acquire the global GC lock. This is used to prevent new Nix + processes from starting after the temporary root files have been + read. To be precise: when they try to create a new temporary root + file, they will block until the garbage collector has finished / + yielded the GC lock. */ +int LocalStore::openGCLock(LockType lockType) +{ + Path fnGCLock = (format("%1%/%2%") + % settings.nixStateDir % gcLockName).str(); + + debug(format("acquiring global GC lock `%1%'") % fnGCLock); + + AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600); + if (fdGCLock == -1) + throw SysError(format("opening global GC lock `%1%'") % fnGCLock); + closeOnExec(fdGCLock); + + if (!lockFile(fdGCLock, lockType, false)) { + printMsg(lvlError, format("waiting for the big garbage collector lock...")); + lockFile(fdGCLock, lockType, true); + } + + /* !!! Restrict read permission on the GC root. Otherwise any + process that can open the file for reading can DoS the + collector. */ + + return fdGCLock.borrow(); +} + + +void createSymlink(const Path & link, const Path & target) +{ + /* Create directories up to `gcRoot'. */ + createDirs(dirOf(link)); + + /* Create the new symlink. */ + Path tempLink = (format("%1%.tmp-%2%-%3%") + % link % getpid() % rand()).str(); + if (symlink(target.c_str(), tempLink.c_str()) == -1) + throw SysError(format("symlinking `%1%' to `%2%'") + % tempLink % target); + + /* Atomically replace the old one. */ + if (rename(tempLink.c_str(), link.c_str()) == -1) + throw SysError(format("cannot rename `%1%' to `%2%'") + % tempLink % link); +} + + +void LocalStore::syncWithGC() +{ + AutoCloseFD fdGCLock = openGCLock(ltRead); +} + + +void LocalStore::addIndirectRoot(const Path & path) +{ + string hash = printHash32(hashString(htSHA1, path)); + Path realRoot = canonPath((format("%1%/%2%/auto/%3%") + % settings.nixStateDir % gcRootsDir % hash).str()); + createSymlink(realRoot, path); +} + + +Path addPermRoot(StoreAPI & store, const Path & _storePath, + const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) +{ + Path storePath(canonPath(_storePath)); + Path gcRoot(canonPath(_gcRoot)); + assertStorePath(storePath); + + if (isInStore(gcRoot)) + throw Error(format( + "creating a garbage collector root (%1%) in the Nix store is forbidden " + "(are you running nix-build inside the store?)") % gcRoot); + + if (indirect) { + /* Don't clobber the the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error(format("cannot create symlink `%1%'; already exists") % gcRoot); + createSymlink(gcRoot, storePath); + store.addIndirectRoot(gcRoot); + } + + else { + if (!allowOutsideRootsDir) { + Path rootsDir = canonPath((format("%1%/%2%") % settings.nixStateDir % gcRootsDir).str()); + + if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") + throw Error(format( + "path `%1%' is not a valid garbage collector root; " + "it's not in the directory `%2%'") + % gcRoot % rootsDir); + } + + createSymlink(gcRoot, storePath); + } + + /* Check that the root can be found by the garbage collector. + !!! This can be very slow on machines that have many roots. + Instead of reading all the roots, it would be more efficient to + check if the root is in a directory in or linked from the + gcroots directory. */ + if (settings.checkRootReachability) { + Roots roots = store.findRoots(); + if (roots.find(gcRoot) == roots.end()) + printMsg(lvlError, + format( + "warning: `%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, `%2%' might be removed by the garbage collector") + % gcRoot % storePath); + } + + /* Grab the global GC root, causing us to block while a GC is in + progress. This prevents the set of permanent roots from + increasing while a GC is in progress. */ + store.syncWithGC(); + + return gcRoot; +} + + +/* The file to which we write our temporary roots. */ +static Path fnTempRoots; +static AutoCloseFD fdTempRoots; + + +void LocalStore::addTempRoot(const Path & path) +{ + /* Create the temporary roots file for this process. */ + if (fdTempRoots == -1) { + + while (1) { + Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); + createDirs(dir); + + fnTempRoots = (format("%1%/%2%") + % dir % getpid()).str(); + + AutoCloseFD fdGCLock = openGCLock(ltRead); + + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); + + fdTempRoots = openLockFile(fnTempRoots, true); + + fdGCLock.close(); + + debug(format("acquiring read lock on `%1%'") % fnTempRoots); + lockFile(fdTempRoots, ltRead, true); + + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(fdTempRoots, &st) == -1) + throw SysError(format("statting `%1%'") % fnTempRoots); + if (st.st_size == 0) break; + + /* The garbage collector deleted this file before we could + get a lock. (It won't delete the file after we get a + lock.) Try again. */ + } + + } + + /* Upgrade the lock to a write lock. This will cause us to block + if the garbage collector is holding our lock. */ + debug(format("acquiring write lock on `%1%'") % fnTempRoots); + lockFile(fdTempRoots, ltWrite, true); + + string s = path + '\0'; + writeFull(fdTempRoots, (const unsigned char *) s.data(), s.size()); + + /* Downgrade to a read lock. */ + debug(format("downgrading to read lock on `%1%'") % fnTempRoots); + lockFile(fdTempRoots, ltRead, true); +} + + +void removeTempRoots() +{ + if (fdTempRoots != -1) { + fdTempRoots.close(); + unlink(fnTempRoots.c_str()); + } +} + + +/* Automatically clean up the temporary roots file when we exit. */ +struct RemoveTempRoots +{ + ~RemoveTempRoots() + { + removeTempRoots(); + } +}; + +static RemoveTempRoots autoRemoveTempRoots __attribute__((unused)); + + +typedef boost::shared_ptr<AutoCloseFD> FDPtr; +typedef list<FDPtr> FDs; + + +static void readTempRoots(PathSet & tempRoots, FDs & fds) +{ + /* Read the `temproots' directory for per-process temporary root + files. */ + Strings tempRootFiles = readDirectory( + (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str()); + + foreach (Strings::iterator, i, tempRootFiles) { + Path path = (format("%1%/%2%/%3%") % settings.nixStateDir % tempRootsDir % *i).str(); + + debug(format("reading temporary root file `%1%'") % path); + FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666))); + if (*fd == -1) { + /* It's okay if the file has disappeared. */ + if (errno == ENOENT) continue; + throw SysError(format("opening temporary roots file `%1%'") % path); + } + + /* This should work, but doesn't, for some reason. */ + //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); + //if (*fd == -1) continue; + + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(*fd, ltWrite, false)) { + printMsg(lvlError, format("removing stale temporary roots file `%1%'") % path); + unlink(path.c_str()); + writeFull(*fd, (const unsigned char *) "d", 1); + continue; + } + + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on `%1%'") % path); + lockFile(*fd, ltRead, true); + + /* Read the entire file. */ + string contents = readFile(*fd); + + /* Extract the roots. */ + string::size_type pos = 0, end; + + while ((end = contents.find((char) 0, pos)) != string::npos) { + Path root(contents, pos, end - pos); + debug(format("got temporary root `%1%'") % root); + assertStorePath(root); + tempRoots.insert(root); + pos = end + 1; + } + + fds.push_back(fd); /* keep open */ + } +} + + +static void foundRoot(StoreAPI & store, + const Path & path, const Path & target, Roots & roots) +{ + Path storePath = toStorePath(target); + if (store.isValidPath(storePath)) + roots[path] = storePath; + else + printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'") % path % storePath); +} + + +static void findRoots(StoreAPI & store, const Path & path, Roots & roots) +{ + try { + + struct stat st = lstat(path); + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + findRoots(store, path + "/" + *i, roots); + } + + else if (S_ISLNK(st.st_mode)) { + Path target = readLink(path); + if (isInStore(target)) + foundRoot(store, path, target, roots); + + /* Handle indirect roots. */ + else { + target = absPath(target, dirOf(path)); + if (!pathExists(target)) { + if (isInDir(path, settings.nixStateDir + "/" + gcRootsDir + "/auto")) { + printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target); + unlink(path.c_str()); + } + } else { + struct stat st2 = lstat(target); + if (!S_ISLNK(st2.st_mode)) return; + Path target2 = readLink(target); + if (isInStore(target2)) foundRoot(store, target, target2, roots); + } + } + } + + } + + catch (SysError & e) { + /* We only ignore permanent failures. */ + if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) + printMsg(lvlInfo, format("cannot read potential root `%1%'") % path); + else + throw; + } +} + + +Roots LocalStore::findRoots() +{ + Roots roots; + + /* Process direct roots in {gcroots,manifests,profiles}. */ + nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, roots); + nix::findRoots(*this, settings.nixStateDir + "/manifests", roots); + nix::findRoots(*this, settings.nixStateDir + "/profiles", roots); + + return roots; +} + + +static void addAdditionalRoots(StoreAPI & store, PathSet & roots) +{ + Path rootFinder = getEnv("NIX_ROOT_FINDER", + settings.nixLibexecDir + "/nix/find-runtime-roots.pl"); + + if (rootFinder.empty()) return; + + debug(format("executing `%1%' to find additional roots") % rootFinder); + + string result = runProgram(rootFinder); + + StringSet paths = tokenizeString<StringSet>(result, "\n"); + + foreach (StringSet::iterator, i, paths) { + if (isInStore(*i)) { + Path path = toStorePath(*i); + if (roots.find(path) == roots.end() && store.isValidPath(path)) { + debug(format("got additional root `%1%'") % path); + roots.insert(path); + } + } + } +} + + +struct GCLimitReached { }; + + +struct LocalStore::GCState +{ + GCOptions options; + GCResults & results; + PathSet roots; + PathSet tempRoots; + PathSet dead; + PathSet alive; + bool gcKeepOutputs; + bool gcKeepDerivations; + unsigned long long bytesInvalidated; + Path trashDir; + bool shouldDelete; + GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } +}; + + +bool LocalStore::isActiveTempFile(const GCState & state, + const Path & path, const string & suffix) +{ + return hasSuffix(path, suffix) + && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); +} + + +void LocalStore::deleteGarbage(GCState & state, const Path & path) +{ + unsigned long long bytesFreed; + deletePath(path, bytesFreed); + state.results.bytesFreed += bytesFreed; +} + + +void LocalStore::deletePathRecursive(GCState & state, const Path & path) +{ + checkInterrupt(); + + unsigned long long size = 0; + + if (isValidPath(path)) { + PathSet referrers; + queryReferrers(path, referrers); + foreach (PathSet::iterator, i, referrers) + if (*i != path) deletePathRecursive(state, *i); + size = queryPathInfo(path).narSize; + invalidatePathChecked(path); + } + + struct stat st; + if (lstat(path.c_str(), &st)) { + if (errno == ENOENT) return; + throw SysError(format("getting status of %1%") % path); + } + + printMsg(lvlInfo, format("deleting `%1%'") % path); + + state.results.paths.insert(path); + + /* If the path is not a regular file or symlink, move it to the + trash directory. The move is to ensure that later (when we're + not holding the global GC lock) we can delete the path without + being afraid that the path has become alive again. Otherwise + delete it right away. */ + if (S_ISDIR(st.st_mode)) { + // Estimate the amount freed using the narSize field. FIXME: + // if the path was not valid, need to determine the actual + // size. + state.bytesInvalidated += size; + // Mac OS X cannot rename directories if they are read-only. + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making `%1%' writable") % path); + Path tmp = state.trashDir + "/" + baseNameOf(path); + if (rename(path.c_str(), tmp.c_str())) + throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp); + } else + deleteGarbage(state, path); + + if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { + printMsg(lvlInfo, format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); + throw GCLimitReached(); + } +} + + +bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) +{ + if (visited.find(path) != visited.end()) return false; + + if (state.alive.find(path) != state.alive.end()) { + return true; + } + + if (state.dead.find(path) != state.dead.end()) { + return false; + } + + if (state.roots.find(path) != state.roots.end()) { + printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path); + state.alive.insert(path); + return true; + } + + visited.insert(path); + + if (!isValidPath(path)) return false; + + PathSet incoming; + + /* Don't delete this path if any of its referrers are alive. */ + queryReferrers(path, incoming); + + /* If gc-keep-derivations is set and this is a derivation, then + don't delete the derivation if any of the outputs are alive. */ + if (state.gcKeepDerivations && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (isValidPath(*i) && queryDeriver(*i) == path) + incoming.insert(*i); + } + + /* If gc-keep-outputs is set, then don't delete this path if there + are derivers of this path that are not garbage. */ + if (state.gcKeepOutputs) { + PathSet derivers = queryValidDerivers(path); + foreach (PathSet::iterator, i, derivers) + incoming.insert(*i); + } + + foreach (PathSet::iterator, i, incoming) + if (*i != path) + if (canReachRoot(state, visited, *i)) { + state.alive.insert(path); + return true; + } + + return false; +} + + +void LocalStore::tryToDelete(GCState & state, const Path & path) +{ + checkInterrupt(); + + if (path == linksDir || path == state.trashDir) return; + + startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path); + + if (!isValidPath(path)) { + /* A lock file belonging to a path that we're building right + now isn't garbage. */ + if (isActiveTempFile(state, path, ".lock")) return; + + /* Don't delete .chroot directories for derivations that are + currently being built. */ + if (isActiveTempFile(state, path, ".chroot")) return; + } + + PathSet visited; + + if (canReachRoot(state, visited, path)) { + printMsg(lvlDebug, format("cannot delete `%1%' because it's still reachable") % path); + } else { + /* No path we visited was a root, so everything is garbage. + But we only delete ‘path’ and its referrers here so that + ‘nix-store --delete’ doesn't have the unexpected effect of + recursing into derivations and outputs. */ + state.dead.insert(visited.begin(), visited.end()); + if (state.shouldDelete) + deletePathRecursive(state, path); + } +} + + +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks(const GCState & state) +{ + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + long long actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % path); + + if (st.st_nlink != 1) { + unsigned long long size = st.st_blocks * 512ULL; + actualSize += size; + unsharedSize += (st.st_nlink - 1) * size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting `%1%'") % path); + + state.results.bytesFreed += st.st_blocks * 512; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % linksDir); + long long overhead = st.st_blocks * 512ULL; + + printMsg(lvlInfo, format("note: currently hard linking saves %.2f MiB") + % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); +} + + +void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + GCState state(results); + state.options = options; + state.trashDir = settings.nixStore + "/trash"; + state.gcKeepOutputs = settings.gcKeepOutputs; + state.gcKeepDerivations = settings.gcKeepDerivations; + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `gc-keep-outputs' or `gc-keep-derivations' are + true (the garbage collector will recurse into deleting the + outputs or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } + + state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + + /* Acquire the global GC root. This prevents + a) New roots from being added. + b) Processes from creating new temporary root files. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printMsg(lvlError, format("finding garbage collector roots...")); + Roots rootMap = options.ignoreLiveness ? Roots() : findRoots(); + + foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second); + + /* Add additional roots returned by the program specified by the + NIX_ROOT_FINDER environment variable. This is typically used + to add running programs to the set of roots (to prevent them + from being garbage collected). */ + if (!options.ignoreLiveness) + addAdditionalRoots(*this, state.roots); + + /* Read the temporary roots. This acquires read locks on all + per-process temporary root files. So after this point no paths + can be added to the set of temporary roots. */ + FDs fds; + readTempRoots(state.tempRoots, fds); + state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + + /* After this point the set of roots or temporary roots cannot + increase, since we hold locks on everything. So everything + that is not reachable from `roots'. */ + + if (state.shouldDelete) { + if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir); + createDirs(state.trashDir); + } + + /* Now either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ + + if (options.action == GCOptions::gcDeleteSpecific) { + + foreach (PathSet::iterator, i, options.pathsToDelete) { + assertStorePath(*i); + tryToDelete(state, *i); + if (state.dead.find(*i) == state.dead.end()) + throw Error(format("cannot delete path `%1%' since it is still alive") % *i); + } + + } else if (options.maxFreed > 0) { + + if (state.shouldDelete) + printMsg(lvlError, format("deleting garbage...")); + else + printMsg(lvlError, format("determining live/dead paths...")); + + try { + + AutoCloseDir dir = opendir(settings.nixStore.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % settings.nixStore); + + /* Read the store and immediately delete all paths that + aren't valid. When using --max-freed etc., deleting + invalid paths is preferred over deleting unreachable + paths, since unreachable paths could become reachable + again. We don't use readDirectory() here so that GCing + can start faster. */ + Paths entries; + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = settings.nixStore + "/" + name; + if (isValidPath(path)) + entries.push_back(path); + else + tryToDelete(state, path); + } + + dir.close(); + + /* Now delete the unreachable valid paths. Randomise the + order in which we delete entries to make the collector + less biased towards deleting paths that come + alphabetically first (e.g. /nix/store/000...). This + matters when using --max-freed etc. */ + vector<Path> entries_(entries.begin(), entries.end()); + random_shuffle(entries_.begin(), entries_.end()); + + foreach (vector<Path>::iterator, i, entries_) + tryToDelete(state, *i); + + } catch (GCLimitReached & e) { + } + } + + if (state.options.action == GCOptions::gcReturnLive) { + state.results.paths = state.alive; + return; + } + + if (state.options.action == GCOptions::gcReturnDead) { + state.results.paths = state.dead; + return; + } + + /* Allow other processes to add to the store from here on. */ + fdGCLock.close(); + fds.clear(); + + /* Delete the trash directory. */ + printMsg(lvlInfo, format("deleting `%1%'") % state.trashDir); + deleteGarbage(state, state.trashDir); + + /* Clean up the links directory. */ + if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { + printMsg(lvlError, format("deleting unused links...")); + removeUnusedLinks(state); + } + + /* While we're at it, vacuum the database. */ + if (options.action == GCOptions::gcDeleteDead) vacuumDB(); +} + + +} diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc new file mode 100644 index 000000000000..68add1982f43 --- /dev/null +++ b/src/libstore/globals.cc @@ -0,0 +1,218 @@ +#include "config.h" + +#include "globals.hh" +#include "util.hh" + +#include <map> +#include <algorithm> + + +namespace nix { + + +/* The default location of the daemon socket, relative to nixStateDir. + The socket is in a directory to allow you to control access to the + Nix daemon by setting the mode/ownership of the directory + appropriately. (This wouldn't work on the socket itself since it + must be deleted and recreated on startup.) */ +#define DEFAULT_SOCKET_PATH "/daemon-socket/socket" + + +Settings settings; + + +Settings::Settings() +{ + keepFailed = false; + keepGoing = false; + tryFallback = false; + buildVerbosity = lvlError; + maxBuildJobs = 1; + buildCores = 1; + readOnlyMode = false; + thisSystem = SYSTEM; + maxSilentTime = 0; + buildTimeout = 0; + useBuildHook = true; + printBuildTrace = false; + reservedSize = 1024 * 1024; + fsyncMetadata = true; + useSQLiteWAL = true; + syncBeforeRegistering = false; + useSubstitutes = true; + useChroot = false; + dirsInChroot.insert("/dev"); + dirsInChroot.insert("/dev/pts"); + impersonateLinux26 = false; + keepLog = true; + compressLog = true; + maxLogSize = 0; + cacheFailure = false; + pollInterval = 5; + checkRootReachability = false; + gcKeepOutputs = false; + gcKeepDerivations = true; + autoOptimiseStore = false; + envKeepDerivations = false; + lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; + showTrace = false; +} + + +void Settings::processEnvironment() +{ + nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); + nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); + nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); + nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); + nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db"); + nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); + nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); + nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); + nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); + + string subs = getEnv("NIX_SUBSTITUTERS", "default"); + if (subs == "default") { +#if 0 + if (getEnv("NIX_OTHER_STORES") != "") + substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); +#endif + substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); + } else + substituters = tokenizeString<Strings>(subs, ":"); +} + + +void Settings::loadConfFile() +{ + Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); + if (!pathExists(settingsFile)) return; + string contents = readFile(settingsFile); + + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2 || tokens[1] != "=") + throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile); + + string name = tokens[0]; + + vector<string>::iterator i = tokens.begin(); + advance(i, 2); + settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow + }; +} + + +void Settings::set(const string & name, const string & value) +{ + settings[name] = value; + overrides[name] = value; +} + + +void Settings::update() +{ + get(tryFallback, "build-fallback"); + get(maxBuildJobs, "build-max-jobs"); + get(buildCores, "build-cores"); + get(thisSystem, "system"); + get(maxSilentTime, "build-max-silent-time"); + get(buildTimeout, "build-timeout"); + get(reservedSize, "gc-reserved-space"); + get(fsyncMetadata, "fsync-metadata"); + get(useSQLiteWAL, "use-sqlite-wal"); + get(syncBeforeRegistering, "sync-before-registering"); + get(useSubstitutes, "build-use-substitutes"); + get(buildUsersGroup, "build-users-group"); + get(useChroot, "build-use-chroot"); + get(dirsInChroot, "build-chroot-dirs"); + get(impersonateLinux26, "build-impersonate-linux-26"); + get(keepLog, "build-keep-log"); + get(compressLog, "build-compress-log"); + get(maxLogSize, "build-max-log-size"); + get(cacheFailure, "build-cache-failure"); + get(pollInterval, "build-poll-interval"); + get(checkRootReachability, "gc-check-reachability"); + get(gcKeepOutputs, "gc-keep-outputs"); + get(gcKeepDerivations, "gc-keep-derivations"); + get(autoOptimiseStore, "auto-optimise-store"); + get(envKeepDerivations, "env-keep-derivations"); +} + + +void Settings::get(string & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res = i->second; +} + + +void Settings::get(bool & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + if (i->second == "true") res = true; + else if (i->second == "false") res = false; + else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'") + % name % i->second); +} + + +void Settings::get(StringSet & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res.clear(); + Strings ss = tokenizeString<Strings>(i->second); + res.insert(ss.begin(), ss.end()); +} + + +template<class N> void Settings::get(N & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + if (!string2Int(i->second, res)) + throw Error(format("configuration setting `%1%' should have an integer value") % name); +} + + +string Settings::pack() +{ + string s; + foreach (SettingsMap::iterator, i, settings) { + if (i->first.find('\n') != string::npos || + i->first.find('=') != string::npos || + i->second.find('\n') != string::npos) + throw Error("illegal option name/value"); + s += i->first; s += '='; s += i->second; s += '\n'; + } + return s; +} + + +Settings::SettingsMap Settings::getOverrides() +{ + return overrides; +} + + +const string nixVersion = PACKAGE_VERSION; + + +} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh new file mode 100644 index 000000000000..9300edbe9615 --- /dev/null +++ b/src/libstore/globals.hh @@ -0,0 +1,209 @@ +#pragma once + +#include "types.hh" + +#include <map> +#include <sys/types.h> + + +namespace nix { + + +struct Settings { + + typedef std::map<string, string> SettingsMap; + + Settings(); + + void processEnvironment(); + + void loadConfFile(); + + void set(const string & name, const string & value); + + void update(); + + string pack(); + + SettingsMap getOverrides(); + + /* The directory where we store sources and derived files. */ + Path nixStore; + + Path nixDataDir; /* !!! fix */ + + /* The directory where we log various operations. */ + Path nixLogDir; + + /* The directory where state is stored. */ + Path nixStateDir; + + /* The directory where we keep the SQLite database. */ + Path nixDBPath; + + /* The directory where configuration files are stored. */ + Path nixConfDir; + + /* The directory where internal helper programs are stored. */ + Path nixLibexecDir; + + /* The directory where the main programs are stored. */ + Path nixBinDir; + + /* File name of the socket the daemon listens to. */ + Path nixDaemonSocketFile; + + /* Whether to keep temporary directories of failed builds. */ + bool keepFailed; + + /* Whether to keep building subgoals when a sibling (another + subgoal of the same goal) fails. */ + bool keepGoing; + + /* Whether, if we cannot realise the known closure corresponding + to a derivation, we should try to normalise the derivation + instead. */ + bool tryFallback; + + /* Verbosity level for build output. */ + Verbosity buildVerbosity; + + /* Maximum number of parallel build jobs. 0 means unlimited. */ + unsigned int maxBuildJobs; + + /* Number of CPU cores to utilize in parallel within a build, + i.e. by passing this number to Make via '-j'. 0 means that the + number of actual CPU cores on the local host ought to be + auto-detected. */ + unsigned int buildCores; + + /* Read-only mode. Don't copy stuff to the store, don't change + the database. */ + bool readOnlyMode; + + /* The canonical system name, as returned by config.guess. */ + string thisSystem; + + /* The maximum time in seconds that a builer can go without + producing any output on stdout/stderr before it is killed. 0 + means infinity. */ + time_t maxSilentTime; + + /* The maximum duration in seconds that a builder can run. 0 + means infinity. */ + time_t buildTimeout; + + /* The substituters. There are programs that can somehow realise + a store path without building, e.g., by downloading it or + copying it from a CD. */ + Paths substituters; + + /* Whether to use build hooks (for distributed builds). Sometimes + users want to disable this from the command-line. */ + bool useBuildHook; + + /* Whether buildDerivations() should print out lines on stderr in + a fixed format to allow its progress to be monitored. Each + line starts with a "@". The following are defined: + + @ build-started <drvpath> <outpath> <system> <logfile> + @ build-failed <drvpath> <outpath> <exitcode> <error text> + @ build-succeeded <drvpath> <outpath> + @ substituter-started <outpath> <substituter> + @ substituter-failed <outpath> <exitcode> <error text> + @ substituter-succeeded <outpath> + + Best combined with --no-build-output, otherwise stderr might + conceivably contain lines in this format printed by the + builders. */ + bool printBuildTrace; + + /* Amount of reserved space for the garbage collector + (/nix/var/nix/db/reserved). */ + off_t reservedSize; + + /* Whether SQLite should use fsync. */ + bool fsyncMetadata; + + /* Whether SQLite should use WAL mode. */ + bool useSQLiteWAL; + + /* Whether to call sync() before registering a path as valid. */ + bool syncBeforeRegistering; + + /* Whether to use substitutes. */ + bool useSubstitutes; + + /* The Unix group that contains the build users. */ + string buildUsersGroup; + + /* Whether to build in chroot. */ + bool useChroot; + + /* The directories from the host filesystem to be included in the + chroot. */ + StringSet dirsInChroot; + + /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ + bool impersonateLinux26; + + /* Whether to store build logs. */ + bool keepLog; + + /* Whether to compress logs. */ + bool compressLog; + + /* Maximum number of bytes a builder can write to stdout/stderr + before being killed (0 means no limit). */ + unsigned long maxLogSize; + + /* Whether to cache build failures. */ + bool cacheFailure; + + /* How often (in seconds) to poll for locks. */ + unsigned int pollInterval; + + /* Whether to check if new GC roots can in fact be found by the + garbage collector. */ + bool checkRootReachability; + + /* Whether the garbage collector should keep outputs of live + derivations. */ + bool gcKeepOutputs; + + /* Whether the garbage collector should keep derivers of live + paths. */ + bool gcKeepDerivations; + + /* Whether to automatically replace files with identical contents + with hard links. */ + bool autoOptimiseStore; + + /* Whether to add derivations as a dependency of user environments + (to prevent them from being GCed). */ + bool envKeepDerivations; + + /* Whether to lock the Nix client and worker to the same CPU. */ + bool lockCPU; + + /* Whether to show a stack trace if Nix evaluation fails. */ + bool showTrace; + +private: + SettingsMap settings, overrides; + + void get(string & res, const string & name); + void get(bool & res, const string & name); + void get(StringSet & res, const string & name); + template<class N> void get(N & res, const string & name); +}; + + +// FIXME: don't use a global variable. +extern Settings settings; + + +extern const string nixVersion; + + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc new file mode 100644 index 000000000000..3c367c080815 --- /dev/null +++ b/src/libstore/local-store.cc @@ -0,0 +1,1996 @@ +#include "config.h" +#include "local-store.hh" +#include "globals.hh" +#include "archive.hh" +#include "pathlocks.hh" +#include "worker-protocol.hh" +#include "derivations.hh" +#include "affinity.hh" + +#include <iostream> +#include <algorithm> +#include <cstring> + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> +#include <utime.h> +#include <fcntl.h> +#include <errno.h> +#include <stdio.h> +#include <time.h> + +#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H +#include <sched.h> +#include <sys/statvfs.h> +#include <sys/mount.h> +#endif + +#if HAVE_LINUX_FS_H +#include <linux/fs.h> +#include <sys/ioctl.h> +#include <errno.h> +#endif + +#include <sqlite3.h> + + +namespace nix { + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + + +static void throwSQLiteError(sqlite3 * db, const format & f) + __attribute__ ((noreturn)); + +static void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + + +/* Convenience macros for retrying a SQLite transaction. */ +#define retry_sqlite while (1) { try { +#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } + + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + + +void SQLiteStmt::reset() +{ + assert(stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); + curArg = 1; +} + + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + + +void SQLiteStmt::bind(const string & value) +{ + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind(int value) +{ + if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind64(long long value) +{ + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(db, "binding argument"); +} + + +/* Helper class to ensure that prepared statements are reset when + leaving the scope that uses them. Unfinished prepared statements + prevent transactions from being aborted, and can cause locks to be + kept when they should be released. */ +struct SQLiteStmtUse +{ + SQLiteStmt & stmt; + SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) + { + stmt.reset(); + } + ~SQLiteStmtUse() + { + try { + stmt.reset(); + } catch (...) { + ignoreException(); + } + } +}; + + +struct SQLiteTxn +{ + bool active; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db) : active(false) { + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; + } + + void commit() + { + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; + } + + ~SQLiteTxn() + { + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } + } +}; + + +void checkStoreNotSymlink() +{ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; + Path path = settings.nixStore; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if (S_ISLNK(st.st_mode)) + throw Error(format( + "the path `%1%' is a symlink; " + "this is not allowed for the Nix store and its parent directories") + % path); + path = dirOf(path); + } +} + + +LocalStore::LocalStore(bool reserveSpace) + : didSetSubstituterEnv(false) +{ + schemaPath = settings.nixDBPath + "/schema"; + + if (settings.readOnlyMode) { + openDB(false); + return; + } + + /* Create missing state directories if they don't already exist. */ + createDirs(settings.nixStore); + makeStoreWritable(); + createDirs(linksDir = settings.nixStore + "/.links"); + Path profilesDir = settings.nixStateDir + "/profiles"; + createDirs(settings.nixStateDir + "/profiles"); + createDirs(settings.nixStateDir + "/temproots"); + createDirs(settings.nixDBPath); + Path gcRootsDir = settings.nixStateDir + "/gcroots"; + if (!pathExists(gcRootsDir)) { + createDirs(gcRootsDir); + if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1) + throw SysError(format("creating symlink to `%1%'") % profilesDir); + } + + checkStoreNotSymlink(); + + /* We can't open a SQLite database if the disk is full. Since + this prevents the garbage collector from running when it's most + needed, we reserve some dummy space that we can free just + before doing a garbage collection. */ + try { + Path reservedPath = settings.nixDBPath + "/reserved"; + if (reserveSpace) { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) + writeFile(reservedPath, string(settings.reservedSize, 'X')); + } + else + deletePath(reservedPath); + } catch (SysError & e) { /* don't care about errors */ + } + + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ + try { + Path globalLockPath = settings.nixDBPath + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } catch (SysError & e) { + if (e.errNo != EACCES) throw; + settings.readOnlyMode = true; + openDB(false); + return; + } + + if (!lockFile(globalLock, ltRead, false)) { + printMsg(lvlError, "waiting for the big Nix store lock..."); + lockFile(globalLock, ltRead, true); + } + + /* Check the current database schema and if necessary do an + upgrade. */ + int curSchema = getSchema(); + if (curSchema > nixSchemaVersion) + throw Error(format("current Nix store schema is version %1%, but I only support %2%") + % curSchema % nixSchemaVersion); + + else if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + openDB(true); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + } + + else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + + if (!lockFile(globalLock, ltWrite, false)) { + printMsg(lvlError, "waiting for exclusive access to the Nix store..."); + lockFile(globalLock, ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 6) upgradeStore6(); + else if (curSchema < 7) { upgradeStore7(); openDB(true); } + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock, ltRead, true); + } + + else openDB(false); +} + + +LocalStore::~LocalStore() +{ + try { + foreach (RunningSubstituters::iterator, i, runningSubstituters) { + if (i->second.disabled) continue; + i->second.to.close(); + i->second.from.close(); + i->second.error.close(); + i->second.pid.wait(true); + } + } catch (...) { + ignoreException(); + } +} + + +int LocalStore::getSchema() +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error(format("`%1%' is corrupt") % schemaPath); + } + return curSchema; +} + + +void LocalStore::openDB(bool create) +{ + /* Open the Nix database. */ + if (sqlite3_open_v2((settings.nixDBPath + "/db.sqlite").c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "enabling foreign keys"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = settings.fsyncMetadata ? "normal" : "off"; + if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting synchronous mode"); + + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) + throwSQLiteError(db, "querying journal mode"); + prevMode = string((const char *) sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting journal mode"); + + /* Increase the auto-checkpoint interval to 40000 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting autocheckpoint interval"); + + /* Initialise the database schema, if necessary. */ + if (create) { + const char * schema = +#include "schema.sql.hh" + ; + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "initialising database schema"); + } + + /* Prepare SQL statements. */ + stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); + stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ? where path = ?;"); + stmtAddReference.create(db, + "insert or replace into Refs (referrer, reference) values (?, ?);"); + stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); + stmtQueryReferences.create(db, + "select path from Refs join ValidPaths on reference = id where referrer = ?;"); + stmtQueryReferrers.create(db, + "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); + stmtInvalidatePath.create(db, + "delete from ValidPaths where path = ?;"); + stmtRegisterFailedPath.create(db, + "insert or ignore into FailedPaths (path, time) values (?, ?);"); + stmtHasPathFailed.create(db, + "select time from FailedPaths where path = ?;"); + stmtQueryFailedPaths.create(db, + "select path from FailedPaths;"); + // If the path is a derivation, then clear its outputs. + stmtClearFailedPath.create(db, + "delete from FailedPaths where ?1 = '*' or path = ?1 " + "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); + stmtAddDerivationOutput.create(db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); + stmtQueryValidDerivers.create(db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); + stmtQueryDerivationOutputs.create(db, + "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + stmtQueryPathFromHashPart.create(db, + "select path from ValidPaths where path >= ? limit 1;"); +} + + +/* To improve purity, users may want to make the Nix store a read-only + bind mount. So make the Nix store writable for this process. */ +void LocalStore::makeStoreWritable() +{ +#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT) + if (getuid() != 0) return; + /* Check if /nix/store is on a read-only mount. */ + struct statvfs stat; + if (statvfs(settings.nixStore.c_str(), &stat) !=0) + throw SysError("Getting info of nix store mountpoint"); + + if (stat.f_flag & ST_RDONLY) { + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError(format("remounting %1% writable") % settings.nixStore); + } +#endif +} + + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of `%1%'") % path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + canonicaliseTimestampAndPermissions(path, st); +} + + +static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + /* Really make sure that the path is of a supported type. This + has already been checked in dumpPath(). */ + assert(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)); + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { + assert(!S_ISDIR(st.st_mode)); + if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + throw BuildError(format("invalid ownership on file `%1%'") % path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), (gid_t) -1) == -1) +#endif + throw SysError(format("changing owner of `%1%' to %2%") + % path % geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + canonicalisePathMetaData_(path + "/" + *i, fromUid, inodesSeen); + } +} + + +void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, fromUid, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error(format("wrong ownership of top-level store path `%1%'") % path); + } +} + + +void canonicalisePathMetaData(const Path & path, uid_t fromUid) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, fromUid, inodesSeen); +} + + +void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) +{ + string drvName = storePathToName(drvPath); + assert(isDerivation(drvName)); + drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + + if (isFixedOutputDrv(drv)) { + DerivationOutputs::const_iterator out = drv.outputs.find("out"); + if (out == drv.outputs.end()) + throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath); + + bool recursive; HashType ht; Hash h; + out->second.parseHashInfo(recursive, ht, h); + Path outPath = makeFixedOutputPath(recursive, ht, h, drvName); + + StringPairs::const_iterator j = drv.env.find("out"); + if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) + throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'") + % drvPath % out->second.path % outPath); + } + + else { + Derivation drvCopy(drv); + foreach (DerivationOutputs::iterator, i, drvCopy.outputs) { + i->second.path = ""; + drvCopy.env[i->first] = ""; + } + + Hash h = hashDerivationModulo(*this, drvCopy); + + foreach (DerivationOutputs::const_iterator, i, drv.outputs) { + Path outPath = makeOutputPath(i->first, h, drvName); + StringPairs::const_iterator j = drv.env.find(i->first); + if (i->second.path != outPath || j == drv.env.end() || j->second != outPath) + throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'") + % drvPath % i->second.path % outPath); + } + } +} + + +unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +{ + SQLiteStmtUse use(stmtRegisterValidPath); + stmtRegisterValidPath.bind(info.path); + stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); + stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); + if (info.deriver != "") + stmtRegisterValidPath.bind(info.deriver); + else + stmtRegisterValidPath.bind(); // null + if (info.narSize != 0) + stmtRegisterValidPath.bind64(info.narSize); + else + stmtRegisterValidPath.bind(); // null + if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) + throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); + unsigned long long id = sqlite3_last_insert_rowid(db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + Derivation drv = parseDerivation(readFile(info.path)); + + /* Verify that the output paths in the derivation are correct + (i.e., follow the scheme for computing output paths from + derivations). Note that if this throws an error, then the + DB transaction is rolled back, so the path validity + registration above is undone. */ + if (checkOutputs) checkDerivationOutputs(info.path, drv); + + foreach (DerivationOutputs::iterator, i, drv.outputs) { + SQLiteStmtUse use(stmtAddDerivationOutput); + stmtAddDerivationOutput.bind(id); + stmtAddDerivationOutput.bind(i->first); + stmtAddDerivationOutput.bind(i->second.path); + if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) + throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + } + } + + return id; +} + + +void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +{ + SQLiteStmtUse use(stmtAddReference); + stmtAddReference.bind(referrer); + stmtAddReference.bind(reference); + if (sqlite3_step(stmtAddReference) != SQLITE_DONE) + throwSQLiteError(db, "adding reference to database"); +} + + +void LocalStore::registerFailedPath(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtRegisterFailedPath); + stmtRegisterFailedPath.bind(path); + stmtRegisterFailedPath.bind(time(0)); + if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) + throwSQLiteError(db, format("registering failed path `%1%'") % path); + } end_retry_sqlite; +} + + +bool LocalStore::hasPathFailed(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtHasPathFailed); + stmtHasPathFailed.bind(path); + int res = sqlite3_step(stmtHasPathFailed); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throwSQLiteError(db, "querying whether path failed"); + return res == SQLITE_ROW; + } end_retry_sqlite; +} + + +PathSet LocalStore::queryFailedPaths() +{ + retry_sqlite { + SQLiteStmtUse use(stmtQueryFailedPaths); + + PathSet res; + int r; + while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, "error querying failed paths"); + + return res; + } end_retry_sqlite; +} + + +void LocalStore::clearFailedPaths(const PathSet & paths) +{ + retry_sqlite { + SQLiteTxn txn(db); + + foreach (PathSet::const_iterator, i, paths) { + SQLiteStmtUse use(stmtClearFailedPath); + stmtClearFailedPath.bind(*i); + if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) + throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); + } + + txn.commit(); + } end_retry_sqlite; +} + + +Hash parseHashField(const Path & path, const string & s) +{ + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'") + % s % path); + HashType ht = parseHashType(string(s, 0, colon)); + if (ht == htUnknown) + throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'") + % string(s, 0, colon) % path); + return parseHash(ht, string(s, colon + 1)); +} + + +ValidPathInfo LocalStore::queryPathInfo(const Path & path) +{ + ValidPathInfo info; + info.path = path; + + assertStorePath(path); + + retry_sqlite { + + /* Get the path info. */ + SQLiteStmtUse use1(stmtQueryPathInfo); + + stmtQueryPathInfo.bind(path); + + int r = sqlite3_step(stmtQueryPathInfo); + if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + + info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + + const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); + assert(s); + info.hash = parseHashField(path, s); + + info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + + s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + if (s) info.deriver = s; + + /* Note that narSize = NULL yields 0. */ + info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + + /* Get the references. */ + SQLiteStmtUse use2(stmtQueryReferences); + + stmtQueryReferences.bind(info.id); + + while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { + s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); + assert(s); + info.references.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting references of `%1%'") % path); + + return info; + } end_retry_sqlite; +} + + +/* Update path info in the database. Currently only updates the + narSize field. */ +void LocalStore::updatePathInfo(const ValidPathInfo & info) +{ + SQLiteStmtUse use(stmtUpdatePathInfo); + if (info.narSize != 0) + stmtUpdatePathInfo.bind64(info.narSize); + else + stmtUpdatePathInfo.bind(); // null + stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); + stmtUpdatePathInfo.bind(info.path); + if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) + throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); +} + + +unsigned long long LocalStore::queryValidPathId(const Path & path) +{ + SQLiteStmtUse use(stmtQueryPathInfo); + stmtQueryPathInfo.bind(path); + int res = sqlite3_step(stmtQueryPathInfo); + if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); + if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + throwSQLiteError(db, "querying path in database"); +} + + +bool LocalStore::isValidPath_(const Path & path) +{ + SQLiteStmtUse use(stmtQueryPathInfo); + stmtQueryPathInfo.bind(path); + int res = sqlite3_step(stmtQueryPathInfo); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throwSQLiteError(db, "querying path in database"); + return res == SQLITE_ROW; +} + + +bool LocalStore::isValidPath(const Path & path) +{ + retry_sqlite { + return isValidPath_(path); + } end_retry_sqlite; +} + + +PathSet LocalStore::queryValidPaths(const PathSet & paths) +{ + retry_sqlite { + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath_(*i)) res.insert(*i); + return res; + } end_retry_sqlite; +} + + +PathSet LocalStore::queryAllValidPaths() +{ + retry_sqlite { + SQLiteStmt stmt; + stmt.create(db, "select path from ValidPaths"); + + PathSet res; + int r; + while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmt, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, "error getting valid paths"); + + return res; + } end_retry_sqlite; +} + + +void LocalStore::queryReferences(const Path & path, + PathSet & references) +{ + ValidPathInfo info = queryPathInfo(path); + references.insert(info.references.begin(), info.references.end()); +} + + +void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +{ + SQLiteStmtUse use(stmtQueryReferrers); + + stmtQueryReferrers.bind(path); + + int r; + while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); + assert(s); + referrers.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting references of `%1%'") % path); +} + + +void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +{ + assertStorePath(path); + retry_sqlite { + queryReferrers_(path, referrers); + } end_retry_sqlite; +} + + +Path LocalStore::queryDeriver(const Path & path) +{ + return queryPathInfo(path).deriver; +} + + +PathSet LocalStore::queryValidDerivers(const Path & path) +{ + assertStorePath(path); + + retry_sqlite { + SQLiteStmtUse use(stmtQueryValidDerivers); + stmtQueryValidDerivers.bind(path); + + PathSet derivers; + int r; + while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); + assert(s); + derivers.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); + + return derivers; + } end_retry_sqlite; +} + + +PathSet LocalStore::queryDerivationOutputs(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtQueryDerivationOutputs); + stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + + PathSet outputs; + int r; + while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); + assert(s); + outputs.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting outputs of `%1%'") % path); + + return outputs; + } end_retry_sqlite; +} + + +StringSet LocalStore::queryDerivationOutputNames(const Path & path) +{ + retry_sqlite { + SQLiteStmtUse use(stmtQueryDerivationOutputs); + stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + + StringSet outputNames; + int r; + while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); + assert(s); + outputNames.insert(s); + } + + if (r != SQLITE_DONE) + throwSQLiteError(db, format("error getting output names of `%1%'") % path); + + return outputNames; + } end_retry_sqlite; +} + + +Path LocalStore::queryPathFromHashPart(const string & hashPart) +{ + if (hashPart.size() != 32) throw Error("invalid hash part"); + + Path prefix = settings.nixStore + "/" + hashPart; + + retry_sqlite { + SQLiteStmtUse use(stmtQueryPathFromHashPart); + stmtQueryPathFromHashPart.bind(prefix); + + int res = sqlite3_step(stmtQueryPathFromHashPart); + if (res == SQLITE_DONE) return ""; + if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + + const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; + } end_retry_sqlite; +} + + +void LocalStore::setSubstituterEnv() +{ + if (didSetSubstituterEnv) return; + + /* Pass configuration options (including those overridden with + --option) to substituters. */ + setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); + + didSetSubstituterEnv = true; +} + + +void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) +{ + if (run.disabled || run.pid != -1) return; + + debug(format("starting substituter program `%1%'") % substituter); + + Pipe toPipe, fromPipe, errorPipe; + + toPipe.create(); + fromPipe.create(); + errorPipe.create(); + + setSubstituterEnv(); + + run.pid = maybeVfork(); + + switch (run.pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + restoreAffinity(); + if (dup2(toPipe.readSide, STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("dupping stderr"); + execl(substituter.c_str(), substituter.c_str(), "--query", NULL); + throw SysError(format("executing `%1%'") % substituter); + } catch (std::exception & e) { + std::cerr << "error: " << e.what() << std::endl; + } + _exit(1); + } + + /* Parent. */ + + run.program = baseNameOf(substituter); + run.to = toPipe.writeSide.borrow(); + run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); + run.error = errorPipe.readSide.borrow(); + + toPipe.readSide.close(); + fromPipe.writeSide.close(); + errorPipe.writeSide.close(); + + /* The substituter may exit right away if it's disabled in any way + (e.g. copy-from-other-stores.pl will exit if no other stores + are configured). */ + try { + getLineFromSubstituter(run); + } catch (EndOfFile & e) { + run.to.close(); + run.from.close(); + run.error.close(); + run.disabled = true; + if (run.pid.wait(true) != 0) throw; + } +} + + +/* Read a line from the substituter's stdout, while also processing + its stderr. */ +string LocalStore::getLineFromSubstituter(RunningSubstituter & run) +{ + string res, err; + + /* We might have stdout data left over from the last time. */ + if (run.fromBuf.hasData()) goto haveData; + + while (1) { + checkInterrupt(); + + fd_set fds; + FD_ZERO(&fds); + FD_SET(run.from, &fds); + FD_SET(run.error, &fds); + + /* Wait for data to appear on the substituter's stdout or + stderr. */ + if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { + if (errno == EINTR) continue; + throw SysError("waiting for input from the substituter"); + } + + /* Completely drain stderr before dealing with stdout. */ + if (FD_ISSET(run.error, &fds)) { + char buf[4096]; + ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); + if (n == -1) { + if (errno == EINTR) continue; + throw SysError("reading from substituter's stderr"); + } + if (n == 0) throw EndOfFile(format("substituter `%1%' died unexpectedly") % run.program); + err.append(buf, n); + string::size_type p; + while ((p = err.find('\n')) != string::npos) { + printMsg(lvlError, run.program + ": " + string(err, 0, p)); + err = string(err, p + 1); + } + } + + /* Read from stdout until we get a newline or the buffer is empty. */ + else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { + haveData: + do { + unsigned char c; + run.fromBuf(&c, 1); + if (c == '\n') { + if (!err.empty()) printMsg(lvlError, run.program + ": " + err); + return res; + } + res += c; + } while (run.fromBuf.hasData()); + } + } +} + + +template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) +{ + string s = getLineFromSubstituter(run); + T res; + if (!string2Int(s, res)) throw Error("integer expected from stream"); + return res; +} + + +PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) +{ + PathSet res; + foreach (Paths::iterator, i, settings.substituters) { + if (res.size() == paths.size()) break; + RunningSubstituter & run(runningSubstituters[*i]); + startSubstituter(*i, run); + if (run.disabled) continue; + string s = "have "; + foreach (PathSet::const_iterator, j, paths) + if (res.find(*j) == res.end()) { s += *j; s += " "; } + writeLine(run.to, s); + while (true) { + /* FIXME: we only read stderr when an error occurs, so + substituters should only write (short) messages to + stderr when they fail. I.e. they shouldn't write debug + output. */ + Path path = getLineFromSubstituter(run); + if (path == "") break; + res.insert(path); + } + } + return res; +} + + +void LocalStore::querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos) +{ + RunningSubstituter & run(runningSubstituters[substituter]); + startSubstituter(substituter, run); + if (run.disabled) return; + + string s = "info "; + foreach (PathSet::const_iterator, i, paths) + if (infos.find(*i) == infos.end()) { s += *i; s += " "; } + writeLine(run.to, s); + + while (true) { + Path path = getLineFromSubstituter(run); + if (path == "") break; + if (paths.find(path) == paths.end()) + throw Error(format("got unexpected path `%1%' from substituter") % path); + paths.erase(path); + SubstitutablePathInfo & info(infos[path]); + info.deriver = getLineFromSubstituter(run); + if (info.deriver != "") assertStorePath(info.deriver); + int nrRefs = getIntLineFromSubstituter<int>(run); + while (nrRefs--) { + Path p = getLineFromSubstituter(run); + assertStorePath(p); + info.references.insert(p); + } + info.downloadSize = getIntLineFromSubstituter<long long>(run); + info.narSize = getIntLineFromSubstituter<long long>(run); + } +} + + +void LocalStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + PathSet todo = paths; + foreach (Paths::iterator, i, settings.substituters) { + if (todo.empty()) break; + querySubstitutablePathInfos(*i, todo, infos); + } +} + + +Hash LocalStore::queryPathHash(const Path & path) +{ + return queryPathInfo(path).hash; +} + + +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + + +void LocalStore::registerValidPaths(const ValidPathInfos & infos) +{ + /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. + * So some may want to fsync them before registering the validity, at the + * expense of some speed of the path registering operation. */ + if (settings.syncBeforeRegistering) sync(); + + retry_sqlite { + SQLiteTxn txn(db); + PathSet paths; + + foreach (ValidPathInfos::const_iterator, i, infos) { + assert(i->hash.type == htSHA256); + if (isValidPath_(i->path)) + updatePathInfo(*i); + else + addValidPath(*i); + paths.insert(i->path); + } + + foreach (ValidPathInfos::const_iterator, i, infos) { + unsigned long long referrer = queryValidPathId(i->path); + foreach (PathSet::iterator, j, i->references) + addReference(referrer, queryValidPathId(*j)); + } + + /* Do a topological sort of the paths. This will throw an + error if a cycle is detected and roll back the + transaction. Cycles can only occur when a derivation + has multiple outputs. */ + topoSortPaths(*this, paths); + + txn.commit(); + } end_retry_sqlite; +} + + +/* Invalidate a path. The caller is responsible for checking that + there are no referrers. */ +void LocalStore::invalidatePath(const Path & path) +{ + debug(format("invalidating path `%1%'") % path); + + drvHashes.erase(path); + + SQLiteStmtUse use(stmtInvalidatePath); + + stmtInvalidatePath.bind(path); + + if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) + throwSQLiteError(db, format("invalidating path `%1%' in database") % path); + + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ +} + + +Path LocalStore::addToStoreFromDump(const string & dump, const string & name, + bool recursive, HashType hashAlgo, bool repair) +{ + Hash h = hashString(hashAlgo, dump); + + Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + + if (repair || !isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + if (recursive) { + StringSource source(dump); + restorePath(dstPath, source); + } else + writeFile(dstPath, dump); + + canonicalisePathMetaData(dstPath, -1); + + /* Register the SHA-256 hash of the NAR serialisation of + the path in the database. We may just have computed it + above (if called with recursive == true and hashAlgo == + sha256); otherwise, compute it here. */ + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +Path LocalStore::addToStore(const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + Path srcPath(absPath(_srcPath)); + debug(format("adding `%1%' to the store") % srcPath); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) + dumpPath(srcPath, sink, filter); + else + sink.s = readFile(srcPath); + + return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo, repair); +} + + +Path LocalStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + Path dstPath = computeStorePathForText(name, s, references); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + + if (repair || !isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + writeFile(dstPath, s); + + canonicalisePathMetaData(dstPath, -1); + + HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +struct HashAndWriteSink : Sink +{ + Sink & writeSink; + HashSink hashSink; + HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) + { + } + virtual void operator () (const unsigned char * data, size_t len) + { + writeSink(data, len); + hashSink(data, len); + } + Hash currentHash() + { + return hashSink.currentHash().first; + } +}; + + +#define EXPORT_MAGIC 0x4558494e + + +static void checkSecrecy(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0) + throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path); +} + + +void LocalStore::exportPath(const Path & path, bool sign, + Sink & sink) +{ + assertStorePath(path); + + addTempRoot(path); + if (!isValidPath(path)) + throw Error(format("path `%1%' is not valid") % path); + + HashAndWriteSink hashAndWriteSink(sink); + + dumpPath(path, hashAndWriteSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashAndWriteSink.currentHash(); + Hash storedHash = queryPathHash(path); + if (hash != storedHash && storedHash != Hash(storedHash.type)) + throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path + % printHash(storedHash) % printHash(hash)); + + writeInt(EXPORT_MAGIC, hashAndWriteSink); + + writeString(path, hashAndWriteSink); + + PathSet references; + queryReferences(path, references); + writeStrings(references, hashAndWriteSink); + + Path deriver = queryDeriver(path); + writeString(deriver, hashAndWriteSink); + + if (sign) { + Hash hash = hashAndWriteSink.currentHash(); + + writeInt(1, hashAndWriteSink); + + Path tmpDir = createTempDir(); + AutoDelete delTmp(tmpDir); + Path hashFile = tmpDir + "/hash"; + writeFile(hashFile, printHash(hash)); + + Path secretKey = settings.nixConfDir + "/signing-key.sec"; + checkSecrecy(secretKey); + + Strings args; + args.push_back("rsautl"); + args.push_back("-sign"); + args.push_back("-inkey"); + args.push_back(secretKey); + args.push_back("-in"); + args.push_back(hashFile); + string signature = runProgram(OPENSSL_PATH, true, args); + + writeString(signature, hashAndWriteSink); + + } else + writeInt(0, hashAndWriteSink); +} + + +struct HashAndReadSource : Source +{ + Source & readSource; + HashSink hashSink; + bool hashing; + HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256) + { + hashing = true; + } + size_t read(unsigned char * data, size_t len) + { + size_t n = readSource.read(data, len); + if (hashing) hashSink(data, n); + return n; + } +}; + + +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() +{ + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(settings.nixStore); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + + +Path LocalStore::importPath(bool requireSignature, Source & source) +{ + HashAndReadSource hashAndReadSource(source); + + /* We don't yet know what store path this archive contains (the + store path follows the archive data proper), and besides, we + don't know yet whether the signature is valid. */ + Path tmpDir = createTempDirInStore(); + AutoDelete delTmp(tmpDir); + Path unpacked = tmpDir + "/unpacked"; + + restorePath(unpacked, hashAndReadSource); + + unsigned int magic = readInt(hashAndReadSource); + if (magic != EXPORT_MAGIC) + throw Error("Nix archive cannot be imported; wrong format"); + + Path dstPath = readStorePath(hashAndReadSource); + + printMsg(lvlInfo, format("importing path `%1%'") % dstPath); + + PathSet references = readStorePaths<PathSet>(hashAndReadSource); + + Path deriver = readString(hashAndReadSource); + if (deriver != "") assertStorePath(deriver); + + Hash hash = hashAndReadSource.hashSink.finish().first; + hashAndReadSource.hashing = false; + + bool haveSignature = readInt(hashAndReadSource) == 1; + + if (requireSignature && !haveSignature) + throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); + + if (haveSignature) { + string signature = readString(hashAndReadSource); + + if (requireSignature) { + Path sigFile = tmpDir + "/sig"; + writeFile(sigFile, signature); + + Strings args; + args.push_back("rsautl"); + args.push_back("-verify"); + args.push_back("-inkey"); + args.push_back(settings.nixConfDir + "/signing-key.pub"); + args.push_back("-pubin"); + args.push_back("-in"); + args.push_back(sigFile); + string hash2 = runProgram(OPENSSL_PATH, true, args); + + /* Note: runProgram() throws an exception if the signature + is invalid. */ + + if (printHash(hash) != hash2) + throw Error( + "signed hash doesn't match actual contents of imported " + "archive; archive could be corrupt, or someone is trying " + "to import a Trojan horse"); + } + } + + /* Do the actual import. */ + + /* !!! way too much code duplication with addTextToStore() etc. */ + addTempRoot(dstPath); + + if (!isValidPath(dstPath)) { + + PathLocks outputLock; + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); + if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end()) + outputLock.lockPaths(singleton<PathSet, Path>(dstPath)); + + if (!isValidPath(dstPath)) { + + if (pathExists(dstPath)) deletePath(dstPath); + + if (rename(unpacked.c_str(), dstPath.c_str()) == -1) + throw SysError(format("cannot move `%1%' to `%2%'") + % unpacked % dstPath); + + canonicalisePathMetaData(dstPath, -1); + + /* !!! if we were clever, we could prevent the hashPath() + here. */ + HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +Paths LocalStore::importPaths(bool requireSignature, Source & source) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); + res.push_back(importPath(requireSignature, source)); + } + return res; +} + + +void LocalStore::invalidatePathChecked(const Path & path) +{ + assertStorePath(path); + + retry_sqlite { + SQLiteTxn txn(db); + + if (isValidPath_(path)) { + PathSet referrers; queryReferrers_(path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%") + % path % showPaths(referrers)); + invalidatePath(path); + } + + txn.commit(); + } end_retry_sqlite; +} + + +bool LocalStore::verifyStore(bool checkContents, bool repair) +{ + printMsg(lvlError, format("reading the Nix store...")); + + bool errors = false; + + /* Acquire the global GC lock to prevent a garbage collection. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + Paths entries = readDirectory(settings.nixStore); + PathSet store(entries.begin(), entries.end()); + + /* Check whether all valid paths actually exist. */ + printMsg(lvlInfo, "checking path existence..."); + + PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + + foreach (PathSet::iterator, i, validPaths2) + verifyPath(*i, store, done, validPaths, repair, errors); + + /* Release the GC lock so that checking content hashes (which can + take ages) doesn't block the GC or builds. */ + fdGCLock.close(); + + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + printMsg(lvlInfo, "checking hashes..."); + + Hash nullHash(htSHA256); + + foreach (PathSet::iterator, i, validPaths) { + try { + ValidPathInfo info = queryPathInfo(*i); + + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); + HashResult current = hashPath(info.hash.type, *i); + + if (info.hash != nullHash && info.hash != current.first) { + printMsg(lvlError, format("path `%1%' was modified! " + "expected hash `%2%', got `%3%'") + % *i % printHash(info.hash) % printHash(current.first)); + if (repair) repairPath(*i); else errors = true; + } else { + + bool update = false; + + /* Fill in missing hashes. */ + if (info.hash == nullHash) { + printMsg(lvlError, format("fixing missing hash on `%1%'") % *i); + info.hash = current.first; + update = true; + } + + /* Fill in missing narSize fields (from old stores). */ + if (info.narSize == 0) { + printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); + info.narSize = current.second; + update = true; + } + + if (update) updatePathInfo(info); + + } + + } catch (Error & e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(*i)) + printMsg(lvlError, format("error: %1%") % e.msg()); + else + printMsg(lvlError, format("warning: %1%") % e.msg()); + errors = true; + } + } + } + + return errors; +} + + +void LocalStore::verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths, bool repair, bool & errors) +{ + checkInterrupt(); + + if (done.find(path) != done.end()) return; + done.insert(path); + + if (!isStorePath(path)) { + printMsg(lvlError, format("path `%1%' is not in the Nix store") % path); + invalidatePath(path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; queryReferrers(path, referrers); + foreach (PathSet::iterator, i, referrers) + if (*i != path) { + verifyPath(*i, store, done, validPaths, repair, errors); + if (validPaths.find(*i) != validPaths.end()) + canInvalidate = false; + } + + if (canInvalidate) { + printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path); + invalidatePath(path); + } else { + printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path); + if (repair) + try { + repairPath(path); + } catch (Error & e) { + printMsg(lvlError, format("warning: %1%") % e.msg()); + errors = true; + } + else errors = true; + } + + return; + } + + validPaths.insert(path); +} + + +bool LocalStore::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path `%1%'...") % path); + ValidPathInfo info = queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info.hash.type, path); + Hash nullHash(htSHA256); + res = info.hash == nullHash || info.hash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path); + return res; +} + + +void LocalStore::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + +/* Functions for upgrading from the pre-SQLite database. */ + +PathSet LocalStore::queryValidPathsOld() +{ + PathSet paths; + Strings entries = readDirectory(settings.nixDBPath + "/info"); + foreach (Strings::iterator, i, entries) + if (i->at(0) != '.') paths.insert(settings.nixStore + "/" + *i); + return paths; +} + + +ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) +{ + ValidPathInfo res; + res.path = path; + + /* Read the info file. */ + string baseName = baseNameOf(path); + Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); + if (!pathExists(infoFile)) + throw Error(format("path `%1%' is not valid") % path); + string info = readFile(infoFile); + + /* Parse it. */ + Strings lines = tokenizeString<Strings>(info, "\n"); + + foreach (Strings::iterator, i, lines) { + string::size_type p = i->find(':'); + if (p == string::npos) + throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); + string name(*i, 0, p); + string value(*i, p + 2); + if (name == "References") { + Strings refs = tokenizeString<Strings>(value, " "); + res.references = PathSet(refs.begin(), refs.end()); + } else if (name == "Deriver") { + res.deriver = value; + } else if (name == "Hash") { + res.hash = parseHashField(path, value); + } else if (name == "Registered-At") { + int n = 0; + string2Int(value, n); + res.registrationTime = n; + } + } + + return res; +} + + +/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ +void LocalStore::upgradeStore6() +{ + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + openDB(true); + + PathSet validPaths = queryValidPathsOld(); + + SQLiteTxn txn(db); + + foreach (PathSet::iterator, i, validPaths) { + addValidPath(queryPathInfoOld(*i), false); + std::cerr << "."; + } + + std::cerr << "|"; + + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfoOld(*i); + unsigned long long referrer = queryValidPathId(*i); + foreach (PathSet::iterator, j, info.references) + addReference(referrer, queryValidPathId(*j)); + std::cerr << "."; + } + + std::cerr << "\n"; + + txn.commit(); +} + + +#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) + +static void makeMutable(const Path & path) +{ + checkInterrupt(); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + makeMutable(path + "/" + *i); + } + + /* The O_NOFOLLOW is important to prevent us from changing the + mutable bit on the target of a symlink (which would be a + security hole). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW); + if (fd == -1) { + if (errno == ELOOP) return; // it's a symlink + throw SysError(format("opening file `%1%'") % path); + } + + unsigned int flags = 0, old; + + /* Silently ignore errors getting/setting the immutable flag so + that we work correctly on filesystems that don't support it. */ + if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; + old = flags; + flags &= ~FS_IMMUTABLE_FL; + if (old == flags) return; + if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; +} + +/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ +void LocalStore::upgradeStore7() +{ + if (getuid() != 0) return; + printMsg(lvlError, "removing immutable bits from the Nix store (this may take a while)..."); + makeMutable(settings.nixStore); +} + +#else + +void LocalStore::upgradeStore7() +{ +} + +#endif + + +void LocalStore::vacuumDB() +{ + if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "vacuuming SQLite database"); +} + + +} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh new file mode 100644 index 000000000000..1ace7cec996b --- /dev/null +++ b/src/libstore/local-store.hh @@ -0,0 +1,333 @@ +#pragma once + +#include <string> + +#include "store-api.hh" +#include "util.hh" +#include "pathlocks.hh" + + +class sqlite3; +class sqlite3_stmt; + + +namespace nix { + + +/* Nix store and database schema version. Version 1 (or 0) was Nix <= + 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + Nix 1.0. Version 7 is Nix 1.3. */ +const int nixSchemaVersion = 7; + + +extern string drvsLogDir; + + +struct Derivation; + + +struct OptimiseStats +{ + unsigned long totalFiles; + unsigned long sameContents; + unsigned long filesLinked; + unsigned long long bytesFreed; + unsigned long long blocksFreed; + OptimiseStats() + { + totalFiles = sameContents = filesLinked = 0; + bytesFreed = blocksFreed = 0; + } +}; + + +struct RunningSubstituter +{ + Path program; + Pid pid; + AutoCloseFD to, from, error; + FdSource fromBuf; + bool disabled; + RunningSubstituter() : disabled(false) { }; +}; + + +/* Wrapper object to close the SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + + +/* Wrapper object to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db; + sqlite3_stmt * stmt; + unsigned int curArg; + SQLiteStmt() { stmt = 0; } + void create(sqlite3 * db, const string & s); + void reset(); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + void bind(const string & value); + void bind(int value); + void bind64(long long value); + void bind(); +}; + + +class LocalStore : public StoreAPI +{ +private: + typedef std::map<Path, RunningSubstituter> RunningSubstituters; + RunningSubstituters runningSubstituters; + + Path linksDir; + +public: + + /* Initialise the local store, upgrading the schema if + necessary. */ + LocalStore(bool reserveSpace = true); + + ~LocalStore(); + + /* Implementations of abstract store API methods. */ + + bool isValidPath(const Path & path); + + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); + + ValidPathInfo queryPathInfo(const Path & path); + + Hash queryPathHash(const Path & path); + + void queryReferences(const Path & path, PathSet & references); + + void queryReferrers(const Path & path, PathSet & referrers); + + Path queryDeriver(const Path & path); + + PathSet queryValidDerivers(const Path & path); + + PathSet queryDerivationOutputs(const Path & path); + + StringSet queryDerivationOutputNames(const Path & path); + + Path queryPathFromHashPart(const string & hashPart); + + PathSet querySubstitutablePaths(const PathSet & paths); + + void querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos); + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); + + Path addToStore(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false); + + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ + Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false); + + void exportPath(const Path & path, bool sign, + Sink & sink); + + Paths importPaths(bool requireSignature, Source & source); + + void buildPaths(const PathSet & paths, bool repair = false); + + void ensurePath(const Path & path); + + void addTempRoot(const Path & path); + + void addIndirectRoot(const Path & path); + + void syncWithGC(); + + Roots findRoots(); + + void collectGarbage(const GCOptions & options, GCResults & results); + + /* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ + void optimiseStore(OptimiseStats & stats); + + /* Optimise a single store path. */ + void optimisePath(const Path & path); + + /* Check the integrity of the Nix store. Returns true if errors + remain. */ + bool verifyStore(bool checkContents, bool repair); + + /* Register the validity of a path, i.e., that `path' exists, that + the paths referenced by it exists, and in the case of an output + path of a derivation, that it has been produced by a successful + execution of the derivation (or something equivalent). Also + register the hash of the file system contents of the path. The + hash must be a SHA-256 hash. */ + void registerValidPath(const ValidPathInfo & info); + + void registerValidPaths(const ValidPathInfos & infos); + + /* Register that the build of a derivation with output `path' has + failed. */ + void registerFailedPath(const Path & path); + + /* Query whether `path' previously failed to build. */ + bool hasPathFailed(const Path & path); + + PathSet queryFailedPaths(); + + void clearFailedPaths(const PathSet & paths); + + void vacuumDB(); + + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + void repairPath(const Path & path); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); + + void setSubstituterEnv(); + +private: + + Path schemaPath; + + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; + + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtRegisterFailedPath; + SQLiteStmt stmtHasPathFailed; + SQLiteStmt stmtQueryFailedPaths; + SQLiteStmt stmtClearFailedPath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + + bool didSetSubstituterEnv; + + int getSchema(); + + void openDB(bool create); + + void makeStoreWritable(); + + unsigned long long queryValidPathId(const Path & path); + + unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + + void addReference(unsigned long long referrer, unsigned long long reference); + + void appendReferrer(const Path & from, const Path & to, bool lock); + + void rewriteReferrers(const Path & path, bool purge, PathSet referrers); + + void invalidatePath(const Path & path); + + /* Delete a path from the Nix store. */ + void invalidatePathChecked(const Path & path); + + void verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths, bool repair, bool & errors); + + void updatePathInfo(const ValidPathInfo & info); + + void upgradeStore6(); + void upgradeStore7(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path & path); + + struct GCState; + + void deleteGarbage(GCState & state, const Path & path); + + void tryToDelete(GCState & state, const Path & path); + + bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + + void deletePathRecursive(GCState & state, const Path & path); + + bool isActiveTempFile(const GCState & state, + const Path & path, const string & suffix); + + int openGCLock(LockType lockType); + + void removeUnusedLinks(const GCState & state); + + void startSubstituter(const Path & substituter, + RunningSubstituter & runningSubstituter); + + string getLineFromSubstituter(RunningSubstituter & run); + + template<class T> T getIntLineFromSubstituter(RunningSubstituter & run); + + Path createTempDirInStore(); + + Path importPath(bool requireSignature, Source & source); + + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); + + // Internal versions that are not wrapped in retry_sqlite. + bool isValidPath_(const Path & path); + void queryReferrers_(const Path & path, PathSet & referrers); +}; + + +typedef std::pair<dev_t, ino_t> Inode; +typedef set<Inode> InodesSeen; + + +/* "Fix", or canonicalise, the meta-data of the files in a store path + after it has been built. In particular: + - the last modification date on each file is set to 1 (i.e., + 00:00:01 1/1/1970 UTC) + - the permissions are set of 444 or 555 (i.e., read-only with or + without execute permission; setuid bits etc. are cleared) + - the owner and group are set to the Nix user and group, if we're + running as root. */ +void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); +void canonicalisePathMetaData(const Path & path, uid_t fromUid); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/libstore/local.mk b/src/libstore/local.mk new file mode 100644 index 000000000000..2dddce740dcc --- /dev/null +++ b/src/libstore/local.mk @@ -0,0 +1,28 @@ +libraries += libstore + +libstore_NAME = libnixstore + +libstore_DIR := $(d) + +libstore_SOURCES := $(wildcard $(d)/*.cc) + +libstore_LIBS = libutil libformat + +libstore_LDFLAGS = -lsqlite3 -lbz2 + +libstore_CXXFLAGS = \ + -DNIX_STORE_DIR=\"$(storedir)\" \ + -DNIX_DATA_DIR=\"$(datadir)\" \ + -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ + -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ + -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ + -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ + -DNIX_BIN_DIR=\"$(bindir)\" \ + -DPACKAGE_VERSION=\"$(PACKAGE_VERSION)\" + +$(d)/local-store.cc: $(d)/schema.sql.hh + +%.sql.hh: %.sql + $(trace-gen) sed -e 's/"/\\"/g' -e 's/\(.*\)/"\1\\n"/' < $< > $@ || (rm $@ && exit 1) + +clean-files += $(d)/schema.sql.hh diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc new file mode 100644 index 000000000000..1bf3f9378245 --- /dev/null +++ b/src/libstore/misc.cc @@ -0,0 +1,220 @@ +#include "misc.hh" +#include "store-api.hh" +#include "local-store.hh" +#include "globals.hh" + + +namespace nix { + + +Derivation derivationFromPath(StoreAPI & store, const Path & drvPath) +{ + assertStorePath(drvPath); + store.ensurePath(drvPath); + return parseDerivation(readFile(drvPath)); +} + + +void computeFSClosure(StoreAPI & store, const Path & path, + PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers) +{ + if (paths.find(path) != paths.end()) return; + paths.insert(path); + + PathSet edges; + + if (flipDirection) { + store.queryReferrers(path, edges); + + if (includeOutputs) { + PathSet derivers = store.queryValidDerivers(path); + foreach (PathSet::iterator, i, derivers) + edges.insert(*i); + } + + if (includeDerivers && isDerivation(path)) { + PathSet outputs = store.queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (store.isValidPath(*i) && store.queryDeriver(*i) == path) + edges.insert(*i); + } + + } else { + store.queryReferences(path, edges); + + if (includeOutputs && isDerivation(path)) { + PathSet outputs = store.queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (store.isValidPath(*i)) edges.insert(*i); + } + + if (includeDerivers) { + Path deriver = store.queryDeriver(path); + if (store.isValidPath(deriver)) edges.insert(deriver); + } + } + + foreach (PathSet::iterator, i, edges) + computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers); +} + + +Path findOutput(const Derivation & drv, string id) +{ + foreach (DerivationOutputs::const_iterator, i, drv.outputs) + if (i->first == id) return i->second.path; + throw Error(format("derivation has no output `%1%'") % id); +} + + +void queryMissing(StoreAPI & store, const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) +{ + downloadSize = narSize = 0; + + PathSet todo(targets.begin(), targets.end()), done; + + /* Getting substitute info has high latency when using the binary + cache substituter. Thus it's essential to do substitute + queries in parallel as much as possible. To accomplish this + we do the following: + + - For all paths still to be processed (‘todo’), we add all + paths for which we need info to the set ‘query’. For an + unbuilt derivation this is the output paths; otherwise, it's + the path itself. + + - We get info about all paths in ‘query’ in parallel. + + - We process the results and add new items to ‘todo’ if + necessary. E.g. if a path is substitutable, then we need to + get info on its references. + + - Repeat until ‘todo’ is empty. + */ + + while (!todo.empty()) { + + PathSet query, todoDrv, todoNonDrv; + + foreach (PathSet::iterator, i, todo) { + if (done.find(*i) != done.end()) continue; + done.insert(*i); + + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + + if (isDerivation(i2.first)) { + if (!store.isValidPath(i2.first)) { + // FIXME: we could try to substitute p. + unknown.insert(*i); + continue; + } + Derivation drv = derivationFromPath(store, i2.first); + + PathSet invalid; + foreach (DerivationOutputs::iterator, j, drv.outputs) + if (wantOutput(j->first, i2.second) + && !store.isValidPath(j->second.path)) + invalid.insert(j->second.path); + if (invalid.empty()) continue; + + todoDrv.insert(*i); + if (settings.useSubstitutes && !willBuildLocally(drv)) + query.insert(invalid.begin(), invalid.end()); + } + + else { + if (store.isValidPath(*i)) continue; + query.insert(*i); + todoNonDrv.insert(*i); + } + } + + todo.clear(); + + SubstitutablePathInfos infos; + store.querySubstitutablePathInfos(query, infos); + + foreach (PathSet::iterator, i, todoDrv) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + + // FIXME: cache this + Derivation drv = derivationFromPath(store, i2.first); + + PathSet outputs; + bool mustBuild = false; + if (settings.useSubstitutes && !willBuildLocally(drv)) { + foreach (DerivationOutputs::iterator, j, drv.outputs) { + if (!wantOutput(j->first, i2.second)) continue; + if (!store.isValidPath(j->second.path)) { + if (infos.find(j->second.path) == infos.end()) + mustBuild = true; + else + outputs.insert(j->second.path); + } + } + } else + mustBuild = true; + + if (mustBuild) { + willBuild.insert(i2.first); + todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + foreach (DerivationInputs::iterator, j, drv.inputDrvs) + todo.insert(makeDrvPathWithOutputs(j->first, j->second)); + } else + todoNonDrv.insert(outputs.begin(), outputs.end()); + } + + foreach (PathSet::iterator, i, todoNonDrv) { + done.insert(*i); + SubstitutablePathInfos::iterator info = infos.find(*i); + if (info != infos.end()) { + willSubstitute.insert(*i); + downloadSize += info->second.downloadSize; + narSize += info->second.narSize; + todo.insert(info->second.references.begin(), info->second.references.end()); + } else + unknown.insert(*i); + } + } +} + + +static void dfsVisit(StoreAPI & store, const PathSet & paths, + const Path & path, PathSet & visited, Paths & sorted, + PathSet & parents) +{ + if (parents.find(path) != parents.end()) + throw BuildError(format("cycle detected in the references of `%1%'") % path); + + if (visited.find(path) != visited.end()) return; + visited.insert(path); + parents.insert(path); + + PathSet references; + if (store.isValidPath(path)) + store.queryReferences(path, references); + + foreach (PathSet::iterator, i, references) + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (*i != path && paths.find(*i) != paths.end()) + dfsVisit(store, paths, *i, visited, sorted, parents); + + sorted.push_front(path); + parents.erase(path); +} + + +Paths topoSortPaths(StoreAPI & store, const PathSet & paths) +{ + Paths sorted; + PathSet visited, parents; + foreach (PathSet::const_iterator, i, paths) + dfsVisit(store, paths, *i, visited, sorted, parents); + return sorted; +} + + +} diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh new file mode 100644 index 000000000000..144cb7f457cd --- /dev/null +++ b/src/libstore/misc.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "derivations.hh" + + +namespace nix { + + +/* Read a derivation, after ensuring its existence through + ensurePath(). */ +Derivation derivationFromPath(StoreAPI & store, const Path & drvPath); + +/* Place in `paths' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly or + indirectly reached from it. `paths' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ +void computeFSClosure(StoreAPI & store, const Path & path, + PathSet & paths, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + +/* Return the path corresponding to the output identifier `id' in the + given derivation. */ +Path findOutput(const Derivation & drv, string id); + +/* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths that + will be substituted. */ +void queryMissing(StoreAPI & store, const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize); + +bool willBuildLocally(const Derivation & drv); + + +} diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc new file mode 100644 index 000000000000..d833f3aa05b5 --- /dev/null +++ b/src/libstore/optimise-store.cc @@ -0,0 +1,180 @@ +#include "config.h" + +#include "util.hh" +#include "local-store.hh" +#include "globals.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <errno.h> +#include <stdio.h> + + +namespace nix { + + +static void makeWritable(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("changing writability of `%1%'") % path); +} + + +struct MakeReadOnly +{ + Path path; + MakeReadOnly(const Path & path) : path(path) { } + ~MakeReadOnly() + { + try { + /* This will make the path read-only. */ + if (path != "") canonicaliseTimestampAndPermissions(path); + } catch (...) { + ignoreException(); + } + } +}; + + +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + optimisePath_(stats, path + "/" + *i); + return; + } + + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + && !S_ISLNK(st.st_mode) +#endif + ) return; + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. FIXME: check the modification time. */ + if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { + printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); + return; + } + + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + printHash32(hash); + + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + if (link(path.c_str(), linkPath.c_str()) == 0) return; + if (errno != EEXIST) + throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); + /* Fall through if another process created ‘linkPath’ before + we did. */ + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path `%1%'") % linkPath); + + stats.sameContents++; + if (st.st_ino == stLink.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); + return; + } + + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % settings.nixStore % getpid() % rand()).str(); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + if (st.st_size) + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (unlink(tempLink.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + if (st.st_size) + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; +} + + +void LocalStore::optimiseStore(OptimiseStats & stats) +{ + PathSet paths = queryAllValidPaths(); + + foreach (PathSet::iterator, i, paths) { + addTempRoot(*i); + if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ + startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + OptimiseStats stats; + if (settings.autoOptimiseStore) optimisePath_(stats, path); +} + + +} diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc new file mode 100644 index 000000000000..b858ed238de0 --- /dev/null +++ b/src/libstore/pathlocks.cc @@ -0,0 +1,199 @@ +#include "pathlocks.hh" +#include "util.hh" + +#include <cerrno> +#include <cstdlib> + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + + +namespace nix { + + +int openLockFile(const Path & path, bool create) +{ + AutoCloseFD fd; + + fd = open(path.c_str(), O_RDWR | (create ? O_CREAT : 0), 0600); + if (fd == -1 && (create || errno != ENOENT)) + throw SysError(format("opening lock file `%1%'") % path); + + closeOnExec(fd); + + return fd.borrow(); +} + + +void deleteLockFile(const Path & path, int fd) +{ + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(fd, (const unsigned char *) "d", 1); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ +} + + +bool lockFile(int fd, LockType lockType, bool wait) +{ + struct flock lock; + if (lockType == ltRead) lock.l_type = F_RDLCK; + else if (lockType == ltWrite) lock.l_type = F_WRLCK; + else if (lockType == ltNone) lock.l_type = F_UNLCK; + else abort(); + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; /* entire file */ + + if (wait) { + while (fcntl(fd, F_SETLKW, &lock) != 0) { + checkInterrupt(); + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } + } else { + while (fcntl(fd, F_SETLK, &lock) != 0) { + checkInterrupt(); + if (errno == EACCES || errno == EAGAIN) return false; + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } + } + + return true; +} + + +/* This enables us to check whether are not already holding a lock on + a file ourselves. POSIX locks (fcntl) suck in this respect: if we + close a descriptor, the previous lock will be closed as well. And + there is no way to query whether we already have a lock (F_GETLK + only works on locks held by other processes). */ +static StringSet lockedPaths; /* !!! not thread-safe */ + + +PathLocks::PathLocks() + : deletePaths(false) +{ +} + + +PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) + : deletePaths(false) +{ + lockPaths(paths, waitMsg); +} + + +bool PathLocks::lockPaths(const PathSet & _paths, + const string & waitMsg, bool wait) +{ + assert(fds.empty()); + + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Sort the paths. This assures that locks are always acquired in + the same order, thus preventing deadlocks. */ + Paths paths(_paths.begin(), _paths.end()); + paths.sort(); + + /* Acquire the lock for each path. */ + foreach (Paths::iterator, i, paths) { + checkInterrupt(); + Path path = *i; + Path lockPath = path + ".lock"; + + debug(format("locking path `%1%'") % path); + + if (lockedPaths.find(lockPath) != lockedPaths.end()) + throw Error("deadlock: trying to re-acquire self-held lock"); + + AutoCloseFD fd; + + while (1) { + + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd, ltWrite, false)) { + if (wait) { + if (waitMsg != "") printMsg(lvlError, waitMsg); + lockFile(fd, ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + debug(format("lock acquired on `%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError(format("statting lock file `%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file `%1%' has become stale") % lockPath); + else + break; + } + + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.borrow(), lockPath)); + lockedPaths.insert(lockPath); + } + + return true; +} + + +PathLocks::~PathLocks() +{ + unlock(); +} + + +void PathLocks::unlock() +{ + foreach (list<FDPair>::iterator, i, fds) { + if (deletePaths) deleteLockFile(i->second, i->first); + + lockedPaths.erase(i->second); + if (close(i->first) == -1) + printMsg(lvlError, + format("error (ignored): cannot close lock file on `%1%'") % i->second); + + debug(format("lock released on `%1%'") % i->second); + } + + fds.clear(); +} + + +void PathLocks::setDeletion(bool deletePaths) +{ + this->deletePaths = deletePaths; +} + + +bool pathIsLockedByMe(const Path & path) +{ + Path lockPath = path + ".lock"; + return lockedPaths.find(lockPath) != lockedPaths.end(); +} + + +} diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh new file mode 100644 index 000000000000..8a6b1450da2a --- /dev/null +++ b/src/libstore/pathlocks.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "types.hh" + + +namespace nix { + + +/* Open (possibly create) a lock file and return the file descriptor. + -1 is returned if create is false and the lock could not be opened + because it doesn't exist. Any other error throws an exception. */ +int openLockFile(const Path & path, bool create); + +/* Delete an open lock file. */ +void deleteLockFile(const Path & path, int fd); + +enum LockType { ltRead, ltWrite, ltNone }; + +bool lockFile(int fd, LockType lockType, bool wait); + + +class PathLocks +{ +private: + typedef std::pair<int, Path> FDPair; + list<FDPair> fds; + bool deletePaths; + +public: + PathLocks(); + PathLocks(const PathSet & paths, + const string & waitMsg = ""); + bool lockPaths(const PathSet & _paths, + const string & waitMsg = "", + bool wait = true); + ~PathLocks(); + void unlock(); + void setDeletion(bool deletePaths); +}; + + +bool pathIsLockedByMe(const Path & path); + + +} diff --git a/src/libstore/references.cc b/src/libstore/references.cc new file mode 100644 index 000000000000..282b848938b3 --- /dev/null +++ b/src/libstore/references.cc @@ -0,0 +1,122 @@ +#include "references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include <map> +#include <cstdlib> + + +namespace nix { + + +static unsigned int refLength = 32; /* characters */ + + +static void search(const unsigned char * s, unsigned int len, + StringSet & hashes, StringSet & seen) +{ + static bool initialised = false; + static bool isBase32[256]; + if (!initialised) { + for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; + for (unsigned int i = 0; i < base32Chars.size(); ++i) + isBase32[(unsigned char) base32Chars[i]] = true; + initialised = true; + } + + for (unsigned int i = 0; i + refLength <= len; ) { + int j; + bool match = true; + for (j = refLength - 1; j >= 0; --j) + if (!isBase32[(unsigned char) s[i + j]]) { + i += j + 1; + match = false; + break; + } + if (!match) continue; + string ref((const char *) s + i, refLength); + if (hashes.find(ref) != hashes.end()) { + debug(format("found reference to `%1%' at offset `%2%'") + % ref % i); + seen.insert(ref); + hashes.erase(ref); + } + ++i; + } +} + + +struct RefScanSink : Sink +{ + HashSink hashSink; + StringSet hashes; + StringSet seen; + + string tail; + + RefScanSink() : hashSink(htSHA256) { } + + void operator () (const unsigned char * data, size_t len); +}; + + +void RefScanSink::operator () (const unsigned char * data, size_t len) +{ + hashSink(data, len); + + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = tail + string((const char *) data, len > refLength ? refLength : len); + search((const unsigned char *) s.data(), s.size(), hashes, seen); + + search(data, len, hashes, seen); + + unsigned int tailLen = len <= refLength ? len : refLength; + tail = + string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + + string((const char *) data + len - tailLen, tailLen); +} + + +PathSet scanForReferences(const string & path, + const PathSet & refs, HashResult & hash) +{ + RefScanSink sink; + std::map<string, Path> backMap; + + /* For efficiency (and a higher hit rate), just search for the + hash part of the file name. (This assumes that all references + have the form `HASH-bla'). */ + foreach (PathSet::const_iterator, i, refs) { + string baseName = baseNameOf(*i); + string::size_type pos = baseName.find('-'); + if (pos == string::npos) + throw Error(format("bad reference `%1%'") % *i); + string s = string(baseName, 0, pos); + assert(s.size() == refLength); + assert(backMap.find(s) == backMap.end()); + // parseHash(htSHA256, s); + sink.hashes.insert(s); + backMap[s] = *i; + } + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + /* Map the hashes found back to their store paths. */ + PathSet found; + foreach (StringSet::iterator, i, sink.seen) { + std::map<string, Path>::iterator j; + if ((j = backMap.find(*i)) == backMap.end()) abort(); + found.insert(j->second); + } + + hash = sink.hashSink.finish(); + + return found; +} + + +} diff --git a/src/libstore/references.hh b/src/libstore/references.hh new file mode 100644 index 000000000000..013809d122f3 --- /dev/null +++ b/src/libstore/references.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +namespace nix { + +PathSet scanForReferences(const Path & path, const PathSet & refs, + HashResult & hash); + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc new file mode 100644 index 000000000000..3017254baf3b --- /dev/null +++ b/src/libstore/remote-store.cc @@ -0,0 +1,602 @@ +#include "serialise.hh" +#include "util.hh" +#include "remote-store.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <fcntl.h> + +#include <iostream> +#include <unistd.h> +#include <cstring> + +namespace nix { + + +Path readStorePath(Source & from) +{ + Path path = readString(from); + assertStorePath(path); + return path; +} + + +template<class T> T readStorePaths(Source & from) +{ + T paths = readStrings<T>(from); + foreach (typename T::iterator, i, paths) assertStorePath(*i); + return paths; +} + +template PathSet readStorePaths(Source & from); + + +RemoteStore::RemoteStore() +{ + initialised = false; +} + + +void RemoteStore::openConnection(bool reserveSpace) +{ + if (initialised) return; + initialised = true; + + string remoteMode = getEnv("NIX_REMOTE"); + + if (remoteMode == "daemon") + /* Connect to a daemon that does the privileged work for + us. */ + connectToDaemon(); + else + throw Error(format("invalid setting for NIX_REMOTE, `%1%'") % remoteMode); + + from.fd = fdSocket; + to.fd = fdSocket; + + /* Send the magic greeting, check for the reply. */ + try { + writeInt(WORKER_MAGIC_1, to); + to.flush(); + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + + daemonVersion = readInt(from); + if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + writeInt(PROTOCOL_VERSION, to); + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) { + writeInt(1, to); + writeInt(cpu, to); + } else + writeInt(0, to); + } + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) + writeInt(reserveSpace, to); + + processStderr(); + } + catch (Error & e) { + throw Error(format("cannot start worker (%1%)") + % e.msg()); + } + + setOptions(); +} + + +void RemoteStore::connectToDaemon() +{ + fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); + if (fdSocket == -1) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket); + + string socketPath = settings.nixDaemonSocketFile; + + /* Urgh, sockaddr_un allows path names of only 108 characters. So + chdir to the socket directory so that we can pass a relative + path name. !!! this is probably a bad idea in multi-threaded + applications... */ + AutoCloseFD fdPrevDir = open(".", O_RDONLY); + if (fdPrevDir == -1) throw SysError("couldn't open current directory"); + chdir(dirOf(socketPath).c_str()); + Path socketPathRel = "./" + baseNameOf(socketPath); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPathRel.size() >= sizeof(addr.sun_path)) + throw Error(format("socket path `%1%' is too long") % socketPathRel); + using namespace std; + strcpy(addr.sun_path, socketPathRel.c_str()); + + if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at `%1%'") % socketPath); + + if (fchdir(fdPrevDir) == -1) + throw SysError("couldn't change back to previous directory"); +} + + +RemoteStore::~RemoteStore() +{ + try { + to.flush(); + fdSocket.close(); + if (child != -1) + child.wait(true); + } catch (...) { + ignoreException(); + } +} + + +void RemoteStore::setOptions() +{ + writeInt(wopSetOptions, to); + + writeInt(settings.keepFailed, to); + writeInt(settings.keepGoing, to); + writeInt(settings.tryFallback, to); + writeInt(verbosity, to); + writeInt(settings.maxBuildJobs, to); + writeInt(settings.maxSilentTime, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) + writeInt(settings.useBuildHook, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) { + writeInt(settings.buildVerbosity, to); + writeInt(logType, to); + writeInt(settings.printBuildTrace, to); + } + if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) + writeInt(settings.buildCores, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) + writeInt(settings.useSubstitutes, to); + + if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) { + Settings::SettingsMap overrides = settings.getOverrides(); + writeInt(overrides.size(), to); + foreach (Settings::SettingsMap::iterator, i, overrides) { + writeString(i->first, to); + writeString(i->second, to); + } + } + + processStderr(); +} + + +bool RemoteStore::isValidPath(const Path & path) +{ + openConnection(); + writeInt(wopIsValidPath, to); + writeString(path, to); + processStderr(); + unsigned int reply = readInt(from); + return reply != 0; +} + + +PathSet RemoteStore::queryValidPaths(const PathSet & paths) +{ + openConnection(); + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath(*i)) res.insert(*i); + return res; + } else { + writeInt(wopQueryValidPaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } +} + + +PathSet RemoteStore::queryAllValidPaths() +{ + openConnection(); + writeInt(wopQueryAllValidPaths, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) +{ + openConnection(); + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) { + writeInt(wopHasSubstitutes, to); + writeString(*i, to); + processStderr(); + if (readInt(from)) res.insert(*i); + } + return res; + } else { + writeInt(wopQuerySubstitutablePaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } +} + + +void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + if (paths.empty()) return; + + openConnection(); + + if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; + + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + + foreach (PathSet::const_iterator, i, paths) { + SubstitutablePathInfo info; + writeInt(wopQuerySubstitutablePathInfo, to); + writeString(*i, to); + processStderr(); + unsigned int reply = readInt(from); + if (reply == 0) continue; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; + infos[*i] = info; + } + + } else { + + writeInt(wopQuerySubstitutablePathInfos, to); + writeStrings(paths, to); + processStderr(); + unsigned int count = readInt(from); + for (unsigned int n = 0; n < count; n++) { + Path path = readStorePath(from); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = readLongLong(from); + } + + } +} + + +ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +{ + openConnection(); + writeInt(wopQueryPathInfo, to); + writeString(path, to); + processStderr(); + ValidPathInfo info; + info.path = path; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.hash = parseHash(htSHA256, readString(from)); + info.references = readStorePaths<PathSet>(from); + info.registrationTime = readInt(from); + info.narSize = readLongLong(from); + return info; +} + + +Hash RemoteStore::queryPathHash(const Path & path) +{ + openConnection(); + writeInt(wopQueryPathHash, to); + writeString(path, to); + processStderr(); + string hash = readString(from); + return parseHash(htSHA256, hash); +} + + +void RemoteStore::queryReferences(const Path & path, + PathSet & references) +{ + openConnection(); + writeInt(wopQueryReferences, to); + writeString(path, to); + processStderr(); + PathSet references2 = readStorePaths<PathSet>(from); + references.insert(references2.begin(), references2.end()); +} + + +void RemoteStore::queryReferrers(const Path & path, + PathSet & referrers) +{ + openConnection(); + writeInt(wopQueryReferrers, to); + writeString(path, to); + processStderr(); + PathSet referrers2 = readStorePaths<PathSet>(from); + referrers.insert(referrers2.begin(), referrers2.end()); +} + + +Path RemoteStore::queryDeriver(const Path & path) +{ + openConnection(); + writeInt(wopQueryDeriver, to); + writeString(path, to); + processStderr(); + Path drvPath = readString(from); + if (drvPath != "") assertStorePath(drvPath); + return drvPath; +} + + +PathSet RemoteStore::queryValidDerivers(const Path & path) +{ + openConnection(); + writeInt(wopQueryValidDerivers, to); + writeString(path, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +PathSet RemoteStore::queryDerivationOutputs(const Path & path) +{ + openConnection(); + writeInt(wopQueryDerivationOutputs, to); + writeString(path, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +PathSet RemoteStore::queryDerivationOutputNames(const Path & path) +{ + openConnection(); + writeInt(wopQueryDerivationOutputNames, to); + writeString(path, to); + processStderr(); + return readStrings<PathSet>(from); +} + + +Path RemoteStore::queryPathFromHashPart(const string & hashPart) +{ + openConnection(); + writeInt(wopQueryPathFromHashPart, to); + writeString(hashPart, to); + processStderr(); + Path path = readString(from); + if (!path.empty()) assertStorePath(path); + return path; +} + + +Path RemoteStore::addToStore(const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + + openConnection(); + + Path srcPath(absPath(_srcPath)); + + writeInt(wopAddToStore, to); + writeString(baseNameOf(srcPath), to); + /* backwards compatibility hack */ + writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); + writeInt(recursive ? 1 : 0, to); + writeString(printHashType(hashAlgo), to); + dumpPath(srcPath, to, filter); + processStderr(); + return readStorePath(from); +} + + +Path RemoteStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + + openConnection(); + writeInt(wopAddTextToStore, to); + writeString(name, to); + writeString(s, to); + writeStrings(references, to); + + processStderr(); + return readStorePath(from); +} + + +void RemoteStore::exportPath(const Path & path, bool sign, + Sink & sink) +{ + openConnection(); + writeInt(wopExportPath, to); + writeString(path, to); + writeInt(sign ? 1 : 0, to); + processStderr(&sink); /* sink receives the actual data */ + readInt(from); +} + + +Paths RemoteStore::importPaths(bool requireSignature, Source & source) +{ + openConnection(); + writeInt(wopImportPaths, to); + /* We ignore requireSignature, since the worker forces it to true + anyway. */ + processStderr(0, &source); + return readStorePaths<Paths>(from); +} + + +void RemoteStore::buildPaths(const PathSet & drvPaths, bool repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + openConnection(); + writeInt(wopBuildPaths, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 13) + writeStrings(drvPaths, to); + else { + /* For backwards compatibility with old daemons, strip output + identifiers. */ + PathSet drvPaths2; + foreach (PathSet::const_iterator, i, drvPaths) + drvPaths2.insert(string(*i, 0, i->find('!'))); + writeStrings(drvPaths2, to); + } + processStderr(); + readInt(from); +} + + +void RemoteStore::ensurePath(const Path & path) +{ + openConnection(); + writeInt(wopEnsurePath, to); + writeString(path, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::addTempRoot(const Path & path) +{ + openConnection(); + writeInt(wopAddTempRoot, to); + writeString(path, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::addIndirectRoot(const Path & path) +{ + openConnection(); + writeInt(wopAddIndirectRoot, to); + writeString(path, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::syncWithGC() +{ + openConnection(); + writeInt(wopSyncWithGC, to); + processStderr(); + readInt(from); +} + + +Roots RemoteStore::findRoots() +{ + openConnection(); + writeInt(wopFindRoots, to); + processStderr(); + unsigned int count = readInt(from); + Roots result; + while (count--) { + Path link = readString(from); + Path target = readStorePath(from); + result[link] = target; + } + return result; +} + + +void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + openConnection(false); + + writeInt(wopCollectGarbage, to); + writeInt(options.action, to); + writeStrings(options.pathsToDelete, to); + writeInt(options.ignoreLiveness, to); + writeLongLong(options.maxFreed, to); + writeInt(0, to); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) { + /* removed options */ + writeInt(0, to); + writeInt(0, to); + } + + processStderr(); + + results.paths = readStrings<PathSet>(from); + results.bytesFreed = readLongLong(from); + readLongLong(from); // obsolete +} + + +PathSet RemoteStore::queryFailedPaths() +{ + openConnection(); + writeInt(wopQueryFailedPaths, to); + processStderr(); + return readStorePaths<PathSet>(from); +} + + +void RemoteStore::clearFailedPaths(const PathSet & paths) +{ + openConnection(); + writeInt(wopClearFailedPaths, to); + writeStrings(paths, to); + processStderr(); + readInt(from); +} + + +void RemoteStore::processStderr(Sink * sink, Source * source) +{ + to.flush(); + unsigned int msg; + while ((msg = readInt(from)) == STDERR_NEXT + || msg == STDERR_READ || msg == STDERR_WRITE) { + if (msg == STDERR_WRITE) { + string s = readString(from); + if (!sink) throw Error("no sink"); + (*sink)((const unsigned char *) s.data(), s.size()); + } + else if (msg == STDERR_READ) { + if (!source) throw Error("no source"); + size_t len = readInt(from); + unsigned char * buf = new unsigned char[len]; + AutoDeleteArray<unsigned char> d(buf); + writeString(buf, source->read(buf, len), to); + to.flush(); + } + else { + string s = readString(from); + writeToStderr(s); + } + } + if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1; + throw Error(format("%1%") % error, status); + } + else if (msg != STDERR_LAST) + throw Error("protocol error processing standard error"); +} + + +} diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh new file mode 100644 index 000000000000..14253b92cd0e --- /dev/null +++ b/src/libstore/remote-store.hh @@ -0,0 +1,104 @@ +#pragma once + +#include <string> + +#include "store-api.hh" + + +namespace nix { + + +class Pipe; +class Pid; +struct FdSink; +struct FdSource; + + +class RemoteStore : public StoreAPI +{ +public: + + RemoteStore(); + + ~RemoteStore(); + + /* Implementations of abstract store API methods. */ + + bool isValidPath(const Path & path); + + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); + + ValidPathInfo queryPathInfo(const Path & path); + + Hash queryPathHash(const Path & path); + + void queryReferences(const Path & path, PathSet & references); + + void queryReferrers(const Path & path, PathSet & referrers); + + Path queryDeriver(const Path & path); + + PathSet queryValidDerivers(const Path & path); + + PathSet queryDerivationOutputs(const Path & path); + + StringSet queryDerivationOutputNames(const Path & path); + + Path queryPathFromHashPart(const string & hashPart); + + PathSet querySubstitutablePaths(const PathSet & paths); + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); + + Path addToStore(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false); + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false); + + void exportPath(const Path & path, bool sign, + Sink & sink); + + Paths importPaths(bool requireSignature, Source & source); + + void buildPaths(const PathSet & paths, bool repair = false); + + void ensurePath(const Path & path); + + void addTempRoot(const Path & path); + + void addIndirectRoot(const Path & path); + + void syncWithGC(); + + Roots findRoots(); + + void collectGarbage(const GCOptions & options, GCResults & results); + + PathSet queryFailedPaths(); + + void clearFailedPaths(const PathSet & paths); + +private: + AutoCloseFD fdSocket; + FdSink to; + FdSource from; + Pid child; + unsigned int daemonVersion; + bool initialised; + + void openConnection(bool reserveSpace = true); + + void processStderr(Sink * sink = 0, Source * source = 0); + + void connectToDaemon(); + + void setOptions(); +}; + + +} diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql new file mode 100644 index 000000000000..c1b4a689afcb --- /dev/null +++ b/src/libstore/schema.sql @@ -0,0 +1,44 @@ +create table if not exists ValidPaths ( + id integer primary key autoincrement not null, + path text unique not null, + hash text not null, + registrationTime integer not null, + deriver text, + narSize integer +); + +create table if not exists Refs ( + referrer integer not null, + reference integer not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict +); + +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger if not exists DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); + +create table if not exists FailedPaths ( + path text primary key not null, + time integer not null +); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc new file mode 100644 index 000000000000..0f250a3c7c04 --- /dev/null +++ b/src/libstore/store-api.cc @@ -0,0 +1,331 @@ +#include "store-api.hh" +#include "globals.hh" +#include "util.hh" + +#include <climits> + + +namespace nix { + + +GCOptions::GCOptions() +{ + action = gcDeleteDead; + ignoreLiveness = false; + maxFreed = ULLONG_MAX; +} + + +bool isInStore(const Path & path) +{ + return isInDir(path, settings.nixStore); +} + + +bool isStorePath(const Path & path) +{ + return isInStore(path) + && path.find('/', settings.nixStore.size() + 1) == Path::npos; +} + + +void assertStorePath(const Path & path) +{ + if (!isStorePath(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); +} + + +Path toStorePath(const Path & path) +{ + if (!isInStore(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); + Path::size_type slash = path.find('/', settings.nixStore.size() + 1); + if (slash == Path::npos) + return path; + else + return Path(path, 0, slash); +} + + +Path followLinksToStore(const Path & _path) +{ + Path path = absPath(_path); + while (!isInStore(path)) { + if (!isLink(path)) break; + string target = readLink(path); + path = absPath(target, dirOf(path)); + } + if (!isInStore(path)) + throw Error(format("path `%1%' is not in the Nix store") % path); + return path; +} + + +Path followLinksToStorePath(const Path & path) +{ + return toStorePath(followLinksToStore(path)); +} + + +string storePathToName(const Path & path) +{ + assertStorePath(path); + return string(path, settings.nixStore.size() + 34); +} + + +void checkStoreName(const string & name) +{ + string validChars = "+-._?="; + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (string(name, 0, 1) == ".") + throw Error(format("illegal name: `%1%'") % name); + foreach (string::const_iterator, i, name) + if (!((*i >= 'A' && *i <= 'Z') || + (*i >= 'a' && *i <= 'z') || + (*i >= '0' && *i <= '9') || + validChars.find(*i) != string::npos)) + { + throw Error(format("invalid character `%1%' in name `%2%'") + % *i % name); + } +} + + +/* Store paths have the following form: + + <store>/<h>-<name> + + where + + <store> = the location of the Nix store, usually /nix/store + + <name> = a human readable name for the path, typically obtained + from the name attribute of the derivation, or the name of the + source file from which the store path is created. For derivation + outputs other than the default "out" output, the string "-<id>" + is suffixed to <name>. + + <h> = base-32 representation of the first 160 bits of a SHA-256 + hash of <s>; the hash part of the store name + + <s> = the string "<type>:sha256:<h2>:<store>:<name>"; + note that it includes the location of the store as well as the + name to make sure that changes to either of those are reflected + in the hash (e.g. you won't get /nix/store/<h>-name1 and + /nix/store/<h>-name2 with equal hash parts). + + <type> = one of: + "text:<r1>:<r2>:...<rN>" + for plain text files written to the store using + addTextToStore(); <r1> ... <rN> are the references of the + path. + "source" + for paths copied to the store using addToStore() when recursive + = true and hashAlgo = "sha256" + "output:<id>" + for either the outputs created by derivations, OR paths copied + to the store using addToStore() with recursive != true or + hashAlgo != "sha256" (in that case "source" is used; it's + silly, but it's done that way for compatibility). <id> is the + name of the output (usually, "out"). + + <h2> = base-16 representation of a SHA-256 hash of: + if <type> = "text:...": + the string written to the resulting store path + if <type> = "source": + the serialisation of the path from which this store path is + copied, as returned by hashPath() + if <type> = "output:out": + for non-fixed derivation outputs: + the derivation (see hashDerivationModulo() in + primops.cc) + for paths copied by addToStore() or produced by fixed-output + derivations: + the string "fixed:out:<rec><algo>:<hash>:", where + <rec> = "r:" for recursive (path) hashes, or "" or flat + (file) hashes + <algo> = "md5", "sha1" or "sha256" + <hash> = base-16 representation of the path or flat hash of + the contents of the path (or expected contents of the + path for fixed-output derivations) + + It would have been nicer to handle fixed-output derivations under + "source", e.g. have something like "source:<rec><algo>", but we're + stuck with this for now... + + The main reason for this way of computing names is to prevent name + collisions (for security). For instance, it shouldn't be feasible + to come up with a derivation whose output path collides with the + path for a copied source. The former would have a <s> starting with + "output:out:", while the latter would have a <2> starting with + "source:". +*/ + + +Path makeStorePath(const string & type, + const Hash & hash, const string & name) +{ + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + string s = type + ":sha256:" + printHash(hash) + ":" + + settings.nixStore + ":" + name; + + checkStoreName(name); + + return settings.nixStore + "/" + + printHash32(compressHash(hashString(htSHA256, s), 20)) + + "-" + name; +} + + +Path makeOutputPath(const string & id, + const Hash & hash, const string & name) +{ + return makeStorePath("output:" + id, hash, + name + (id == "out" ? "" : "-" + id)); +} + + +Path makeFixedOutputPath(bool recursive, + HashType hashAlgo, Hash hash, string name) +{ + return hashAlgo == htSHA256 && recursive + ? makeStorePath("source", hash, name) + : makeStorePath("output:out", hashString(htSHA256, + "fixed:out:" + (recursive ? (string) "r:" : "") + + printHashType(hashAlgo) + ":" + printHash(hash) + ":"), + name); +} + + +std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter) +{ + HashType ht(hashAlgo); + Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath); + string name = baseNameOf(srcPath); + Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); + return std::pair<Path, Hash>(dstPath, h); +} + + +Path computeStorePathForText(const string & name, const string & s, + const PathSet & references) +{ + Hash hash = hashString(htSHA256, s); + /* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ + string type = "text"; + foreach (PathSet::const_iterator, i, references) { + type += ":"; + type += *i; + } + return makeStorePath(type, hash, name); +} + + +/* Return a string accepted by decodeValidPathInfo() that + registers the specified paths as valid. Note: it's the + responsibility of the caller to provide a closure. */ +string StoreAPI::makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash) +{ + string s = ""; + + foreach (PathSet::iterator, i, paths) { + s += *i + "\n"; + + ValidPathInfo info = queryPathInfo(*i); + + if (showHash) { + s += printHash(info.hash) + "\n"; + s += (format("%1%\n") % info.narSize).str(); + } + + Path deriver = showDerivers ? info.deriver : ""; + s += deriver + "\n"; + + s += (format("%1%\n") % info.references.size()).str(); + + foreach (PathSet::iterator, j, info.references) + s += *j + "\n"; + } + + return s; +} + + +ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) +{ + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { info.path = ""; return info; } + if (hashGiven) { + string s; + getline(str, s); + info.hash = parseHash(htSHA256, s); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); + } + getline(str, info.deriver); + string s; int n; + getline(str, s); + if (!string2Int(s, n)) throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) throw Error("missing input"); + return info; +} + + +string showPaths(const PathSet & paths) +{ + string s; + foreach (PathSet::const_iterator, i, paths) { + if (s.size() != 0) s += ", "; + s += "`" + *i + "'"; + } + return s; +} + + +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink) +{ + foreach (Paths::const_iterator, i, paths) { + writeInt(1, sink); + store.exportPath(*i, sign, sink); + } + writeInt(0, sink); +} + + +} + + +#include "local-store.hh" +#include "serialise.hh" +#include "remote-store.hh" + + +namespace nix { + + +boost::shared_ptr<StoreAPI> store; + + +boost::shared_ptr<StoreAPI> openStore(bool reserveSpace) +{ + if (getEnv("NIX_REMOTE") == "") + return boost::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); + else + return boost::shared_ptr<StoreAPI>(new RemoteStore()); +} + + +} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh new file mode 100644 index 000000000000..a82fe3221639 --- /dev/null +++ b/src/libstore/store-api.hh @@ -0,0 +1,364 @@ +#pragma once + +#include "hash.hh" +#include "serialise.hh" + +#include <string> +#include <map> + +#include <boost/shared_ptr.hpp> + + +namespace nix { + + +typedef std::map<Path, Path> Roots; + + +struct GCOptions +{ + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; + + GCAction action; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness; + + /* For `gcDeleteSpecific', the paths to delete. */ + PathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + unsigned long long maxFreed; + + GCOptions(); +}; + + +struct GCResults +{ + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + unsigned long long bytesFreed; + + GCResults() + { + bytesFreed = 0; + } +}; + + +struct SubstitutablePathInfo +{ + Path deriver; + PathSet references; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; + + +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; + time_t registrationTime; + unsigned long long narSize; // 0 = unknown + unsigned long long id; // internal use only + ValidPathInfo() : registrationTime(0), narSize(0) { } +}; + +typedef list<ValidPathInfo> ValidPathInfos; + + +class StoreAPI +{ +public: + + virtual ~StoreAPI() { } + + /* Check whether a path is valid. */ + virtual bool isValidPath(const Path & path) = 0; + + /* Query which of the given paths is valid. */ + virtual PathSet queryValidPaths(const PathSet & paths) = 0; + + /* Query the set of all valid paths. */ + virtual PathSet queryAllValidPaths() = 0; + + /* Query information about a valid path. */ + virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + + /* Query the hash of a valid path. */ + virtual Hash queryPathHash(const Path & path) = 0; + + /* Query the set of outgoing FS references for a store path. The + result is not cleared. */ + virtual void queryReferences(const Path & path, + PathSet & references) = 0; + + /* Queries the set of incoming FS references for a store path. + The result is not cleared. */ + virtual void queryReferrers(const Path & path, + PathSet & referrers) = 0; + + /* Query the deriver of a store path. Return the empty string if + no deriver has been set. */ + virtual Path queryDeriver(const Path & path) = 0; + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + virtual PathSet queryValidDerivers(const Path & path) = 0; + + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path & path) = 0; + + /* Query the output names of the derivation denoted by `path'. */ + virtual StringSet queryDerivationOutputNames(const Path & path) = 0; + + /* Query the full store path given the hash part of a valid store + path, or "" if the path doesn't exist. */ + virtual Path queryPathFromHashPart(const string & hashPart) = 0; + + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) = 0; + + /* Copy the contents of a path to the store and register the + validity the resulting path. The resulting path is returned. + The function object `filter' can be used to exclude files (see + libutil/archive.hh). */ + virtual Path addToStore(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false) = 0; + + /* Like addToStore, but the contents written to the output path is + a regular file containing the given string. */ + virtual Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false) = 0; + + /* Export a store path, that is, create a NAR dump of the store + path and append its references and its deriver. Optionally, a + cryptographic signature (created by OpenSSL) of the preceding + data is attached. */ + virtual void exportPath(const Path & path, bool sign, + Sink & sink) = 0; + + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. */ + virtual Paths importPaths(bool requireSignature, Source & source) = 0; + + /* For each path, if it's a derivation, build it. Building a + derivation means ensuring that the output paths are valid. If + they are already valid, this is a no-op. Otherwise, validity + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. For inputs that are + not derivations, substitute them. */ + virtual void buildPaths(const PathSet & paths, bool repair = false) = 0; + + /* Ensure that a path is valid. If it is not currently valid, it + may be made valid by running a substitute (if defined for the + path). */ + virtual void ensurePath(const Path & path) = 0; + + /* Add a store path as a temporary root of the garbage collector. + The root disappears as soon as we exit. */ + virtual void addTempRoot(const Path & path) = 0; + + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path & path) = 0; + + /* Acquire the global GC lock, then immediately release it. This + function must be called after registering a new permanent root, + but before exiting. Otherwise, it is possible that a running + garbage collector doesn't see the new root and deletes the + stuff we've just built. By acquiring the lock briefly, we + ensure that either: + + - The collector is already running, and so we block until the + collector is finished. The collector will know about our + *temporary* locks, which should include whatever it is we + want to register as a permanent lock. + + - The collector isn't running, or it's just started but hasn't + acquired the GC lock yet. In that case we get and release + the lock right away, then exit. The collector scans the + permanent root and sees our's. + + In either case the permanent root is seen by the collector. */ + virtual void syncWithGC() = 0; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. */ + virtual Roots findRoots() = 0; + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /* Return the set of paths that have failed to build.*/ + virtual PathSet queryFailedPaths() = 0; + + /* Clear the "failed" status of the given paths. The special + value `*' causes all failed paths to be cleared. */ + virtual void clearFailedPaths(const PathSet & paths) = 0; + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); +}; + + +/* !!! These should be part of the store API, I guess. */ + +/* Throw an exception if `path' is not directly in the Nix store. */ +void assertStorePath(const Path & path); + +bool isInStore(const Path & path); +bool isStorePath(const Path & path); + +/* Extract the name part of the given store path. */ +string storePathToName(const Path & path); + +void checkStoreName(const string & name); + + +/* Chop off the parts after the top-level store name, e.g., + /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ +Path toStorePath(const Path & path); + + +/* Follow symlinks until we end up with a path in the Nix store. */ +Path followLinksToStore(const Path & path); + + +/* Same as followLinksToStore(), but apply toStorePath() to the + result. */ +Path followLinksToStorePath(const Path & path); + + +/* Constructs a unique store path name. */ +Path makeStorePath(const string & type, + const Hash & hash, const string & name); + +Path makeOutputPath(const string & id, + const Hash & hash, const string & name); + +Path makeFixedOutputPath(bool recursive, + HashType hashAlgo, Hash hash, string name); + + +/* This is the preparatory part of addToStore() and addToStoreFixed(); + it computes the store path to which srcPath is to be copied. + Returns the store path and the cryptographic hash of the + contents of srcPath. */ +std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter); + +/* Preparatory part of addTextToStore(). + + !!! Computation of the path should take the references given to + addTextToStore() into account, otherwise we have a (relatively + minor) security hole: a caller can register a source file with + bogus references. If there are too many references, the path may + not be garbage collected when it has to be (not really a problem, + the caller could create a root anyway), or it may be garbage + collected when it shouldn't be (more serious). + + Hashing the references would solve this (bogus references would + simply yield a different store path, so other users wouldn't be + affected), but it has some backwards compatibility issues (the + hashing scheme changes), so I'm not doing that for now. */ +Path computeStorePathForText(const string & name, const string & s, + const PathSet & references); + + +/* Remove the temporary roots file for this process. Any temporary + root becomes garbage after this point unless it has been registered + as a (permanent) root. */ +void removeTempRoots(); + + +/* Register a permanent GC root. */ +Path addPermRoot(StoreAPI & store, const Path & storePath, + const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + + +/* Sort a set of paths topologically under the references relation. + If p refers to q, then p preceeds q in this list. */ +Paths topoSortPaths(StoreAPI & store, const PathSet & paths); + + +/* For now, there is a single global store API object, but we'll + purify that in the future. */ +extern boost::shared_ptr<StoreAPI> store; + + +/* Factory method: open the Nix database, either through the local or + remote implementation. */ +boost::shared_ptr<StoreAPI> openStore(bool reserveSpace = true); + + +/* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ +string showPaths(const PathSet & paths); + + +ValidPathInfo decodeValidPathInfo(std::istream & str, + bool hashGiven = false); + + +/* Export multiple paths in the format expected by ‘nix-store + --import’. */ +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink); + + +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ + + +} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh new file mode 100644 index 000000000000..9317f89c3763 --- /dev/null +++ b/src/libstore/worker-protocol.hh @@ -0,0 +1,60 @@ +#pragma once + +namespace nix { + + +#define WORKER_MAGIC_1 0x6e697863 +#define WORKER_MAGIC_2 0x6478696f + +#define PROTOCOL_VERSION 0x10e +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + + +typedef enum { + wopQuit = 0, + wopIsValidPath = 1, + wopHasSubstitutes = 3, + wopQueryPathHash = 4, + wopQueryReferences = 5, + wopQueryReferrers = 6, + wopAddToStore = 7, + wopAddTextToStore = 8, + wopBuildPaths = 9, + wopEnsurePath = 10, + wopAddTempRoot = 11, + wopAddIndirectRoot = 12, + wopSyncWithGC = 13, + wopFindRoots = 14, + wopExportPath = 16, + wopQueryDeriver = 18, + wopSetOptions = 19, + wopCollectGarbage = 20, + wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, + wopQueryAllValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, + wopQueryPathInfo = 26, + wopImportPaths = 27, + wopQueryDerivationOutputNames = 28, + wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, + wopQueryValidDerivers = 33, +} WorkerOp; + + +#define STDERR_NEXT 0x6f6c6d67 +#define STDERR_READ 0x64617461 // data needed from source +#define STDERR_WRITE 0x64617416 // data for sink +#define STDERR_LAST 0x616c7473 +#define STDERR_ERROR 0x63787470 + + +Path readStorePath(Source & from); +template<class T> T readStorePaths(Source & from); + + +} |