From 96de272b48f8e9bdabffddb699ed4f2292d4f1d7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Jan 2005 16:39:47 +0000 Subject: * Renamed `normalise.cc' -> `build.cc', `storeexprs.cc' -> `derivations.cc', etc. * Store the SHA-256 content hash of store paths in the database after they have been built/added. This is so that we can check whether the store has been messed with (a la `rpm --verify'). * When registering path validity, verify that the closure property holds. --- src/libstore/Makefile.am | 14 +- src/libstore/build.cc | 1758 +++++++++++++++++++++++++++++++++++++ src/libstore/build.hh | 47 + src/libstore/derivations-ast.def | 6 + src/libstore/derivations.cc | 142 +++ src/libstore/derivations.hh | 62 ++ src/libstore/gc.cc | 2 +- src/libstore/gc.hh | 2 +- src/libstore/misc.cc | 2 +- src/libstore/normalise.cc | 1804 -------------------------------------- src/libstore/normalise.hh | 47 - src/libstore/store.cc | 72 +- src/libstore/store.hh | 20 +- src/libstore/storeexpr-ast.def | 6 - src/libstore/storeexpr.cc | 142 --- src/libstore/storeexpr.hh | 62 -- 16 files changed, 2109 insertions(+), 2079 deletions(-) create mode 100644 src/libstore/build.cc create mode 100644 src/libstore/build.hh create mode 100644 src/libstore/derivations-ast.def create mode 100644 src/libstore/derivations.cc create mode 100644 src/libstore/derivations.hh delete mode 100644 src/libstore/normalise.cc delete mode 100644 src/libstore/normalise.hh delete mode 100644 src/libstore/storeexpr-ast.def delete mode 100644 src/libstore/storeexpr.cc delete mode 100644 src/libstore/storeexpr.hh (limited to 'src/libstore') diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 7e0f32d1f836..37d2f82b4833 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -1,18 +1,18 @@ noinst_LIBRARIES = libstore.a libstore_a_SOURCES = \ - store.cc store.hh storeexpr.cc storeexpr.hh \ - normalise.cc misc.cc normalise.hh \ + store.cc store.hh derivations.cc derivations.hh \ + build.cc misc.cc build.hh \ globals.cc globals.hh db.cc db.hh \ references.cc references.hh pathlocks.cc pathlocks.hh \ - gc.cc gc.hh storeexpr-ast.hh + gc.cc gc.hh derivations-ast.hh -EXTRA_DIST = storeexpr-ast.def storeexpr-ast.cc +EXTRA_DIST = derivations-ast.def derivations-ast.cc AM_CXXFLAGS = -Wall \ -I.. ${bdb_include} ${aterm_include} -I../libutil -storeexpr-ast.cc storeexpr-ast.hh: ../aterm-helper.pl storeexpr-ast.def - $(perl) ../aterm-helper.pl storeexpr-ast.hh storeexpr-ast.cc < storeexpr-ast.def +derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def + $(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def -storeexpr.cc: storeexpr-ast.hh \ No newline at end of file +derivations.cc: derivations-ast.hh \ No newline at end of file diff --git a/src/libstore/build.cc b/src/libstore/build.cc new file mode 100644 index 000000000000..60e72c9dc195 --- /dev/null +++ b/src/libstore/build.cc @@ -0,0 +1,1758 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "build.hh" +#include "references.hh" +#include "pathlocks.hh" +#include "globals.hh" + + +/* !!! TODO storeExprFromPath shouldn't be used here */ + + +static string pathNullDevice = "/dev/null"; + + +/* Forward definition. */ +class Worker; + + +/* A pointer to a goal. */ +class Goal; +typedef shared_ptr GoalPtr; +typedef weak_ptr WeakGoalPtr; + +/* Set of goals. */ +typedef set Goals; +typedef set WeakGoals; + +/* A map of paths to goals (and the other way around). */ +typedef map WeakGoalMap; + + + +class Goal : public enable_shared_from_this +{ +protected: + + /* Backlink to the worker. */ + Worker & worker; + + /* Goals that this goal is waiting for. */ + Goals waitees; + + /* Goals waiting for this one to finish. Must use weak pointers + here to prevent cycles. */ + WeakGoals waiters; + + /* Number of goals we are/were waiting for that have failed. */ + unsigned int nrFailed; + + /* Whether amDone() has been called. */ + bool done; + + + Goal(Worker & worker) : worker(worker) + { + done = false; + nrFailed = 0; + } + + virtual ~Goal() + { + printMsg(lvlVomit, "goal destroyed"); + } + +public: + virtual void work() = 0; + + virtual string name() = 0; + + void addWaitee(GoalPtr waitee); + + virtual void waiteeDone(GoalPtr waitee, bool success); + + virtual void writeLog(int fd, const unsigned char * buf, size_t count) + { + abort(); + } + + void trace(const format & f); + +protected: + void amDone(bool success = true); +}; + + +/* A mapping used to remember for each child process to what goal it + belongs, and a file descriptor for receiving log data. */ +struct Child +{ + WeakGoalPtr goal; + int fdOutput; + bool inBuildSlot; +}; + +typedef map Children; + + +/* The worker class. */ +class Worker +{ +private: + + /* Note: the worker should only have strong pointers to the + top-level goals. */ + + /* The top-level goals of the worker. */ + Goals topGoals; + + /* Goals that are ready to do some work. */ + WeakGoals awake; + + /* Goals waiting for a build slot. */ + WeakGoals wantingToBuild; + + /* Child processes currently running. */ + Children children; + + /* Number of build slots occupied. Not all child processes + (namely build hooks) count as occupied build slots. */ + unsigned int nrChildren; + + /* Maps used to prevent multiple instantiations of a goal for the + same expression / path. */ + WeakGoalMap derivationGoals; + WeakGoalMap substitutionGoals; + +public: + + Worker(); + ~Worker(); + + /* Make a goal (with caching). */ + GoalPtr makeDerivationGoal(const Path & drvPath); + GoalPtr makeSubstitutionGoal(const Path & storePath); + + /* Remove a dead goal. */ + void removeGoal(GoalPtr goal); + + /* Wake up a goal (i.e., there is something for it to do). */ + void wakeUp(GoalPtr goal); + + /* Can we start another child process? */ + bool canBuildMore(); + + /* Registers / unregisters a running child process. */ + void childStarted(GoalPtr goal, pid_t pid, int fdOutput, + bool inBuildSlot); + void childTerminated(pid_t pid, bool wakeSleepers = true); + + /* Add a goal to the set of goals waiting for a build slot. */ + void waitForBuildSlot(GoalPtr goal, bool reallyWait = false); + + /* Loop until the specified top-level goal has finished. Returns + true if it has finished succesfully. */ + bool run(const Goals & topGoals); + + /* Wait for input to become available. */ + void waitForInput(); +}; + + +class SubstError : public Error +{ +public: + SubstError(const format & f) : Error(f) { }; +}; + + +class BuildError : public Error +{ +public: + BuildError(const format & f) : Error(f) { }; +}; + + + +////////////////////////////////////////////////////////////////////// + + +void Goal::addWaitee(GoalPtr waitee) +{ + waitees.insert(waitee); + waitee->waiters.insert(shared_from_this()); +} + + +void Goal::waiteeDone(GoalPtr waitee, bool success) +{ + assert(waitees.find(waitee) != waitees.end()); + waitees.erase(waitee); + + if (!success) ++nrFailed; + + if (waitees.empty() || (!success && !keepGoing)) { + + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) { + GoalPtr goal = *i; + WeakGoals waiters2; + for (WeakGoals::iterator j = goal->waiters.begin(); + j != goal->waiters.end(); ++j) + if (j->lock() != shared_from_this()) + waiters2.insert(*j); + goal->waiters = waiters2; + } + waitees.clear(); + + worker.wakeUp(shared_from_this()); + } +} + + +void Goal::amDone(bool success) +{ + trace("done"); + assert(!done); + done = true; + for (WeakGoals::iterator i = waiters.begin(); i != waiters.end(); ++i) { + GoalPtr goal = i->lock(); + if (goal) goal->waiteeDone(shared_from_this(), success); + } + waiters.clear(); + worker.removeGoal(shared_from_this()); +} + + +void Goal::trace(const format & f) +{ + debug(format("%1%: %2%") % name() % f); +} + + + +////////////////////////////////////////////////////////////////////// + + +/* Common initialisation performed in child processes. */ +void commonChildInit(Pipe & logPipe) +{ + /* Put the child in a separate process group so that it doesn't + receive terminal signals. */ + if (setpgid(0, 0) == -1) + throw SysError(format("setting process group")); + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + logPipe.readSide.close(); + + /* Dup stderr to stdin. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError(format("cannot open `%1%'") % pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); +} + + +/* Convert a string list to an array of char pointers. Careful: the + string list should outlive the array. */ +const char * * strings2CharPtrs(const Strings & ss) +{ + const char * * arr = new const char * [ss.size() + 1]; + const char * * p = arr; + for (Strings::const_iterator i = ss.begin(); i != ss.end(); ++i) + *p++ = i->c_str(); + *p = 0; + return arr; +} + + + +////////////////////////////////////////////////////////////////////// + + +class DerivationGoal : public Goal +{ +private: + /* The path of the derivation store expression. */ + Path drvPath; + + /* The derivation store expression stored at drvPath. */ + Derivation drv; + + /* The remainder is state held during the build. */ + + /* Locks on the output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + PathSet inputPaths; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* File descriptor for the log file. */ + AutoCloseFD fdLogFile; + + /* Pipe for the builder's standard output/error. */ + Pipe logPipe; + + /* Pipes for talking to the build hook (if any). */ + Pipe toHook; + Pipe fromHook; + + typedef void (DerivationGoal::*GoalState)(); + GoalState state; + +public: + DerivationGoal(const Path & drvPath, Worker & worker); + ~DerivationGoal(); + + void work(); + +private: + /* The states. */ + void init(); + void haveStoreExpr(); + void inputsRealised(); + void tryToBuild(); + void buildDone(); + + /* Is the build hook willing to perform the build? */ + typedef enum {rpAccept, rpDecline, rpPostpone, rpDone} HookReply; + HookReply tryBuildHook(); + + /* Synchronously wait for a build hook to finish. */ + void terminateBuildHook(); + + /* Acquires locks on the output paths and gathers information + about the build (e.g., the input closures). During this + process its possible that we find out that the build is + unnecessary, in which case we return false (this is not an + error condition!). */ + bool prepareBuild(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Must be called after the output paths have become valid (either + due to a successful build or hook, or because they already + were). */ + void computeClosure(); + + /* Open a log file and a pipe to it. */ + void openLogFile(); + + /* Common initialisation to be performed in child processes (i.e., + both in builders and in build hooks. */ + void initChild(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void writeLog(int fd, const unsigned char * buf, size_t count); + + /* Return true iff all output paths are valid. */ + bool allOutputsValid(); + + string name(); +}; + + +DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker) + : Goal(worker) +{ + this->drvPath = drvPath; + state = &DerivationGoal::init; +} + + +DerivationGoal::~DerivationGoal() +{ + if (pid != -1) worker.childTerminated(pid); + + /* Careful: we should never ever throw an exception from a + destructor. */ + try { + deleteTmpDir(false); + } catch (Error & e) { + printMsg(lvlError, format("error (ignored): %1%") % e.msg()); + } +} + + +void DerivationGoal::work() +{ + (this->*state)(); +} + + +void DerivationGoal::init() +{ + trace("init"); + + /* The first thing to do is to make sure that the store expression + exists. If it doesn't, it may be created through a + substitute. */ + addWaitee(worker.makeSubstitutionGoal(drvPath)); + + state = &DerivationGoal::haveStoreExpr; +} + + +void DerivationGoal::haveStoreExpr() +{ + trace("loading derivation"); + + if (nrFailed != 0) { + printMsg(lvlError, + format("cannot build missing derivation `%1%'") + % drvPath); + amDone(false); + return; + } + + assert(isValidPath(drvPath)); + + /* Get the derivation. */ + drv = derivationFromPath(drvPath); + + /* If all the outputs already exist, then we're done. */ + if (allOutputsValid()) { + amDone(true); + return; + } + + /* Inputs must be built before we can build this goal. */ + for (PathSet::iterator i = drv.inputDrvs.begin(); + i != drv.inputDrvs.end(); ++i) + addWaitee(worker.makeDerivationGoal(*i)); + + for (PathSet::iterator i = drv.inputSrcs.begin(); + i != drv.inputSrcs.end(); ++i) + addWaitee(worker.makeSubstitutionGoal(*i)); + + state = &DerivationGoal::inputsRealised; +} + + +void DerivationGoal::inputsRealised() +{ + trace("all inputs realised"); + + if (nrFailed != 0) { + printMsg(lvlError, + format("cannot build derivation `%1%': " + "%2% inputs could not be realised") + % drvPath % nrFailed); + amDone(false); + return; + } + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); +} + + +void DerivationGoal::tryToBuild() +{ + trace("trying to build"); + + try { + + /* Is the build hook willing to accept this job? */ + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + state = &DerivationGoal::buildDone; + return; + case rpPostpone: + /* Not now; wait until at least one child finishes. */ + worker.waitForBuildSlot(shared_from_this(), true); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + case rpDone: + /* Somebody else did it. */ + amDone(); + return; + } + + /* Make sure that we are allowed to start a build. */ + if (!worker.canBuildMore()) { + worker.waitForBuildSlot(shared_from_this()); + return; + } + + /* Acquire locks and such. If we then see that the build has + been done by somebody else, we're done. */ + if (!prepareBuild()) { + amDone(); + return; + } + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + printMsg(lvlError, e.msg()); + amDone(false); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; +} + + +void DerivationGoal::buildDone() +{ + trace("build done"); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe --- just don't do that + :-) */ + /* !!! this could block! */ + pid_t savedPid = pid; + int status = pid.wait(true); + + /* So the child is gone now. */ + worker.childTerminated(savedPid); + + /* Close the read side of the logger pipe. */ + logPipe.readSide.close(); + + /* Close the log file. */ + fdLogFile.close(); + + debug(format("builder process for `%1%' finished") % drvPath); + + /* Check the exit status. */ + if (!statusOk(status)) { + deleteTmpDir(false); + printMsg(lvlError, format("builder for `%1%' %2%") + % drvPath % statusToString(status)); + amDone(false); + return; + } + + deleteTmpDir(true); + + /* Compute the FS closure of the outputs and register them as + being valid. */ + try { + computeClosure(); + } catch (BuildError & e) { + printMsg(lvlError, e.msg()); + amDone(false); + return; + } + + amDone(); +} + + +static string readLine(int fd) +{ + string s; + while (1) { + char ch; + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw Error("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +static void writeLine(int fd, string s) +{ + s += '\n'; + writeFull(fd, (const unsigned char *) s.c_str(), s.size()); +} + + +/* !!! ugly hack */ +static void drain(int fd) +{ + unsigned char buffer[1024]; + while (1) { + ssize_t rd = read(fd, buffer, sizeof buffer); + if (rd == -1) { + if (errno != EINTR) + throw SysError("draining"); + } else if (rd == 0) break; + else writeFull(STDERR_FILENO, buffer, rd); + } +} + + +PathSet outputPaths(const DerivationOutputs & outputs) +{ + PathSet paths; + for (DerivationOutputs::const_iterator i = outputs.begin(); + i != outputs.end(); ++i) + paths.insert(i->second.path); + return paths; +} + + +string showPaths(const PathSet & paths) +{ + string s; + for (PathSet::const_iterator i = paths.begin(); + i != paths.end(); ++i) + { + if (s.size() != 0) s += ", "; + s += "`" + *i + "'"; + } + return s; +} + + +DerivationGoal::HookReply DerivationGoal::tryBuildHook() +{ + Path buildHook = getEnv("NIX_BUILD_HOOK"); + if (buildHook == "") return rpDecline; + buildHook = absPath(buildHook); + + /* Create a directory where we will store files used for + communication between us and the build hook. */ + tmpDir = createTempDir(); + + /* Create the log file and pipe. */ + openLogFile(); + + /* Create the communication pipes. */ + toHook.create(); + fromHook.create(); + + /* Fork the hook. */ + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + initChild(); + + execl(buildHook.c_str(), buildHook.c_str(), + (worker.canBuildMore() ? (string) "1" : "0").c_str(), + thisSystem.c_str(), + drv.platform.c_str(), + drvPath.c_str(), 0); + + throw SysError(format("executing `%1%'") % buildHook); + + } catch (exception & e) { + cerr << format("build error: %1%\n") % e.what(); + } + _exit(1); + } + + /* parent */ + logPipe.writeSide.close(); + worker.childStarted(shared_from_this(), + pid, logPipe.readSide, false); + + fromHook.writeSide.close(); + toHook.readSide.close(); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. !!! potential + for deadlock here: we should also read from the child's logger + pipe. */ + string reply; + try { + reply = readLine(fromHook.readSide); + } catch (Error & e) { + drain(logPipe.readSide); + throw; + } + + debug(format("hook reply is `%1%'") % reply); + + if (reply == "decline" || reply == "postpone") { + /* Clean up the child. !!! hacky / should verify */ + drain(logPipe.readSide); + terminateBuildHook(); + return reply == "decline" ? rpDecline : rpPostpone; + } + + else if (reply == "accept") { + + /* Acquire locks and such. If we then see that the output + paths are now valid, we're done. */ + if (!prepareBuild()) { + /* Tell the hook to exit. */ + writeLine(toHook.writeSide, "cancel"); + terminateBuildHook(); + return rpDone; + } + + printMsg(lvlInfo, format("running hook to build path(s) %1%") + % showPaths(outputPaths(drv.outputs))); + + /* Write the information that the hook needs to perform the + build, i.e., the set of input paths (including closure + expressions), the set of output paths, and [!!!]. */ + + Path inputListFN = tmpDir + "/inputs"; + Path outputListFN = tmpDir + "/outputs"; + + string s; + for (PathSet::iterator i = inputPaths.begin(); + i != inputPaths.end(); ++i) + s += *i + "\n"; + writeStringToFile(inputListFN, s); + + s = ""; + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + s += i->second.path + "\n"; + writeStringToFile(outputListFN, s); + + /* Tell the hook to proceed. */ + writeLine(toHook.writeSide, "okay"); + + return rpAccept; + } + + else throw Error(format("bad hook reply `%1%'") % reply); +} + + +void DerivationGoal::terminateBuildHook() +{ + /* !!! drain stdout of hook */ + debug("terminating build hook"); + pid_t savedPid = pid; + pid.wait(true); + worker.childTerminated(savedPid, false); + fromHook.readSide.close(); + toHook.writeSide.close(); + fdLogFile.close(); + logPipe.readSide.close(); + deleteTmpDir(true); /* get rid of the hook's temporary directory */ +} + + +bool DerivationGoal::prepareBuild() +{ + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. */ + /* !!! BUG: this could block, which is not allowed. */ + outputLocks.lockPaths(outputPaths(drv.outputs)); + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this expression, so no further checks are necessary. */ + if (allOutputsValid()) { + debug(format("skipping build of derivation `%1%', someone beat us to it") + % drvPath); + outputLocks.setDeletion(true); + return false; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* The outputs are referenceable paths. */ + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + { + debug(format("building path `%1%'") % i->second.path); + allPaths.insert(i->second.path); + } + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + for (PathSet::iterator i = drv.inputDrvs.begin(); + i != drv.inputDrvs.end(); ++i) + { + /* Add all the output closures of the input derivation `*i' as + input paths. !!! there should be a way to indicate + specific outputs. */ + /* !!! is `*i' present? */ + assert(isValidPath(*i)); + Derivation inDrv = derivationFromPath(*i); + for (DerivationOutputs::iterator j = inDrv.outputs.begin(); + j != inDrv.outputs.end(); ++j) + computeFSClosure(j->second.path, inputPaths); + } + + for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i) + debug(format("INPUT %1%") % *i); + + allPaths.insert(inputPaths.begin(), inputPaths.end()); + + /* Second, the input sources. */ + for (PathSet::iterator i = drv.inputSrcs.begin(); + i != drv.inputSrcs.end(); ++i) + computeFSClosure(*i, inputPaths); + + /* We can skip running the builder if all output paths are already + valid. */ + bool fastBuild = true; + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + if (!isValidPath(i->second.path)) { + fastBuild = false; + break; + } + + if (fastBuild) { + printMsg(lvlChatty, format("skipping build; output paths already exist")); + computeClosure(); + return false; + } + + return true; +} + + +void DerivationGoal::startBuilder() +{ + startNest(nest, lvlInfo, + format("building path(s) %1%") % showPaths(outputPaths(drv.outputs))) + + /* Right platform? */ + if (drv.platform != thisSystem) + throw BuildError( + format("a `%1%' is required to build `%3%', but I am a `%2%'") + % drv.platform % thisSystem % drvPath); + + /* If any of the outputs already exist but are not registered, + delete them. */ + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + { + Path path = i->second.path; + if (isValidPath(path)) + throw Error(format("obstructed build: path `%1%' exists") % path); + if (pathExists(path)) { + debug(format("removing unregistered path `%1%'") % path); + deletePath(path); + } + } + + /* Construct the environment passed to the builder. */ + typedef map Environment; + Environment env; + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = "/homeless-shelter"; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = nixStore; + + /* Add all bindings specified in the derivation expression. */ + for (StringPairs::iterator i = drv.env.begin(); + i != drv.env.end(); ++i) + env[i->first] = i->second; + + /* Create a temporary directory where the build will take + place. */ + tmpDir = createTempDir(); + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDir; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; + + /* Run the builder. */ + printMsg(lvlChatty, format("executing builder `%1%'") % + drv.builder); + + /* Create the log file and pipe. */ + openLogFile(); + + /* Fork a child to build the package. Note that while we + currently use forks to run and wait for the children, it + shouldn't be hard to use threads for this on systems where + fork() is unavailable or inefficient. */ + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + + /* Warning: in the child we should absolutely not make any + Berkeley DB calls! */ + + try { /* child */ + + initChild(); + + /* Fill in the arguments. */ + Strings args(drv.args); + args.push_front(baseNameOf(drv.builder)); + const char * * argArr = strings2CharPtrs(args); + + /* Fill in the environment. */ + Strings envStrs; + for (Environment::const_iterator i = env.begin(); + i != env.end(); ++i) + envStrs.push_back(i->first + "=" + i->second); + const char * * envArr = strings2CharPtrs(envStrs); + + /* Execute the program. This should not return. */ + execve(drv.builder.c_str(), + (char * *) argArr, (char * *) envArr); + + throw SysError(format("executing `%1%'") + % drv.builder); + + } catch (exception & e) { + cerr << format("build error: %1%\n") % e.what(); + } + _exit(1); + } + + /* parent */ + pid.setSeparatePG(true); + logPipe.writeSide.close(); + worker.childStarted(shared_from_this(), + pid, logPipe.readSide, true); +} + + +void DerivationGoal::computeClosure() +{ + startNest(nest, lvlTalkative, + format("determining closure for `%1%'") % drvPath); + + map allReferences; + map contentHashes; + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + { + Path path = i->second.path; + if (!pathExists(path)) { + throw BuildError( + format("builder for `%1%' failed to produce output path `%2%'") + % drvPath % path); + } + + /* Check that fixed-output derivations produced the right + outputs (i.e., the content hash should match the specified + hash). */ + if (i->second.hash != "") { + HashType ht = parseHashType(i->second.hashAlgo); + if (ht == htUnknown) + throw Error(format("unknown hash algorithm `%1%'") % i->second.hashAlgo); + Hash h = parseHash(ht, i->second.hash); + Hash h2 = hashFile(ht, path); + if (h != h2) + throw Error( + format("output path `%1% should have %2% hash `%3%', instead has `%4%'") + % path % i->second.hashAlgo % printHash(h) % printHash(h2)); + + /* Also, the output path should be a regular file withouth + execute permission. */ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw Error( + format("output path `%1% should be a non-executable regular file") + % path); + } + + canonicalisePathMetaData(path); + + /* For this output path, find the references to other paths contained + in it. */ + PathSet references; + if (!pathExists(path + "/nix-support/no-scan")) { + startNest(nest2, lvlChatty, + format("scanning for store references in `%1%'") % path); + Paths references2; + references2 = filterReferences(path, + Paths(allPaths.begin(), allPaths.end())); + references = PathSet(references2.begin(), references2.end()); + + /* For debugging, print out the referenced and + unreferenced paths. */ + for (PathSet::iterator i = inputPaths.begin(); + i != inputPaths.end(); ++i) + { + PathSet::iterator j = references.find(*i); + if (j == references.end()) + debug(format("unreferenced input: `%1%'") % *i); + else + debug(format("referenced input: `%1%'") % *i); + } + + nest2.close(); + } + + allReferences[path] = references; + + /* Hash the contents of the path. The hash is stored in the + database so that we can verify later on whether nobody has + messed with the store. !!! inefficient: it would be nice + if we could combine this with filterReferences(). */ + contentHashes[path] = hashPath(htSHA256, path); + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. This is wrapped in one + database transaction to ensure that if we crash, either + everything is registered or nothing is. This is for + recoverability: unregistered paths in the store can be deleted + arbitrarily, while registered paths can only be deleted by + running the garbage collector. + + The reason that we do the transaction here and not on the fly + while we are scanning (above) is so that we don't hold database + locks for too long. */ + Transaction txn; + createStoreTransaction(txn); + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + { + registerValidPath(txn, i->second.path, + contentHashes[i->second.path]); + setReferences(txn, i->second.path, + allReferences[i->second.path]); + } + txn.commit(); + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will not + create new lock files with the same names as the old (unlinked) + lock files. */ + outputLocks.setDeletion(true); +} + + +void DerivationGoal::openLogFile() +{ + /* Create a log file. */ + Path logFileName = nixLogDir + "/" + baseNameOf(drvPath); + fdLogFile = open(logFileName.c_str(), + O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (fdLogFile == -1) + throw SysError(format("creating log file `%1%'") % logFileName); + + /* Create a pipe to get the output of the child. */ + logPipe.create(); +} + + +void DerivationGoal::initChild() +{ + commonChildInit(logPipe); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into `%1%'") % tmpDir); + + /* When running a hook, dup the communication pipes. */ + bool inHook = fromHook.writeSide.isOpen(); + if (inHook) { + fromHook.readSide.close(); + if (dup2(fromHook.writeSide, 3) == -1) + throw SysError("dupping from-hook write side"); + + toHook.writeSide.close(); + if (dup2(toHook.readSide, 4) == -1) + throw SysError("dupping to-hook read side"); + } + + /* Close all other file descriptors. */ + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO + && (!inHook || (fd != 3 && fd != 4))) + close(fd); /* ignore result */ +} + + +void DerivationGoal::deleteTmpDir(bool force) +{ + if (tmpDir != "") { + if (keepFailed && !force) + printMsg(lvlError, + format("builder for `%1%' failed; keeping build directory `%2%'") + % drvPath % tmpDir); + else + deletePath(tmpDir); + tmpDir = ""; + } +} + + +void DerivationGoal::writeLog(int fd, + const unsigned char * buf, size_t count) +{ + assert(fd == logPipe.readSide); + writeFull(fdLogFile, buf, count); +} + + +bool DerivationGoal::allOutputsValid() +{ + unsigned int nrValid = 0; + for (DerivationOutputs::iterator i = drv.outputs.begin(); + i != drv.outputs.end(); ++i) + if (isValidPath(i->second.path)) nrValid++; + + if (nrValid != 0) { + if (nrValid == drv.outputs.size()) return true; + throw Error( + format("derivation `%1%' is blocked by its output paths") + % drvPath); + } + + return false; +} + + +string DerivationGoal::name() +{ + return (format("building of `%1%'") % drvPath).str(); +} + + + +////////////////////////////////////////////////////////////////////// + + +class SubstitutionGoal : public Goal +{ +private: + /* The store path that should be realised through a substitute. */ + Path storePath; + + /* The remaining substitutes for this path. */ + Substitutes subs; + + /* The current substitute. */ + Substitute sub; + + /* Pipe for the substitute's standard output/error. */ + Pipe logPipe; + + /* The process ID of the builder. */ + Pid pid; + + /* Lock on the store path. */ + shared_ptr outputLock; + + typedef void (SubstitutionGoal::*GoalState)(); + GoalState state; + +public: + SubstitutionGoal(const Path & storePath, Worker & worker); + ~SubstitutionGoal(); + + void work(); + + /* The states. */ + void init(); + void tryNext(); + void tryToRun(); + void finished(); + + /* Callback used by the worker to write to the log. */ + void writeLog(int fd, const unsigned char * buf, size_t count); + + string name(); +}; + + +SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker) + : Goal(worker) +{ + this->storePath = storePath; + state = &SubstitutionGoal::init; +} + + +SubstitutionGoal::~SubstitutionGoal() +{ + if (pid != -1) worker.childTerminated(pid); +} + + +void SubstitutionGoal::work() +{ + (this->*state)(); +} + + +void SubstitutionGoal::init() +{ + trace("init"); + + /* If the path already exists we're done. */ + if (isValidPath(storePath)) { + amDone(); + return; + } + + /* !!! build the outgoing references of this path first to + maintain the closure invariant! */ + + /* Otherwise, get the substitutes. */ + subs = querySubstitutes(storePath); + + /* Try the first one. */ + tryNext(); +} + + +void SubstitutionGoal::tryNext() +{ + trace("trying next substitute"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + printMsg(lvlError, + format("path `%1%' is required, but it has no (remaining) substitutes") + % storePath); + amDone(false); + return; + } + sub = subs.front(); + subs.pop_front(); + + /* Wait until we can run the substitute program. */ + state = &SubstitutionGoal::tryToRun; + worker.waitForBuildSlot(shared_from_this()); +} + + +void SubstitutionGoal::tryToRun() +{ + trace("trying to run"); + + /* Make sure that we are allowed to start a build. */ + if (!worker.canBuildMore()) { + worker.waitForBuildSlot(shared_from_this()); + return; + } + + /* Acquire a lock on the output path. */ + PathSet lockPath; + lockPath.insert(storePath); + outputLock = shared_ptr(new PathLocks); + outputLock->lockPaths(lockPath); + + /* Check again whether the path is invalid. */ + if (isValidPath(storePath)) { + debug(format("store path `%1%' has become valid") % storePath); + outputLock->setDeletion(true); + amDone(); + return; + } + + printMsg(lvlInfo, + format("substituting path `%1%' using substituter `%2%'") + % storePath % sub.program); + + logPipe.create(); + + /* Remove the (stale) output path if it exists. */ + if (pathExists(storePath)) + deletePath(storePath); + + /* Fork the substitute program. */ + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + logPipe.readSide.close(); + + /* !!! close other handles */ + + commonChildInit(logPipe); + + /* Fill in the arguments. */ + Strings args(sub.args); + args.push_front(storePath); + args.push_front(baseNameOf(sub.program)); + const char * * argArr = strings2CharPtrs(args); + + execv(sub.program.c_str(), (char * *) argArr); + + throw SysError(format("executing `%1%'") % sub.program); + + } catch (exception & e) { + cerr << format("substitute error: %1%\n") % e.what(); + } + _exit(1); + } + + /* parent */ + pid.setSeparatePG(true); + logPipe.writeSide.close(); + worker.childStarted(shared_from_this(), + pid, logPipe.readSide, true); + + state = &SubstitutionGoal::finished; +} + + +void SubstitutionGoal::finished() +{ + trace("substitute finished"); + + /* Since we got an EOF on the logger pipe, the substitute is + presumed to have terminated. */ + /* !!! this could block! */ + pid_t savedPid = pid; + int status = pid.wait(true); + + /* So the child is gone now. */ + worker.childTerminated(savedPid); + + /* Close the read side of the logger pipe. */ + logPipe.readSide.close(); + + debug(format("substitute for `%1%' finished") % storePath); + + /* Check the exit status and the build result. */ + try { + + if (!statusOk(status)) + throw SubstError(format("builder for `%1%' %2%") + % storePath % statusToString(status)); + + if (!pathExists(storePath)) + throw SubstError( + format("substitute did not produce path `%1%'") + % storePath); + + } catch (SubstError & e) { + + printMsg(lvlInfo, + format("substitution of path `%1%' using substituter `%2%' failed: %3%") + % storePath % sub.program % e.msg()); + + /* Try the next substitute. */ + state = &SubstitutionGoal::tryNext; + worker.wakeUp(shared_from_this()); + return; + } + + canonicalisePathMetaData(storePath); + + Hash contentHash = hashPath(htSHA256, storePath); + + Transaction txn; + createStoreTransaction(txn); + registerValidPath(txn, storePath, contentHash); + txn.commit(); + + outputLock->setDeletion(true); + + printMsg(lvlChatty, + format("substitution of path `%1%' succeeded") % storePath); + + amDone(); +} + + +void SubstitutionGoal::writeLog(int fd, + const unsigned char * buf, size_t count) +{ + assert(fd == logPipe.readSide); + /* Don't write substitution output to a log file for now. We + probably should, though. */ +} + + +string SubstitutionGoal::name() +{ + return (format("substitution of `%1%'") % storePath).str(); +} + + + +////////////////////////////////////////////////////////////////////// + + +/* A fake goal used to receive notification of success or failure of + other goals. */ +class PseudoGoal : public Goal +{ +private: + bool success; + +public: + PseudoGoal(Worker & worker) : Goal(worker) + { + success = true; + } + + void work() + { + abort(); + } + + void waiteeDone(GoalPtr waitee, bool success) + { + if (!success) this->success = false; + } + + bool isOkay() + { + return success; + } + + string name() + { + return "pseudo-goal"; + } +}; + + + +////////////////////////////////////////////////////////////////////// + + +static bool working = false; + + +Worker::Worker() +{ + /* Debugging: prevent recursive workers. */ + if (working) abort(); + working = true; + nrChildren = 0; +} + + +Worker::~Worker() +{ + working = false; + + /* Explicitly get rid of all strong pointers now. After this all + goals that refer to this worker should be gone. (Otherwise we + are in trouble, since goals may call childTerminated() etc. in + their destructors). */ + topGoals.clear(); +} + + +template +static GoalPtr addGoal(const Path & path, + Worker & worker, WeakGoalMap & goalMap) +{ + GoalPtr goal = goalMap[path].lock(); + if (!goal) { + goal = GoalPtr(new T(path, worker)); + goalMap[path] = goal; + worker.wakeUp(goal); + } + return goal; +} + + +GoalPtr Worker::makeDerivationGoal(const Path & nePath) +{ + return addGoal(nePath, *this, derivationGoals); +} + + +GoalPtr Worker::makeSubstitutionGoal(const Path & storePath) +{ + return addGoal(storePath, *this, substitutionGoals); +} + + +static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +{ + /* !!! For now we just let dead goals accumulate. We should + probably periodically sweep the goalMap to remove dead + goals. */ +} + + +void Worker::removeGoal(GoalPtr goal) +{ + topGoals.erase(goal); + ::removeGoal(goal, derivationGoals); + ::removeGoal(goal, substitutionGoals); +} + + +void Worker::wakeUp(GoalPtr goal) +{ + goal->trace("woken up"); + awake.insert(goal); +} + + +bool Worker::canBuildMore() +{ + return nrChildren < maxBuildJobs; +} + + +void Worker::childStarted(GoalPtr goal, + pid_t pid, int fdOutput, bool inBuildSlot) +{ + Child child; + child.goal = goal; + child.fdOutput = fdOutput; + child.inBuildSlot = inBuildSlot; + children[pid] = child; + if (inBuildSlot) nrChildren++; +} + + +void Worker::childTerminated(pid_t pid, bool wakeSleepers) +{ + Children::iterator i = children.find(pid); + assert(i != children.end()); + + if (i->second.inBuildSlot) { + assert(nrChildren > 0); + nrChildren--; + } + + children.erase(pid); + + if (wakeSleepers) { + + /* Wake up goals waiting for a build slot. */ + for (WeakGoals::iterator i = wantingToBuild.begin(); + i != wantingToBuild.end(); ++i) + { + GoalPtr goal = i->lock(); + if (goal) wakeUp(goal); + } + + wantingToBuild.clear(); + + } +} + + +void Worker::waitForBuildSlot(GoalPtr goal, bool reallyWait) +{ + debug("wait for build slot"); + if (reallyWait && children.size() == 0) + throw Error("waiting for a build slot, yet there are no children - " + "maybe the build hook gave an inappropriate `postpone' reply?"); + if (!reallyWait && canBuildMore()) + wakeUp(goal); /* we can do it right away */ + else + wantingToBuild.insert(goal); +} + + +bool Worker::run(const Goals & _topGoals) +{ + /* Wrap the specified top-level goal in a pseudo-goal so that we + can check whether it succeeded. */ + shared_ptr pseudo(new PseudoGoal(*this)); + for (Goals::iterator i = _topGoals.begin(); + i != _topGoals.end(); ++i) + { + assert(*i); + pseudo->addWaitee(*i); + topGoals.insert(*i); + } + + startNest(nest, lvlDebug, format("entered goal loop")); + + while (1) { + + checkInterrupt(); + + /* Call every wake goal. */ + while (!awake.empty()) { + WeakGoals awake2(awake); + awake.clear(); + for (WeakGoals::iterator i = awake2.begin(); i != awake2.end(); ++i) { + checkInterrupt(); + GoalPtr goal = i->lock(); + if (goal) goal->work(); + } + } + + if (topGoals.empty()) break; + + /* !!! not when we're polling */ + assert(!children.empty()); + + /* Wait for input. */ + waitForInput(); + } + + /* If --keep-going is not set, it's possible that the main goal + exited while some of its subgoals were still active. But if + --keep-going *is* set, then they must all be finished now. */ + assert(!keepGoing || awake.empty()); + assert(!keepGoing || wantingToBuild.empty()); + assert(!keepGoing || children.empty()); + + return pseudo->isOkay(); +} + + +void Worker::waitForInput() +{ + printMsg(lvlVomit, "waiting for children"); + + /* Process log output from the children. We also use this to + detect child termination: if we get EOF on the logger pipe of a + build, we assume that the builder has terminated. */ + + /* Use select() to wait for the input side of any logger pipe to + become `available'. Note that `available' (i.e., non-blocking) + includes EOF. */ + fd_set fds; + FD_ZERO(&fds); + int fdMax = 0; + for (Children::iterator i = children.begin(); + i != children.end(); ++i) + { + int fd = i->second.fdOutput; + FD_SET(fd, &fds); + if (fd >= fdMax) fdMax = fd + 1; + } + + if (select(fdMax, &fds, 0, 0, 0) == -1) { + if (errno == EINTR) return; + throw SysError("waiting for input"); + } + + /* Process all available file descriptors. */ + for (Children::iterator i = children.begin(); + i != children.end(); ++i) + { + checkInterrupt(); + GoalPtr goal = i->second.goal.lock(); + assert(goal); + int fd = i->second.fdOutput; + if (FD_ISSET(fd, &fds)) { + unsigned char buffer[4096]; + ssize_t rd = read(fd, buffer, sizeof(buffer)); + if (rd == -1) { + if (errno != EINTR) + throw SysError(format("reading from %1%") + % goal->name()); + } else if (rd == 0) { + debug(format("%1%: got EOF") % goal->name()); + wakeUp(goal); + } else { + printMsg(lvlVomit, format("%1%: read %2% bytes") + % goal->name() % rd); + goal->writeLog(fd, buffer, (size_t) rd); + if (verbosity >= buildVerbosity) + writeFull(STDERR_FILENO, buffer, rd); + } + } + } +} + + +////////////////////////////////////////////////////////////////////// + + +void buildDerivations(const PathSet & drvPaths) +{ + startNest(nest, lvlDebug, + format("building %1%") % showPaths(drvPaths)); + + Worker worker; + + Goals goals; + for (PathSet::const_iterator i = drvPaths.begin(); + i != drvPaths.end(); ++i) + goals.insert(worker.makeDerivationGoal(*i)); + + if (!worker.run(goals)) + throw Error(format("build failed")); +} + + +void ensurePath(const Path & path) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + Worker worker; + Goals goals; + goals.insert(worker.makeSubstitutionGoal(path)); + if (!worker.run(goals)) + throw Error(format("path `%1%' does not exist and cannot be created") % path); +} diff --git a/src/libstore/build.hh b/src/libstore/build.hh new file mode 100644 index 000000000000..bfc0157330f7 --- /dev/null +++ b/src/libstore/build.hh @@ -0,0 +1,47 @@ +#ifndef __BUILD_H +#define __BUILD_H + +#include "derivations.hh" + +/* Perform the specified derivations, if necessary. That is, do + whatever is necessary to create the output paths of the derivation. + If the output paths already exists, we're done. If they have + substitutes, we can use those instead. Otherwise, the build action + described by the derivation is performed, after recursively + building any sub-derivations. */ +void buildDerivations(const PathSet & drvPaths); + +/* Ensure that a path exists, possibly by instantiating it by + realising a substitute. */ +void ensurePath(const Path & storePath); + +/* Read a derivation store expression, after ensuring its existence + through ensurePath(). */ +Derivation derivationFromPath(const Path & drvPath); + +/* Place in `paths' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly or + indirectly reached from it. `paths' is not cleared. */ +void computeFSClosure(const Path & storePath, + PathSet & paths); + +/* Place in `paths' the set of paths that are required to `realise' + the given store path, i.e., all paths necessary for valid + deployment of the path. For a derivation, this is the union of + requisites of the inputs, plus the derivation; for other store + paths, it is the set of paths in the FS closure of the path. If + `includeOutputs' is true, include the requisites of the output + paths of derivations as well. + + Note that this function can be used to implement three different + deployment policies: + + - Source deployment (when called on a derivation). + - Binary deployment (when called on an output path). + - Source/binary deployment (when called on a derivation with + `includeOutputs' set to true). +*/ +void storePathRequisites(const Path & storePath, + bool includeOutputs, PathSet & paths); + +#endif /* !__BUILD_H */ diff --git a/src/libstore/derivations-ast.def b/src/libstore/derivations-ast.def new file mode 100644 index 000000000000..2dce8de12c01 --- /dev/null +++ b/src/libstore/derivations-ast.def @@ -0,0 +1,6 @@ +init initDerivationsHelpers + +Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm | + +| string string | ATerm | EnvBinding | +| string string string string | ATerm | DerivationOutput | diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc new file mode 100644 index 000000000000..2f37c66fb60d --- /dev/null +++ b/src/libstore/derivations.cc @@ -0,0 +1,142 @@ +#include "derivations.hh" +#include "globals.hh" +#include "store.hh" + +#include "derivations-ast.hh" +#include "derivations-ast.cc" + + +Hash hashTerm(ATerm t) +{ + return hashString(htSHA256, atPrint(t)); +} + + +Path writeDerivation(const Derivation & drv, const string & name) +{ + return addTextToStore(name + drvExtension, + atPrint(unparseDerivation(drv))); +} + + +static void checkPath(const string & s) +{ + if (s.size() == 0 || s[0] != '/') + throw Error(format("bad path `%1%' in store expression") % s); +} + + +static void parsePaths(ATermList paths, PathSet & out) +{ + for (ATermIterator i(paths); i; ++i) { + if (ATgetType(*i) != AT_APPL) + throw badTerm("not a path", *i); + string s = aterm2String(*i); + checkPath(s); + out.insert(s); + } +} + + +void throwBadDrv(ATerm t) +{ + throw badTerm("not a valid derivation", t); +} + + +Derivation parseDerivation(ATerm t) +{ + Derivation drv; + ATermList outs, inDrvs, inSrcs, args, bnds; + ATerm builder, platform; + + if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds)) + throwBadDrv(t); + + for (ATermIterator i(outs); i; ++i) { + ATerm id, path, hashAlgo, hash; + if (!matchDerivationOutput(*i, id, path, hashAlgo, hash)) + throwBadDrv(t); + DerivationOutput out; + out.path = aterm2String(path); + checkPath(out.path); + out.hashAlgo = aterm2String(hashAlgo); + out.hash = aterm2String(hash); + drv.outputs[aterm2String(id)] = out; + } + + parsePaths(inDrvs, drv.inputDrvs); + parsePaths(inSrcs, drv.inputSrcs); + + drv.builder = aterm2String(builder); + drv.platform = aterm2String(platform); + + for (ATermIterator i(args); i; ++i) { + if (ATgetType(*i) != AT_APPL) + throw badTerm("string expected", *i); + drv.args.push_back(aterm2String(*i)); + } + + for (ATermIterator i(bnds); i; ++i) { + ATerm s1, s2; + if (!matchEnvBinding(*i, s1, s2)) + throw badTerm("tuple of strings expected", *i); + drv.env[aterm2String(s1)] = aterm2String(s2); + } + + return drv; +} + + +static ATermList unparsePaths(const PathSet & paths) +{ + ATermList l = ATempty; + for (PathSet::const_iterator i = paths.begin(); + i != paths.end(); i++) + l = ATinsert(l, toATerm(*i)); + return ATreverse(l); +} + + +ATerm unparseDerivation(const Derivation & drv) +{ + ATermList outputs = ATempty; + for (DerivationOutputs::const_iterator i = drv.outputs.begin(); + i != drv.outputs.end(); i++) + outputs = ATinsert(outputs, + makeDerivationOutput( + toATerm(i->first), + toATerm(i->second.path), + toATerm(i->second.hashAlgo), + toATerm(i->second.hash))); + + ATermList args = ATempty; + for (Strings::const_iterator i = drv.args.begin(); + i != drv.args.end(); i++) + args = ATinsert(args, toATerm(*i)); + + ATermList env = ATempty; + for (StringPairs::const_iterator i = drv.env.begin(); + i != drv.env.end(); i++) + env = ATinsert(env, + makeEnvBinding( + toATerm(i->first), + toATerm(i->second))); + + return makeDerive( + ATreverse(outputs), + unparsePaths(drv.inputDrvs), + unparsePaths(drv.inputSrcs), + toATerm(drv.platform), + toATerm(drv.builder), + ATreverse(args), + ATreverse(env)); +} + + +bool isDerivation(const string & fileName) +{ + return + fileName.size() >= drvExtension.size() && + string(fileName, fileName.size() - drvExtension.size()) == drvExtension; +} diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh new file mode 100644 index 000000000000..c264981d265c --- /dev/null +++ b/src/libstore/derivations.hh @@ -0,0 +1,62 @@ +#ifndef __DERIVATIONS_H +#define __DERIVATIONS_H + +#include "aterm.hh" +#include "store.hh" + + +/* Extension of derivations in the Nix store. */ +const string drvExtension = ".drv"; + + +/* Abstract syntax of derivations. */ + +struct DerivationOutput +{ + Path path; + string hashAlgo; /* hash used for expected hash computation */ + string hash; /* expected hash, may be null */ + DerivationOutput() + { + } + DerivationOutput(Path path, string hashAlgo, string hash) + { + this->path = path; + this->hashAlgo = hashAlgo; + this->hash = hash; + } +}; + +typedef map DerivationOutputs; +typedef map StringPairs; + +struct Derivation +{ + DerivationOutputs outputs; /* keyed on symbolic IDs */ + PathSet inputDrvs; /* inputs that are sub-derivations */ + PathSet inputSrcs; /* inputs that are sources */ + string platform; + Path builder; + Strings args; + StringPairs env; +}; + + +/* Hash an aterm. */ +Hash hashTerm(ATerm t); + +/* Write a derivation to the Nix store, and return its path. */ +Path writeDerivation(const Derivation & drv, const string & name); + +/* Parse a derivation. */ +Derivation parseDerivation(ATerm t); + +/* Parse a derivation. */ +ATerm unparseDerivation(const Derivation & drv); + +/* Check whether a file name ends with the extensions for + derivations. */ +bool isDerivation(const string & fileName); + + +#endif /* !__DERIVATIONS_H */ diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b6fb66d29cfe..4f3306505440 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,5 +1,5 @@ -#include "normalise.hh" #include "globals.hh" +#include "gc.hh" #include diff --git a/src/libstore/gc.hh b/src/libstore/gc.hh index 1ada419da439..d1ca5c63e4d0 100644 --- a/src/libstore/gc.hh +++ b/src/libstore/gc.hh @@ -1,7 +1,7 @@ #ifndef __GC_H #define __GC_H -#include "storeexpr.hh" +#include "util.hh" /* Determine the set of "live" store paths, given a set of root store diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2757a061ece2..c585a9aeac5c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,4 @@ -#include "normalise.hh" +#include "build.hh" Derivation derivationFromPath(const Path & drvPath) diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc deleted file mode 100644 index 090794ba57bb..000000000000 --- a/src/libstore/normalise.cc +++ /dev/null @@ -1,1804 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "normalise.hh" -#include "references.hh" -#include "pathlocks.hh" -#include "globals.hh" - - -/* !!! TODO storeExprFromPath shouldn't be used here */ - - -static string pathNullDevice = "/dev/null"; - - -/* Forward definition. */ -class Worker; - - -/* A pointer to a goal. */ -class Goal; -typedef shared_ptr GoalPtr; -typedef weak_ptr WeakGoalPtr; - -/* Set of goals. */ -typedef set Goals; -typedef set WeakGoals; - -/* A map of paths to goals (and the other way around). */ -typedef map WeakGoalMap; - - - -class Goal : public enable_shared_from_this -{ -protected: - - /* Backlink to the worker. */ - Worker & worker; - - /* Goals that this goal is waiting for. */ - Goals waitees; - - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ - WeakGoals waiters; - - /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; - - /* Whether amDone() has been called. */ - bool done; - - - Goal(Worker & worker) : worker(worker) - { - done = false; - nrFailed = 0; - } - - virtual ~Goal() - { - printMsg(lvlVomit, "goal destroyed"); - } - -public: - virtual void work() = 0; - - virtual string name() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, bool success); - - virtual void writeLog(int fd, const unsigned char * buf, size_t count) - { - abort(); - } - - void trace(const format & f); - -protected: - void amDone(bool success = true); -}; - - -/* A mapping used to remember for each child process to what goal it - belongs, and a file descriptor for receiving log data. */ -struct Child -{ - WeakGoalPtr goal; - int fdOutput; - bool inBuildSlot; -}; - -typedef map Children; - - -/* The worker class. */ -class Worker -{ -private: - - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /* The top-level goals of the worker. */ - Goals topGoals; - - /* Goals that are ready to do some work. */ - WeakGoals awake; - - /* Goals waiting for a build slot. */ - WeakGoals wantingToBuild; - - /* Child processes currently running. */ - Children children; - - /* Number of build slots occupied. Not all child processes - (namely build hooks) count as occupied build slots. */ - unsigned int nrChildren; - - /* Maps used to prevent multiple instantiations of a goal for the - same expression / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; - -public: - - Worker(); - ~Worker(); - - /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath); - GoalPtr makeSubstitutionGoal(const Path & storePath); - - /* Remove a dead goal. */ - void removeGoal(GoalPtr goal); - - /* Wake up a goal (i.e., there is something for it to do). */ - void wakeUp(GoalPtr goal); - - /* Can we start another child process? */ - bool canBuildMore(); - - /* Registers / unregisters a running child process. */ - void childStarted(GoalPtr goal, pid_t pid, int fdOutput, - bool inBuildSlot); - void childTerminated(pid_t pid, bool wakeSleepers = true); - - /* Add a goal to the set of goals waiting for a build slot. */ - void waitForBuildSlot(GoalPtr goal, bool reallyWait = false); - - /* Loop until the specified top-level goal has finished. Returns - true if it has finished succesfully. */ - bool run(const Goals & topGoals); - - /* Wait for input to become available. */ - void waitForInput(); -}; - - -class SubstError : public Error -{ -public: - SubstError(const format & f) : Error(f) { }; -}; - - -class BuildError : public Error -{ -public: - BuildError(const format & f) : Error(f) { }; -}; - - - -////////////////////////////////////////////////////////////////////// - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - waitee->waiters.insert(shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, bool success) -{ - assert(waitees.find(waitee) != waitees.end()); - waitees.erase(waitee); - - if (!success) ++nrFailed; - - if (waitees.empty() || (!success && !keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) { - GoalPtr goal = *i; - WeakGoals waiters2; - for (WeakGoals::iterator j = goal->waiters.begin(); - j != goal->waiters.end(); ++j) - if (j->lock() != shared_from_this()) - waiters2.insert(*j); - goal->waiters = waiters2; - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(bool success) -{ - trace("done"); - assert(!done); - done = true; - for (WeakGoals::iterator i = waiters.begin(); i != waiters.end(); ++i) { - GoalPtr goal = i->lock(); - if (goal) goal->waiteeDone(shared_from_this(), success); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); -} - - -void Goal::trace(const format & f) -{ - debug(format("%1%: %2%") % name() % f); -} - - - -////////////////////////////////////////////////////////////////////// - - -/* Common initialisation performed in child processes. */ -void commonChildInit(Pipe & logPipe) -{ - /* Put the child in a separate process group so that it doesn't - receive terminal signals. */ - if (setpgid(0, 0) == -1) - throw SysError(format("setting process group")); - - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - logPipe.readSide.close(); - - /* Dup stderr to stdin. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError(format("cannot open `%1%'") % pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); -} - - -/* Convert a string list to an array of char pointers. Careful: the - string list should outlive the array. */ -const char * * strings2CharPtrs(const Strings & ss) -{ - const char * * arr = new const char * [ss.size() + 1]; - const char * * p = arr; - for (Strings::const_iterator i = ss.begin(); i != ss.end(); ++i) - *p++ = i->c_str(); - *p = 0; - return arr; -} - - -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 0 (i.e., - 00:00:00 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - in a setuid Nix installation -*/ -void canonicalisePathMetaData(const Path & path) -{ - checkInterrupt(); - - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); - } - - if (st.st_uid != getuid() || st.st_gid != getgid()) { - if (chown(path.c_str(), getuid(), getgid()) == -1) - throw SysError(format("changing owner/group of `%1%' to %2%/%3%") - % path % getuid() % getgid()); - } - - if (st.st_mtime != 0) { - struct utimbuf utimbuf; - utimbuf.actime = st.st_atime; - utimbuf.modtime = 0; - if (utime(path.c_str(), &utimbuf) == -1) - throw SysError(format("changing modification time of `%1%'") % path); - } - - } - - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - for (Strings::iterator i = names.begin(); i != names.end(); ++i) - canonicalisePathMetaData(path + "/" + *i); - } -} - - - -////////////////////////////////////////////////////////////////////// - - -class DerivationGoal : public Goal -{ -private: - /* The path of the derivation store expression. */ - Path drvPath; - - /* The derivation store expression stored at drvPath. */ - Derivation drv; - - /* The remainder is state held during the build. */ - - /* Locks on the output paths. */ - PathLocks outputLocks; - - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ - PathSet inputPaths; - - /* Referenceable paths (i.e., input and output paths). */ - PathSet allPaths; - - /* The process ID of the builder. */ - Pid pid; - - /* The temporary directory. */ - Path tmpDir; - - /* File descriptor for the log file. */ - AutoCloseFD fdLogFile; - - /* Pipe for the builder's standard output/error. */ - Pipe logPipe; - - /* Pipes for talking to the build hook (if any). */ - Pipe toHook; - Pipe fromHook; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - -public: - DerivationGoal(const Path & drvPath, Worker & worker); - ~DerivationGoal(); - - void work(); - -private: - /* The states. */ - void init(); - void haveStoreExpr(); - void inputsRealised(); - void tryToBuild(); - void buildDone(); - - /* Is the build hook willing to perform the build? */ - typedef enum {rpAccept, rpDecline, rpPostpone, rpDone} HookReply; - HookReply tryBuildHook(); - - /* Synchronously wait for a build hook to finish. */ - void terminateBuildHook(); - - /* Acquires locks on the output paths and gathers information - about the build (e.g., the input closures). During this - process its possible that we find out that the build is - unnecessary, in which case we return false (this is not an - error condition!). */ - bool prepareBuild(); - - /* Start building a derivation. */ - void startBuilder(); - - /* Must be called after the output paths have become valid (either - due to a successful build or hook, or because they already - were). */ - void computeClosure(); - - /* Open a log file and a pipe to it. */ - void openLogFile(); - - /* Common initialisation to be performed in child processes (i.e., - both in builders and in build hooks. */ - void initChild(); - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Callback used by the worker to write to the log. */ - void writeLog(int fd, const unsigned char * buf, size_t count); - - /* Return true iff all output paths are valid. */ - bool allOutputsValid(); - - string name(); -}; - - -DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker) - : Goal(worker) -{ - this->drvPath = drvPath; - state = &DerivationGoal::init; -} - - -DerivationGoal::~DerivationGoal() -{ - if (pid != -1) worker.childTerminated(pid); - - /* Careful: we should never ever throw an exception from a - destructor. */ - try { - deleteTmpDir(false); - } catch (Error & e) { - printMsg(lvlError, format("error (ignored): %1%") % e.msg()); - } -} - - -void DerivationGoal::work() -{ - (this->*state)(); -} - - -void DerivationGoal::init() -{ - trace("init"); - - /* The first thing to do is to make sure that the store expression - exists. If it doesn't, it may be created through a - substitute. */ - addWaitee(worker.makeSubstitutionGoal(drvPath)); - - state = &DerivationGoal::haveStoreExpr; -} - - -void DerivationGoal::haveStoreExpr() -{ - trace("loading derivation"); - - if (nrFailed != 0) { - printMsg(lvlError, - format("cannot build missing derivation `%1%'") - % drvPath); - amDone(false); - return; - } - - assert(isValidPath(drvPath)); - - /* Get the derivation. */ - drv = derivationFromPath(drvPath); - - /* If all the outputs already exist, then we're done. */ - if (allOutputsValid()) { - amDone(true); - return; - } - - /* Inputs must be built before we can build this goal. */ - for (PathSet::iterator i = drv.inputDrvs.begin(); - i != drv.inputDrvs.end(); ++i) - addWaitee(worker.makeDerivationGoal(*i)); - - for (PathSet::iterator i = drv.inputSrcs.begin(); - i != drv.inputSrcs.end(); ++i) - addWaitee(worker.makeSubstitutionGoal(*i)); - - state = &DerivationGoal::inputsRealised; -} - - -void DerivationGoal::inputsRealised() -{ - trace("all inputs realised"); - - if (nrFailed != 0) { - printMsg(lvlError, - format("cannot build derivation `%1%': " - "%2% inputs could not be realised") - % drvPath % nrFailed); - amDone(false); - return; - } - - /* Okay, try to build. Note that here we don't wait for a build - slot to become available, since we don't need one if there is a - build hook. */ - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); -} - - -void DerivationGoal::tryToBuild() -{ - trace("trying to build"); - - try { - - /* Is the build hook willing to accept this job? */ - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - state = &DerivationGoal::buildDone; - return; - case rpPostpone: - /* Not now; wait until at least one child finishes. */ - worker.waitForBuildSlot(shared_from_this(), true); - return; - case rpDecline: - /* We should do it ourselves. */ - break; - case rpDone: - /* Somebody else did it. */ - amDone(); - return; - } - - /* Make sure that we are allowed to start a build. */ - if (!worker.canBuildMore()) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - /* Acquire locks and such. If we then see that the build has - been done by somebody else, we're done. */ - if (!prepareBuild()) { - amDone(); - return; - } - - /* Okay, we have to build. */ - startBuilder(); - - } catch (BuildError & e) { - printMsg(lvlError, e.msg()); - amDone(false); - return; - } - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; -} - - -void DerivationGoal::buildDone() -{ - trace("build done"); - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe --- just don't do that - :-) */ - /* !!! this could block! */ - pid_t savedPid = pid; - int status = pid.wait(true); - - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); - - /* Close the log file. */ - fdLogFile.close(); - - debug(format("builder process for `%1%' finished") % drvPath); - - /* Check the exit status. */ - if (!statusOk(status)) { - deleteTmpDir(false); - printMsg(lvlError, format("builder for `%1%' %2%") - % drvPath % statusToString(status)); - amDone(false); - return; - } - - deleteTmpDir(true); - - /* Compute the FS closure of the outputs and register them as - being valid. */ - try { - computeClosure(); - } catch (BuildError & e) { - printMsg(lvlError, e.msg()); - amDone(false); - return; - } - - amDone(); -} - - -static string readLine(int fd) -{ - string s; - while (1) { - char ch; - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw Error("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -static void writeLine(int fd, string s) -{ - s += '\n'; - writeFull(fd, (const unsigned char *) s.c_str(), s.size()); -} - - -/* !!! ugly hack */ -static void drain(int fd) -{ - unsigned char buffer[1024]; - while (1) { - ssize_t rd = read(fd, buffer, sizeof buffer); - if (rd == -1) { - if (errno != EINTR) - throw SysError("draining"); - } else if (rd == 0) break; - else writeFull(STDERR_FILENO, buffer, rd); - } -} - - -PathSet outputPaths(const DerivationOutputs & outputs) -{ - PathSet paths; - for (DerivationOutputs::const_iterator i = outputs.begin(); - i != outputs.end(); ++i) - paths.insert(i->second.path); - return paths; -} - - -string showPaths(const PathSet & paths) -{ - string s; - for (PathSet::const_iterator i = paths.begin(); - i != paths.end(); ++i) - { - if (s.size() != 0) s += ", "; - s += "`" + *i + "'"; - } - return s; -} - - -DerivationGoal::HookReply DerivationGoal::tryBuildHook() -{ - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (buildHook == "") return rpDecline; - buildHook = absPath(buildHook); - - /* Create a directory where we will store files used for - communication between us and the build hook. */ - tmpDir = createTempDir(); - - /* Create the log file and pipe. */ - openLogFile(); - - /* Create the communication pipes. */ - toHook.create(); - fromHook.create(); - - /* Fork the hook. */ - pid = fork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - initChild(); - - execl(buildHook.c_str(), buildHook.c_str(), - (worker.canBuildMore() ? (string) "1" : "0").c_str(), - thisSystem.c_str(), - drv.platform.c_str(), - drvPath.c_str(), 0); - - throw SysError(format("executing `%1%'") % buildHook); - - } catch (exception & e) { - cerr << format("build error: %1%\n") % e.what(); - } - _exit(1); - } - - /* parent */ - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, logPipe.readSide, false); - - fromHook.writeSide.close(); - toHook.readSide.close(); - - /* Read the first line of input, which should be a word indicating - whether the hook wishes to perform the build. !!! potential - for deadlock here: we should also read from the child's logger - pipe. */ - string reply; - try { - reply = readLine(fromHook.readSide); - } catch (Error & e) { - drain(logPipe.readSide); - throw; - } - - debug(format("hook reply is `%1%'") % reply); - - if (reply == "decline" || reply == "postpone") { - /* Clean up the child. !!! hacky / should verify */ - drain(logPipe.readSide); - terminateBuildHook(); - return reply == "decline" ? rpDecline : rpPostpone; - } - - else if (reply == "accept") { - - /* Acquire locks and such. If we then see that the output - paths are now valid, we're done. */ - if (!prepareBuild()) { - /* Tell the hook to exit. */ - writeLine(toHook.writeSide, "cancel"); - terminateBuildHook(); - return rpDone; - } - - printMsg(lvlInfo, format("running hook to build path(s) %1%") - % showPaths(outputPaths(drv.outputs))); - - /* Write the information that the hook needs to perform the - build, i.e., the set of input paths (including closure - expressions), the set of output paths, and [!!!]. */ - - Path inputListFN = tmpDir + "/inputs"; - Path outputListFN = tmpDir + "/outputs"; - - string s; - for (PathSet::iterator i = inputPaths.begin(); - i != inputPaths.end(); ++i) - s += *i + "\n"; - writeStringToFile(inputListFN, s); - - s = ""; - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - s += i->second.path + "\n"; - writeStringToFile(outputListFN, s); - - /* Tell the hook to proceed. */ - writeLine(toHook.writeSide, "okay"); - - return rpAccept; - } - - else throw Error(format("bad hook reply `%1%'") % reply); -} - - -void DerivationGoal::terminateBuildHook() -{ - /* !!! drain stdout of hook */ - debug("terminating build hook"); - pid_t savedPid = pid; - pid.wait(true); - worker.childTerminated(savedPid, false); - fromHook.readSide.close(); - toHook.writeSide.close(); - fdLogFile.close(); - logPipe.readSide.close(); - deleteTmpDir(true); /* get rid of the hook's temporary directory */ -} - - -bool DerivationGoal::prepareBuild() -{ - /* Obtain locks on all output paths. The locks are automatically - released when we exit this function or Nix crashes. */ - /* !!! BUG: this could block, which is not allowed. */ - outputLocks.lockPaths(outputPaths(drv.outputs)); - - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this expression, so no further checks are necessary. */ - if (allOutputsValid()) { - debug(format("skipping build of derivation `%1%', someone beat us to it") - % drvPath); - outputLocks.setDeletion(true); - return false; - } - - /* Gather information necessary for computing the closure and/or - running the build hook. */ - - /* The outputs are referenceable paths. */ - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - { - debug(format("building path `%1%'") % i->second.path); - allPaths.insert(i->second.path); - } - - /* Determine the full set of input paths. */ - - /* First, the input derivations. */ - for (PathSet::iterator i = drv.inputDrvs.begin(); - i != drv.inputDrvs.end(); ++i) - { - /* Add all the output closures of the input derivation `*i' as - input paths. !!! there should be a way to indicate - specific outputs. */ - /* !!! is `*i' present? */ - assert(isValidPath(*i)); - Derivation inDrv = derivationFromPath(*i); - for (DerivationOutputs::iterator j = inDrv.outputs.begin(); - j != inDrv.outputs.end(); ++j) - computeFSClosure(j->second.path, inputPaths); - } - - for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i) - debug(format("INPUT %1%") % *i); - - allPaths.insert(inputPaths.begin(), inputPaths.end()); - - /* Second, the input sources. */ - for (PathSet::iterator i = drv.inputSrcs.begin(); - i != drv.inputSrcs.end(); ++i) - computeFSClosure(*i, inputPaths); - - /* We can skip running the builder if all output paths are already - valid. */ - bool fastBuild = true; - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - if (!isValidPath(i->second.path)) { - fastBuild = false; - break; - } - - if (fastBuild) { - printMsg(lvlChatty, format("skipping build; output paths already exist")); - computeClosure(); - return false; - } - - return true; -} - - -void DerivationGoal::startBuilder() -{ - startNest(nest, lvlInfo, - format("building path(s) %1%") % showPaths(outputPaths(drv.outputs))) - - /* Right platform? */ - if (drv.platform != thisSystem) - throw BuildError( - format("a `%1%' is required to build `%3%', but I am a `%2%'") - % drv.platform % thisSystem % drvPath); - - /* If any of the outputs already exist but are not registered, - delete them. */ - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - { - Path path = i->second.path; - if (isValidPath(path)) - throw Error(format("obstructed build: path `%1%' exists") % path); - if (pathExists(path)) { - debug(format("removing unregistered path `%1%'") % path); - deletePath(path); - } - } - - /* Construct the environment passed to the builder. */ - typedef map Environment; - Environment env; - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = "/homeless-shelter"; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = nixStore; - - /* Add all bindings specified in the derivation expression. */ - for (StringPairs::iterator i = drv.env.begin(); - i != drv.env.end(); ++i) - env[i->first] = i->second; - - /* Create a temporary directory where the build will take - place. */ - tmpDir = createTempDir(); - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDir; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; - - /* Run the builder. */ - printMsg(lvlChatty, format("executing builder `%1%'") % - drv.builder); - - /* Create the log file and pipe. */ - openLogFile(); - - /* Fork a child to build the package. Note that while we - currently use forks to run and wait for the children, it - shouldn't be hard to use threads for this on systems where - fork() is unavailable or inefficient. */ - pid = fork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - - /* Warning: in the child we should absolutely not make any - Berkeley DB calls! */ - - try { /* child */ - - initChild(); - - /* Fill in the arguments. */ - Strings args(drv.args); - args.push_front(baseNameOf(drv.builder)); - const char * * argArr = strings2CharPtrs(args); - - /* Fill in the environment. */ - Strings envStrs; - for (Environment::const_iterator i = env.begin(); - i != env.end(); ++i) - envStrs.push_back(i->first + "=" + i->second); - const char * * envArr = strings2CharPtrs(envStrs); - - /* Execute the program. This should not return. */ - execve(drv.builder.c_str(), - (char * *) argArr, (char * *) envArr); - - throw SysError(format("executing `%1%'") - % drv.builder); - - } catch (exception & e) { - cerr << format("build error: %1%\n") % e.what(); - } - _exit(1); - } - - /* parent */ - pid.setSeparatePG(true); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, logPipe.readSide, true); -} - - -void DerivationGoal::computeClosure() -{ - startNest(nest, lvlTalkative, - format("determining closure for `%1%'") % drvPath); - - map allReferences; - - /* Check whether the output paths were created, and grep each - output path to determine what other paths it references. Also make all - output paths read-only. */ - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - { - Path path = i->second.path; - if (!pathExists(path)) { - throw BuildError( - format("builder for `%1%' failed to produce output path `%2%'") - % drvPath % path); - } - - /* Check that fixed-output derivations produced the right - outputs (i.e., the content hash should match the specified - hash). */ - if (i->second.hash != "") { - HashType ht = parseHashType(i->second.hashAlgo); - if (ht == htUnknown) - throw Error(format("unknown hash algorithm `%1%'") % i->second.hashAlgo); - Hash h = parseHash(ht, i->second.hash); - Hash h2 = hashFile(ht, path); - if (h != h2) - throw Error( - format("output path `%1% should have %2% hash `%3%', instead has `%4%'") - % path % i->second.hashAlgo % printHash(h) % printHash(h2)); - - /* Also, the output path should be a regular file withouth - execute permission. */ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) - throw Error( - format("output path `%1% should be a non-executable regular file") - % path); - } - - canonicalisePathMetaData(path); - - /* For this output path, find the references to other paths contained - in it. */ - PathSet references; - if (!pathExists(path + "/nix-support/no-scan")) { - startNest(nest2, lvlChatty, - format("scanning for store references in `%1%'") % path); - Paths references2; - references2 = filterReferences(path, - Paths(allPaths.begin(), allPaths.end())); - references = PathSet(references2.begin(), references2.end()); - - /* For debugging, print out the referenced and - unreferenced paths. */ - for (PathSet::iterator i = inputPaths.begin(); - i != inputPaths.end(); ++i) - { - PathSet::iterator j = references.find(*i); - if (j == references.end()) - debug(format("unreferenced input: `%1%'") % *i); - else - debug(format("referenced input: `%1%'") % *i); - } - - nest2.close(); - } - - allReferences[path] = references; - } - - /* Register each output path as valid, and register the sets of - paths referenced by each of them. This is wrapped in one - database transaction to ensure that if we crash, either - everything is registered or nothing is. This is for - recoverability: unregistered paths in the store can be deleted - arbitrarily, while registered paths can only be deleted by - running the garbage collector. - - The reason that we do the transaction here and not on the fly - while we are scanning (above) is so that we don't hold database - locks for too long. */ - Transaction txn; - createStoreTransaction(txn); - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - { - registerValidPath(txn, i->second.path); - setReferences(txn, i->second.path, - allReferences[i->second.path]); - } - txn.commit(); - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will not - create new lock files with the same names as the old (unlinked) - lock files. */ - outputLocks.setDeletion(true); -} - - -void DerivationGoal::openLogFile() -{ - /* Create a log file. */ - Path logFileName = nixLogDir + "/" + baseNameOf(drvPath); - fdLogFile = open(logFileName.c_str(), - O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fdLogFile == -1) - throw SysError(format("creating log file `%1%'") % logFileName); - - /* Create a pipe to get the output of the child. */ - logPipe.create(); -} - - -void DerivationGoal::initChild() -{ - commonChildInit(logPipe); - - if (chdir(tmpDir.c_str()) == -1) - throw SysError(format("changing into `%1%'") % tmpDir); - - /* When running a hook, dup the communication pipes. */ - bool inHook = fromHook.writeSide.isOpen(); - if (inHook) { - fromHook.readSide.close(); - if (dup2(fromHook.writeSide, 3) == -1) - throw SysError("dupping from-hook write side"); - - toHook.writeSide.close(); - if (dup2(toHook.readSide, 4) == -1) - throw SysError("dupping to-hook read side"); - } - - /* Close all other file descriptors. */ - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO - && (!inHook || (fd != 3 && fd != 4))) - close(fd); /* ignore result */ -} - - -void DerivationGoal::deleteTmpDir(bool force) -{ - if (tmpDir != "") { - if (keepFailed && !force) - printMsg(lvlError, - format("builder for `%1%' failed; keeping build directory `%2%'") - % drvPath % tmpDir); - else - deletePath(tmpDir); - tmpDir = ""; - } -} - - -void DerivationGoal::writeLog(int fd, - const unsigned char * buf, size_t count) -{ - assert(fd == logPipe.readSide); - writeFull(fdLogFile, buf, count); -} - - -bool DerivationGoal::allOutputsValid() -{ - unsigned int nrValid = 0; - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - if (isValidPath(i->second.path)) nrValid++; - - if (nrValid != 0) { - if (nrValid == drv.outputs.size()) return true; - throw Error( - format("derivation `%1%' is blocked by its output paths") - % drvPath); - } - - return false; -} - - -string DerivationGoal::name() -{ - return (format("building of `%1%'") % drvPath).str(); -} - - - -////////////////////////////////////////////////////////////////////// - - -class SubstitutionGoal : public Goal -{ -private: - /* The store path that should be realised through a substitute. */ - Path storePath; - - /* The remaining substitutes for this path. */ - Substitutes subs; - - /* The current substitute. */ - Substitute sub; - - /* Pipe for the substitute's standard output/error. */ - Pipe logPipe; - - /* The process ID of the builder. */ - Pid pid; - - /* Lock on the store path. */ - shared_ptr outputLock; - - typedef void (SubstitutionGoal::*GoalState)(); - GoalState state; - -public: - SubstitutionGoal(const Path & storePath, Worker & worker); - ~SubstitutionGoal(); - - void work(); - - /* The states. */ - void init(); - void tryNext(); - void tryToRun(); - void finished(); - - /* Callback used by the worker to write to the log. */ - void writeLog(int fd, const unsigned char * buf, size_t count); - - string name(); -}; - - -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker) - : Goal(worker) -{ - this->storePath = storePath; - state = &SubstitutionGoal::init; -} - - -SubstitutionGoal::~SubstitutionGoal() -{ - if (pid != -1) worker.childTerminated(pid); -} - - -void SubstitutionGoal::work() -{ - (this->*state)(); -} - - -void SubstitutionGoal::init() -{ - trace("init"); - - /* If the path already exists we're done. */ - if (isValidPath(storePath)) { - amDone(); - return; - } - - /* !!! build the outgoing references of this path first to - maintain the closure invariant! */ - - /* Otherwise, get the substitutes. */ - subs = querySubstitutes(storePath); - - /* Try the first one. */ - tryNext(); -} - - -void SubstitutionGoal::tryNext() -{ - trace("trying next substitute"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - printMsg(lvlError, - format("path `%1%' is required, but it has no (remaining) substitutes") - % storePath); - amDone(false); - return; - } - sub = subs.front(); - subs.pop_front(); - - /* Wait until we can run the substitute program. */ - state = &SubstitutionGoal::tryToRun; - worker.waitForBuildSlot(shared_from_this()); -} - - -void SubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a build. */ - if (!worker.canBuildMore()) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - /* Acquire a lock on the output path. */ - PathSet lockPath; - lockPath.insert(storePath); - outputLock = shared_ptr(new PathLocks); - outputLock->lockPaths(lockPath); - - /* Check again whether the path is invalid. */ - if (isValidPath(storePath)) { - debug(format("store path `%1%' has become valid") % storePath); - outputLock->setDeletion(true); - amDone(); - return; - } - - printMsg(lvlInfo, - format("substituting path `%1%' using substituter `%2%'") - % storePath % sub.program); - - logPipe.create(); - - /* Remove the (stale) output path if it exists. */ - if (pathExists(storePath)) - deletePath(storePath); - - /* Fork the substitute program. */ - pid = fork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - logPipe.readSide.close(); - - /* !!! close other handles */ - - commonChildInit(logPipe); - - /* Fill in the arguments. */ - Strings args(sub.args); - args.push_front(storePath); - args.push_front(baseNameOf(sub.program)); - const char * * argArr = strings2CharPtrs(args); - - execv(sub.program.c_str(), (char * *) argArr); - - throw SysError(format("executing `%1%'") % sub.program); - - } catch (exception & e) { - cerr << format("substitute error: %1%\n") % e.what(); - } - _exit(1); - } - - /* parent */ - pid.setSeparatePG(true); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, logPipe.readSide, true); - - state = &SubstitutionGoal::finished; -} - - -void SubstitutionGoal::finished() -{ - trace("substitute finished"); - - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - /* !!! this could block! */ - pid_t savedPid = pid; - int status = pid.wait(true); - - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); - - debug(format("substitute for `%1%' finished") % storePath); - - /* Check the exit status and the build result. */ - try { - - if (!statusOk(status)) - throw SubstError(format("builder for `%1%' %2%") - % storePath % statusToString(status)); - - if (!pathExists(storePath)) - throw SubstError( - format("substitute did not produce path `%1%'") - % storePath); - - } catch (SubstError & e) { - - printMsg(lvlInfo, - format("substitution of path `%1%' using substituter `%2%' failed: %3%") - % storePath % sub.program % e.msg()); - - /* Try the next substitute. */ - state = &SubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - canonicalisePathMetaData(storePath); - - Transaction txn; - createStoreTransaction(txn); - registerValidPath(txn, storePath); - txn.commit(); - - outputLock->setDeletion(true); - - printMsg(lvlChatty, - format("substitution of path `%1%' succeeded") % storePath); - - amDone(); -} - - -void SubstitutionGoal::writeLog(int fd, - const unsigned char * buf, size_t count) -{ - assert(fd == logPipe.readSide); - /* Don't write substitution output to a log file for now. We - probably should, though. */ -} - - -string SubstitutionGoal::name() -{ - return (format("substitution of `%1%'") % storePath).str(); -} - - - -////////////////////////////////////////////////////////////////////// - - -/* A fake goal used to receive notification of success or failure of - other goals. */ -class PseudoGoal : public Goal -{ -private: - bool success; - -public: - PseudoGoal(Worker & worker) : Goal(worker) - { - success = true; - } - - void work() - { - abort(); - } - - void waiteeDone(GoalPtr waitee, bool success) - { - if (!success) this->success = false; - } - - bool isOkay() - { - return success; - } - - string name() - { - return "pseudo-goal"; - } -}; - - - -////////////////////////////////////////////////////////////////////// - - -static bool working = false; - - -Worker::Worker() -{ - /* Debugging: prevent recursive workers. */ - if (working) abort(); - working = true; - nrChildren = 0; -} - - -Worker::~Worker() -{ - working = false; - - /* Explicitly get rid of all strong pointers now. After this all - goals that refer to this worker should be gone. (Otherwise we - are in trouble, since goals may call childTerminated() etc. in - their destructors). */ - topGoals.clear(); -} - - -template -static GoalPtr addGoal(const Path & path, - Worker & worker, WeakGoalMap & goalMap) -{ - GoalPtr goal = goalMap[path].lock(); - if (!goal) { - goal = GoalPtr(new T(path, worker)); - goalMap[path] = goal; - worker.wakeUp(goal); - } - return goal; -} - - -GoalPtr Worker::makeDerivationGoal(const Path & nePath) -{ - return addGoal(nePath, *this, derivationGoals); -} - - -GoalPtr Worker::makeSubstitutionGoal(const Path & storePath) -{ - return addGoal(storePath, *this, substitutionGoals); -} - - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) -{ - /* !!! For now we just let dead goals accumulate. We should - probably periodically sweep the goalMap to remove dead - goals. */ -} - - -void Worker::removeGoal(GoalPtr goal) -{ - topGoals.erase(goal); - ::removeGoal(goal, derivationGoals); - ::removeGoal(goal, substitutionGoals); -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - awake.insert(goal); -} - - -bool Worker::canBuildMore() -{ - return nrChildren < maxBuildJobs; -} - - -void Worker::childStarted(GoalPtr goal, - pid_t pid, int fdOutput, bool inBuildSlot) -{ - Child child; - child.goal = goal; - child.fdOutput = fdOutput; - child.inBuildSlot = inBuildSlot; - children[pid] = child; - if (inBuildSlot) nrChildren++; -} - - -void Worker::childTerminated(pid_t pid, bool wakeSleepers) -{ - Children::iterator i = children.find(pid); - assert(i != children.end()); - - if (i->second.inBuildSlot) { - assert(nrChildren > 0); - nrChildren--; - } - - children.erase(pid); - - if (wakeSleepers) { - - /* Wake up goals waiting for a build slot. */ - for (WeakGoals::iterator i = wantingToBuild.begin(); - i != wantingToBuild.end(); ++i) - { - GoalPtr goal = i->lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); - - } -} - - -void Worker::waitForBuildSlot(GoalPtr goal, bool reallyWait) -{ - debug("wait for build slot"); - if (reallyWait && children.size() == 0) - throw Error("waiting for a build slot, yet there are no children - " - "maybe the build hook gave an inappropriate `postpone' reply?"); - if (!reallyWait && canBuildMore()) - wakeUp(goal); /* we can do it right away */ - else - wantingToBuild.insert(goal); -} - - -bool Worker::run(const Goals & _topGoals) -{ - /* Wrap the specified top-level goal in a pseudo-goal so that we - can check whether it succeeded. */ - shared_ptr pseudo(new PseudoGoal(*this)); - for (Goals::iterator i = _topGoals.begin(); - i != _topGoals.end(); ++i) - { - assert(*i); - pseudo->addWaitee(*i); - topGoals.insert(*i); - } - - startNest(nest, lvlDebug, format("entered goal loop")); - - while (1) { - - checkInterrupt(); - - /* Call every wake goal. */ - while (!awake.empty()) { - WeakGoals awake2(awake); - awake.clear(); - for (WeakGoals::iterator i = awake2.begin(); i != awake2.end(); ++i) { - checkInterrupt(); - GoalPtr goal = i->lock(); - if (goal) goal->work(); - } - } - - if (topGoals.empty()) break; - - /* !!! not when we're polling */ - assert(!children.empty()); - - /* Wait for input. */ - waitForInput(); - } - - /* If --keep-going is not set, it's possible that the main goal - exited while some of its subgoals were still active. But if - --keep-going *is* set, then they must all be finished now. */ - assert(!keepGoing || awake.empty()); - assert(!keepGoing || wantingToBuild.empty()); - assert(!keepGoing || children.empty()); - - return pseudo->isOkay(); -} - - -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process log output from the children. We also use this to - detect child termination: if we get EOF on the logger pipe of a - build, we assume that the builder has terminated. */ - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; - for (Children::iterator i = children.begin(); - i != children.end(); ++i) - { - int fd = i->second.fdOutput; - FD_SET(fd, &fds); - if (fd >= fdMax) fdMax = fd + 1; - } - - if (select(fdMax, &fds, 0, 0, 0) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - /* Process all available file descriptors. */ - for (Children::iterator i = children.begin(); - i != children.end(); ++i) - { - checkInterrupt(); - GoalPtr goal = i->second.goal.lock(); - assert(goal); - int fd = i->second.fdOutput; - if (FD_ISSET(fd, &fds)) { - unsigned char buffer[4096]; - ssize_t rd = read(fd, buffer, sizeof(buffer)); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->name()); - } else if (rd == 0) { - debug(format("%1%: got EOF") % goal->name()); - wakeUp(goal); - } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->name() % rd); - goal->writeLog(fd, buffer, (size_t) rd); - if (verbosity >= buildVerbosity) - writeFull(STDERR_FILENO, buffer, rd); - } - } - } -} - - -////////////////////////////////////////////////////////////////////// - - -void buildDerivations(const PathSet & drvPaths) -{ - startNest(nest, lvlDebug, - format("building %1%") % showPaths(drvPaths)); - - Worker worker; - - Goals goals; - for (PathSet::const_iterator i = drvPaths.begin(); - i != drvPaths.end(); ++i) - goals.insert(worker.makeDerivationGoal(*i)); - - if (!worker.run(goals)) - throw Error(format("build failed")); -} - - -void ensurePath(const Path & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - Worker worker; - Goals goals; - goals.insert(worker.makeSubstitutionGoal(path)); - if (!worker.run(goals)) - throw Error(format("path `%1%' does not exist and cannot be created") % path); -} diff --git a/src/libstore/normalise.hh b/src/libstore/normalise.hh deleted file mode 100644 index 96f546aaaa07..000000000000 --- a/src/libstore/normalise.hh +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef __NORMALISE_H -#define __NORMALISE_H - -#include "storeexpr.hh" - -/* Perform the specified derivations, if necessary. That is, do - whatever is necessary to create the output paths of the derivation. - If the output paths already exists, we're done. If they have - substitutes, we can use those instead. Otherwise, the build action - described by the derivation is performed, after recursively - building any sub-derivations. */ -void buildDerivations(const PathSet & drvPaths); - -/* Ensure that a path exists, possibly by instantiating it by - realising a substitute. */ -void ensurePath(const Path & storePath); - -/* Read a derivation store expression, after ensuring its existence - through ensurePath(). */ -Derivation derivationFromPath(const Path & drvPath); - -/* Place in `paths' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly or - indirectly reached from it. `paths' is not cleared. */ -void computeFSClosure(const Path & storePath, - PathSet & paths); - -/* Place in `paths' the set of paths that are required to `realise' - the given store path, i.e., all paths necessary for valid - deployment of the path. For a derivation, this is the union of - requisites of the inputs, plus the derivation; for other store - paths, it is the set of paths in the FS closure of the path. If - `includeOutputs' is true, include the requisites of the output - paths of derivations as well. - - Note that this function can be used to implement three different - deployment policies: - - - Source deployment (when called on a derivation). - - Binary deployment (when called on an output path). - - Source/binary deployment (when called on a derivation with - `includeOutputs' set to true). -*/ -void storePathRequisites(const Path & storePath, - bool includeOutputs, PathSet & paths); - -#endif /* !__NORMALISE_H */ diff --git a/src/libstore/store.cc b/src/libstore/store.cc index 01c724b918ae..49a9d273422d 100644 --- a/src/libstore/store.cc +++ b/src/libstore/store.cc @@ -2,7 +2,10 @@ #include #include +#include +#include #include +#include #include "store.hh" #include "globals.hh" @@ -181,6 +184,51 @@ void assertStorePath(const Path & path) } +void canonicalisePathMetaData(const Path & path) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); + } + + if (st.st_uid != getuid() || st.st_gid != getgid()) { + if (chown(path.c_str(), getuid(), getgid()) == -1) + throw SysError(format("changing owner/group of `%1%' to %2%/%3%") + % path % getuid() % getgid()); + } + + if (st.st_mtime != 0) { + struct utimbuf utimbuf; + utimbuf.actime = st.st_atime; + utimbuf.modtime = 0; + if (utime(path.c_str(), &utimbuf) == -1) + throw SysError(format("changing modification time of `%1%'") % path); + } + + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + canonicalisePathMetaData(path + "/" + *i); + } +} + + static bool isValidPathTxn(const Path & path, const Transaction & txn) { string s; @@ -318,12 +366,24 @@ void clearSubstitutes() } -void registerValidPath(const Transaction & txn, const Path & _path) +void registerValidPath(const Transaction & txn, + const Path & _path, const Hash & hash) { Path path(canonPath(_path)); assertStorePath(path); + + assert(hash.type == htSHA256); + debug(format("registering path `%1%'") % path); - nixDB.setString(txn, dbValidPaths, path, ""); + nixDB.setString(txn, dbValidPaths, path, "sha256:" + printHash(hash)); + + /* Check that all referenced paths are also valid. */ + Paths references; + nixDB.queryStrings(txn, dbReferences, path, references); + for (Paths::iterator i = references.begin(); i != references.end(); ++i) + if (!isValidPathTxn(*i, txn)) + throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid") + % path % *i); } @@ -385,10 +445,10 @@ Path addToStore(const Path & _srcPath) throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)") % srcPath % dstPath % printHash(h) % printHash(h2)); - makePathReadOnly(dstPath); + canonicalisePathMetaData(dstPath); Transaction txn(nixDB); - registerValidPath(txn, dstPath); + registerValidPath(txn, dstPath, h); txn.commit(); } @@ -417,10 +477,10 @@ Path addTextToStore(const string & suffix, const string & s) writeStringToFile(dstPath, s); - makePathReadOnly(dstPath); + canonicalisePathMetaData(dstPath); Transaction txn(nixDB); - registerValidPath(txn, dstPath); + registerValidPath(txn, dstPath, hashPath(htSHA256, dstPath)); txn.commit(); } diff --git a/src/libstore/store.hh b/src/libstore/store.hh index 65c3baf98af6..239493a88b3f 100644 --- a/src/libstore/store.hh +++ b/src/libstore/store.hh @@ -51,12 +51,28 @@ Substitutes querySubstitutes(const Path & srcPath); /* Deregister all substitutes. */ void clearSubstitutes(); -/* Register the validity of a path. */ -void registerValidPath(const Transaction & txn, const Path & path); +/* Register the validity of a path, i.e., that `path' exists, that the + paths referenced by it exists, and in the case of an output path of + a derivation, that it has been produced by a succesful execution of + the derivation (or something equivalent). Also register the hash + of the file system contents of the path. The hash must be a + SHA-256 hash. */ +void registerValidPath(const Transaction & txn, + const Path & path, const Hash & hash); /* Throw an exception if `path' is not directly in the Nix store. */ void assertStorePath(const Path & path); +/* "Fix", or canonicalise, the meta-data of the files in a store path + after it has been built. In particular: + - the last modification date on each file is set to 0 (i.e., + 00:00:00 1/1/1970 UTC) + - the permissions are set of 444 or 555 (i.e., read-only with or + without execute permission; setuid bits etc. are cleared) + - the owner and group are set to the Nix user and group, if we're + in a setuid Nix installation. */ +void canonicalisePathMetaData(const Path & path); + /* Checks whether a path is valid. */ bool isValidPath(const Path & path); diff --git a/src/libstore/storeexpr-ast.def b/src/libstore/storeexpr-ast.def deleted file mode 100644 index fa7d0387d6dc..000000000000 --- a/src/libstore/storeexpr-ast.def +++ /dev/null @@ -1,6 +0,0 @@ -init initStoreExprHelpers - -Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm | - -| string string | ATerm | EnvBinding | -| string string string string | ATerm | DerivationOutput | diff --git a/src/libstore/storeexpr.cc b/src/libstore/storeexpr.cc deleted file mode 100644 index a9240130e755..000000000000 --- a/src/libstore/storeexpr.cc +++ /dev/null @@ -1,142 +0,0 @@ -#include "storeexpr.hh" -#include "globals.hh" -#include "store.hh" - -#include "storeexpr-ast.hh" -#include "storeexpr-ast.cc" - - -Hash hashTerm(ATerm t) -{ - return hashString(htSHA256, atPrint(t)); -} - - -Path writeDerivation(const Derivation & drv, const string & name) -{ - return addTextToStore(name + drvExtension, - atPrint(unparseDerivation(drv))); -} - - -static void checkPath(const string & s) -{ - if (s.size() == 0 || s[0] != '/') - throw Error(format("bad path `%1%' in store expression") % s); -} - - -static void parsePaths(ATermList paths, PathSet & out) -{ - for (ATermIterator i(paths); i; ++i) { - if (ATgetType(*i) != AT_APPL) - throw badTerm("not a path", *i); - string s = aterm2String(*i); - checkPath(s); - out.insert(s); - } -} - - -void throwBadDrv(ATerm t) -{ - throw badTerm("not a valid derivation", t); -} - - -Derivation parseDerivation(ATerm t) -{ - Derivation drv; - ATermList outs, inDrvs, inSrcs, args, bnds; - ATerm builder, platform; - - if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds)) - throwBadDrv(t); - - for (ATermIterator i(outs); i; ++i) { - ATerm id, path, hashAlgo, hash; - if (!matchDerivationOutput(*i, id, path, hashAlgo, hash)) - throwBadDrv(t); - DerivationOutput out; - out.path = aterm2String(path); - checkPath(out.path); - out.hashAlgo = aterm2String(hashAlgo); - out.hash = aterm2String(hash); - drv.outputs[aterm2String(id)] = out; - } - - parsePaths(inDrvs, drv.inputDrvs); - parsePaths(inSrcs, drv.inputSrcs); - - drv.builder = aterm2String(builder); - drv.platform = aterm2String(platform); - - for (ATermIterator i(args); i; ++i) { - if (ATgetType(*i) != AT_APPL) - throw badTerm("string expected", *i); - drv.args.push_back(aterm2String(*i)); - } - - for (ATermIterator i(bnds); i; ++i) { - ATerm s1, s2; - if (!matchEnvBinding(*i, s1, s2)) - throw badTerm("tuple of strings expected", *i); - drv.env[aterm2String(s1)] = aterm2String(s2); - } - - return drv; -} - - -static ATermList unparsePaths(const PathSet & paths) -{ - ATermList l = ATempty; - for (PathSet::const_iterator i = paths.begin(); - i != paths.end(); i++) - l = ATinsert(l, toATerm(*i)); - return ATreverse(l); -} - - -ATerm unparseDerivation(const Derivation & drv) -{ - ATermList outputs = ATempty; - for (DerivationOutputs::const_iterator i = drv.outputs.begin(); - i != drv.outputs.end(); i++) - outputs = ATinsert(outputs, - makeDerivationOutput( - toATerm(i->first), - toATerm(i->second.path), - toATerm(i->second.hashAlgo), - toATerm(i->second.hash))); - - ATermList args = ATempty; - for (Strings::const_iterator i = drv.args.begin(); - i != drv.args.end(); i++) - args = ATinsert(args, toATerm(*i)); - - ATermList env = ATempty; - for (StringPairs::const_iterator i = drv.env.begin(); - i != drv.env.end(); i++) - env = ATinsert(env, - makeEnvBinding( - toATerm(i->first), - toATerm(i->second))); - - return makeDerive( - ATreverse(outputs), - unparsePaths(drv.inputDrvs), - unparsePaths(drv.inputSrcs), - toATerm(drv.platform), - toATerm(drv.builder), - ATreverse(args), - ATreverse(env)); -} - - -bool isDerivation(const string & fileName) -{ - return - fileName.size() >= drvExtension.size() && - string(fileName, fileName.size() - drvExtension.size()) == drvExtension; -} diff --git a/src/libstore/storeexpr.hh b/src/libstore/storeexpr.hh deleted file mode 100644 index c7b35f8ebe45..000000000000 --- a/src/libstore/storeexpr.hh +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef __STOREEXPR_H -#define __STOREEXPR_H - -#include "aterm.hh" -#include "store.hh" - - -/* Extension of derivations in the Nix store. */ -const string drvExtension = ".drv"; - - -/* Abstract syntax of derivations. */ - -struct DerivationOutput -{ - Path path; - string hashAlgo; /* hash used for expected hash computation */ - string hash; /* expected hash, may be null */ - DerivationOutput() - { - } - DerivationOutput(Path path, string hashAlgo, string hash) - { - this->path = path; - this->hashAlgo = hashAlgo; - this->hash = hash; - } -}; - -typedef map DerivationOutputs; -typedef map StringPairs; - -struct Derivation -{ - DerivationOutputs outputs; /* keyed on symbolic IDs */ - PathSet inputDrvs; /* inputs that are sub-derivations */ - PathSet inputSrcs; /* inputs that are sources */ - string platform; - Path builder; - Strings args; - StringPairs env; -}; - - -/* Hash an aterm. */ -Hash hashTerm(ATerm t); - -/* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(const Derivation & drv, const string & name); - -/* Parse a derivation. */ -Derivation parseDerivation(ATerm t); - -/* Parse a derivation. */ -ATerm unparseDerivation(const Derivation & drv); - -/* Check whether a file name ends with the extensions for - derivations. */ -bool isDerivation(const string & fileName); - - -#endif /* !__STOREEXPR_H */ -- cgit 1.4.1