diff options
Diffstat (limited to 'src/libstore/normalise.cc')
| -rw-r--r-- | src/libstore/normalise.cc | 369 |
1 files changed, 369 insertions, 0 deletions
diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc new file mode 100644 index 000000000000..cb2bf4f5b183 --- /dev/null +++ b/src/libstore/normalise.cc @@ -0,0 +1,369 @@ +#include <map> + +#include "normalise.hh" +#include "references.hh" +#include "exec.hh" +#include "pathlocks.hh" +#include "globals.hh" + + +static Path useSuccessor(const Path & path) +{ + string pathSucc; + if (querySuccessor(path, pathSucc)) { + debug(format("successor %1% -> %2%") % (string) path % pathSucc); + return pathSucc; + } else + return path; +} + + +Path normaliseNixExpr(const Path & _nePath, PathSet pending) +{ + startNest(nest, lvlTalkative, + format("normalising expression in `%1%'") % (string) _nePath); + + /* Try to substitute the expression by any known successors in + order to speed up the rewrite process. */ + Path nePath = useSuccessor(_nePath); + + /* Get the Nix expression. */ + NixExpr ne = exprFromPath(nePath, pending); + + /* If this is a normal form (i.e., a closure) we are done. */ + if (ne.type == NixExpr::neClosure) return nePath; + if (ne.type != NixExpr::neDerivation) abort(); + + + /* Otherwise, it's a derivation expression, and we have to build it to + determine its normal form. */ + + + /* Some variables. */ + + /* Input paths, with their closure elements. */ + ClosureElems inClosures; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* The environment to be passed to the builder. */ + Environment env; + + /* The result. */ + NixExpr nf; + nf.type = NixExpr::neClosure; + + + /* The outputs are referenceable paths. */ + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + debug(format("building path `%1%'") % *i); + allPaths.insert(*i); + } + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. */ + PathLocks outputLocks(ne.derivation.outputs); + + /* Now check again whether there is a successor. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first successor + check above can be omitted, but that would be less efficient.) + Note that since we now hold the locks on the output paths, no + other process can build this expression, so no further checks + are necessary. */ + { + Path nePath2 = useSuccessor(nePath); + if (nePath != nePath2) { + NixExpr ne = exprFromPath(nePath2, pending); + debug(format("skipping build of expression `%1%', someone beat us to it") + % (string) nePath); + if (ne.type != NixExpr::neClosure) abort(); + return nePath2; + } + } + + /* Right platform? */ + if (ne.derivation.platform != thisSystem) + throw Error(format("a `%1%' is required, but I am a `%2%'") + % ne.derivation.platform % thisSystem); + + /* Realise inputs (and remember all input paths). */ + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); i++) + { + Path nfPath = normaliseNixExpr(*i, pending); + realiseClosure(nfPath, pending); + /* !!! nfPath should be a root of the garbage collector while + we are building */ + NixExpr ne = exprFromPath(nfPath, pending); + if (ne.type != NixExpr::neClosure) abort(); + for (ClosureElems::iterator j = ne.closure.elems.begin(); + j != ne.closure.elems.end(); j++) + { + inClosures[j->first] = j->second; + allPaths.insert(j->first); + } + } + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = "/homeless-shelter"; + + /* Build the environment. */ + for (StringPairs::iterator i = ne.derivation.env.begin(); + i != ne.derivation.env.end(); i++) + env[i->first] = i->second; + + /* We can skip running the builder if all output paths are already + valid. */ + bool fastBuild = true; + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + if (!isValidPath(*i)) { + fastBuild = false; + break; + } + } + + if (!fastBuild) { + + /* If any of the outputs already exist but are not registered, + delete them. */ + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + Path path = *i; + if (isValidPath(path)) + throw Error(format("obstructed build: path `%1%' exists") % path); + if (pathExists(path)) { + debug(format("removing unregistered path `%1%'") % path); + deletePath(path); + } + } + + /* Run the builder. */ + printMsg(lvlChatty, format("building...")); + runProgram(ne.derivation.builder, ne.derivation.args, env, + nixLogDir + "/" + baseNameOf(nePath)); + printMsg(lvlChatty, format("build completed")); + + } else + printMsg(lvlChatty, format("fast build succesful")); + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + PathSet usedPaths; + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + { + Path path = *i; + if (!pathExists(path)) + throw Error(format("path `%1%' does not exist") % path); + nf.closure.roots.insert(path); + + makePathReadOnly(path); + + /* For this output path, find the references to other paths contained + in it. */ + Strings refPaths = filterReferences(path, + Strings(allPaths.begin(), allPaths.end())); + + /* Construct a closure element for this output path. */ + ClosureElem elem; + + /* For each path referenced by this output path, add its id to the + closure element and add the id to the `usedPaths' set (so that the + elements referenced by *its* closure are added below). */ + for (Paths::iterator j = refPaths.begin(); + j != refPaths.end(); j++) + { + Path path = *j; + elem.refs.insert(path); + if (inClosures.find(path) != inClosures.end()) + usedPaths.insert(path); + else if (ne.derivation.outputs.find(path) == ne.derivation.outputs.end()) + abort(); + } + + nf.closure.elems[path] = elem; + } + + /* Close the closure. That is, for any referenced path, add the paths + referenced by it. */ + PathSet donePaths; + + while (!usedPaths.empty()) { + PathSet::iterator i = usedPaths.begin(); + Path path = *i; + usedPaths.erase(i); + + if (donePaths.find(path) != donePaths.end()) continue; + donePaths.insert(path); + + ClosureElems::iterator j = inClosures.find(path); + if (j == inClosures.end()) abort(); + + nf.closure.elems[path] = j->second; + + for (PathSet::iterator k = j->second.refs.begin(); + k != j->second.refs.end(); k++) + usedPaths.insert(*k); + } + + /* For debugging, print out the referenced and unreferenced paths. */ + for (ClosureElems::iterator i = inClosures.begin(); + i != inClosures.end(); i++) + { + PathSet::iterator j = donePaths.find(i->first); + if (j == donePaths.end()) + debug(format("NOT referenced: `%1%'") % i->first); + else + debug(format("referenced: `%1%'") % i->first); + } + + /* Write the normal form. This does not have to occur in the + transaction below because writing terms is idem-potent. */ + ATerm nfTerm = unparseNixExpr(nf); + printMsg(lvlVomit, format("normal form: %1%") % atPrint(nfTerm)); + Path nfPath = writeTerm(nfTerm, "-s"); + + /* Register each outpat path, and register the normal form. This + is wrapped in one database transaction to ensure that if we + crash, either everything is registered or nothing is. This is + for recoverability: unregistered paths in the store can be + deleted arbitrarily, while registered paths can only be deleted + by running the garbage collector. */ + Transaction txn; + createStoreTransaction(txn); + for (PathSet::iterator i = ne.derivation.outputs.begin(); + i != ne.derivation.outputs.end(); i++) + registerValidPath(txn, *i); + registerSuccessor(txn, nePath, nfPath); + txn.commit(); + + return nfPath; +} + + +void realiseClosure(const Path & nePath, PathSet pending) +{ + startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); + + NixExpr ne = exprFromPath(nePath, pending); + if (ne.type != NixExpr::neClosure) + throw Error(format("expected closure in `%1%'") % nePath); + + for (ClosureElems::const_iterator i = ne.closure.elems.begin(); + i != ne.closure.elems.end(); i++) + ensurePath(i->first, pending); +} + + +void ensurePath(const Path & path, PathSet pending) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + /* Otherwise, try the substitutes. */ + Paths subPaths = querySubstitutes(path); + + for (Paths::iterator i = subPaths.begin(); + i != subPaths.end(); i++) + { + try { + normaliseNixExpr(*i, pending); + if (isValidPath(path)) return; + throw Error(format("substitute failed to produce expected output path")); + } catch (Error & e) { + printMsg(lvlTalkative, + format("building of substitute `%1%' for `%2%' failed: %3%") + % *i % path % e.what()); + } + } + + throw Error(format("path `%1%' is required, " + "but there are no (successful) substitutes") % path); +} + + +NixExpr exprFromPath(const Path & path, PathSet pending) +{ + ensurePath(path, pending); + ATerm t = ATreadFromNamedFile(path.c_str()); + if (!t) throw Error(format("cannot read aterm from `%1%'") % path); + return parseNixExpr(t); +} + + +PathSet nixExprRoots(const Path & nePath) +{ + PathSet paths; + + NixExpr ne = exprFromPath(nePath); + + if (ne.type == NixExpr::neClosure) + paths.insert(ne.closure.roots.begin(), ne.closure.roots.end()); + else if (ne.type == NixExpr::neDerivation) + paths.insert(ne.derivation.outputs.begin(), + ne.derivation.outputs.end()); + else abort(); + + return paths; +} + + +static void requisitesWorker(const Path & nePath, + bool includeExprs, bool includeSuccessors, + PathSet & paths, PathSet & doneSet) +{ + if (doneSet.find(nePath) != doneSet.end()) return; + doneSet.insert(nePath); + + NixExpr ne = exprFromPath(nePath); + + if (ne.type == NixExpr::neClosure) + for (ClosureElems::iterator i = ne.closure.elems.begin(); + i != ne.closure.elems.end(); i++) + paths.insert(i->first); + + else if (ne.type == NixExpr::neDerivation) + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); i++) + requisitesWorker(*i, + includeExprs, includeSuccessors, paths, doneSet); + + else abort(); + + if (includeExprs) paths.insert(nePath); + + string nfPath; + if (includeSuccessors && (nfPath = useSuccessor(nePath)) != nePath) + requisitesWorker(nfPath, includeExprs, includeSuccessors, + paths, doneSet); +} + + +PathSet nixExprRequisites(const Path & nePath, + bool includeExprs, bool includeSuccessors) +{ + PathSet paths; + PathSet doneSet; + requisitesWorker(nePath, includeExprs, includeSuccessors, + paths, doneSet); + return paths; +} |