diff options
| -rw-r--r-- | src/libstore/build.cc | 94 | ||||
| -rw-r--r-- | src/nix-store/help.txt | 2 | ||||
| -rw-r--r-- | src/nix-store/main.cc | 46 | ||||
| -rw-r--r-- | tests/dependencies.sh | 6 | ||||
| -rw-r--r-- | tests/locking.sh | 4 | ||||
| -rw-r--r-- | tests/parallel.sh | 2 | ||||
| -rw-r--r-- | tests/simple.sh | 2 |
7 files changed, 100 insertions, 56 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 072880765f21..8d9946add446 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -337,6 +337,7 @@ private: /* The states. */ void init(); void haveStoreExpr(); + void outputsSubstituted(); void inputsRealised(); void tryToBuild(); void buildDone(); @@ -376,8 +377,8 @@ private: /* Callback used by the worker to write to the log. */ void writeLog(int fd, const unsigned char * buf, size_t count); - /* Return true iff all output paths are valid. */ - bool allOutputsValid(); + /* Return the set of (in)valid paths. */ + PathSet checkPathValidity(bool returnValid); string name(); }; @@ -441,13 +442,45 @@ void DerivationGoal::haveStoreExpr() /* Get the derivation. */ drv = derivationFromPath(drvPath); - /* If all the outputs already exist, then we're done. */ - if (allOutputsValid()) { + /* Check what outputs paths are not already valid. */ + PathSet invalidOutputs = checkPathValidity(false); + + /* If they are all valid, then we're done. */ + if (invalidOutputs.size() == 0) { + amDone(true); + return; + } + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + for (PathSet::iterator i = invalidOutputs.begin(); + i != invalidOutputs.end(); ++i) + /* Don't bother creating a substitution goal if there are no + substitutes. */ + if (querySubstitutes(*i).size() > 0) + addWaitee(worker.makeSubstitutionGoal(*i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstituted(); + else + state = &DerivationGoal::outputsSubstituted; +} + + +void DerivationGoal::outputsSubstituted() +{ + trace("all outputs substituted (maybe)"); + + if (checkPathValidity(false).size() == 0) { amDone(true); return; } - /* Inputs must be built before we can build this goal. */ + /* Otherwise, at least one of the output paths could not be + produced using a substitute. So we have to build instead. */ + + /* The inputs must be built before we can build this goal. */ /* !!! but if possible, only install the paths that we need */ for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) @@ -792,13 +825,21 @@ bool DerivationGoal::prepareBuild() omitted, but that would be less efficient.) Note that since we now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ - if (allOutputsValid()) { + PathSet validPaths = checkPathValidity(true); + if (validPaths.size() == drv.outputs.size()) { debug(format("skipping build of derivation `%1%', someone beat us to it") % drvPath); outputLocks.setDeletion(true); return false; } + if (validPaths.size() > 0) { + /* !!! fix this; try to delete valid paths */ + throw Error( + format("derivation `%1%' is blocked by its output paths") + % drvPath); + } + /* Gather information necessary for computing the closure and/or running the build hook. */ @@ -823,7 +864,7 @@ bool DerivationGoal::prepareBuild() assert(isValidPath(i->first)); Derivation inDrv = derivationFromPath(i->first); for (StringSet::iterator j = i->second.begin(); - j != i->second.begin(); ++j) + j != i->second.end(); ++j) if (inDrv.outputs.find(*j) != inDrv.outputs.end()) computeFSClosure(inDrv.outputs[*j].path, inputPaths); else @@ -832,8 +873,7 @@ bool DerivationGoal::prepareBuild() % drvPath % *j % i->first); } - for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i) - debug(format("INPUT %1%") % *i); + debug(format("added input paths %1%") % showPaths(inputPaths)); allPaths.insert(inputPaths.begin(), inputPaths.end()); @@ -842,22 +882,6 @@ bool DerivationGoal::prepareBuild() i != drv.inputSrcs.end(); ++i) computeFSClosure(*i, inputPaths); - /* We can skip running the builder if all output paths are already - valid. */ - bool fastBuild = true; - for (DerivationOutputs::iterator i = drv.outputs.begin(); - i != drv.outputs.end(); ++i) - if (!isValidPath(i->second.path)) { - fastBuild = false; - break; - } - - if (fastBuild) { - printMsg(lvlChatty, format("skipping build; output paths already exist")); - computeClosure(); - return false; - } - return true; } @@ -1164,21 +1188,17 @@ void DerivationGoal::writeLog(int fd, } -bool DerivationGoal::allOutputsValid() +PathSet DerivationGoal::checkPathValidity(bool returnValid) { - unsigned int nrValid = 0; + PathSet result; for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - if (isValidPath(i->second.path)) nrValid++; - - if (nrValid != 0) { - if (nrValid == drv.outputs.size()) return true; - throw Error( - format("derivation `%1%' is blocked by its output paths") - % drvPath); - } - - return false; + if (isValidPath(i->second.path)) { + if (returnValid) result.insert(i->second.path); + } else { + if (!returnValid) result.insert(i->second.path); + } + return result; } diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 1bd752fc11c6..deef23ff8f3b 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -4,7 +4,7 @@ nix-store [OPTIONS...] [ARGUMENTS...] Operations: - --build / -b: build a Nix derivation + --realise / -r: build a Nix derivation --add / -A: copy a path to the Nix store --query / -q: query information diff --git a/src/nix-store/main.cc b/src/nix-store/main.cc index e922a9755cc6..addf9c7d0d67 100644 --- a/src/nix-store/main.cc +++ b/src/nix-store/main.cc @@ -27,19 +27,39 @@ static Path findOutput(const Derivation & drv, string id) } -/* Build the given derivations. */ -static void opBuild(Strings opFlags, Strings opArgs) +/* Realisation the given path. For a derivation that means build it; + for other paths it means ensure their validity. */ +static Path realisePath(const Path & path) +{ + if (isDerivation(path)) { + PathSet paths; + paths.insert(path); + buildDerivations(paths); + return findOutput(derivationFromPath(path), "out"); + } else { + ensurePath(path); + return path; + } +} + + +/* Realise the given paths. */ +static void opRealise(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - buildDerivations(PathSet(opArgs.begin(), opArgs.end())); + if (opArgs.size() > 1) { + PathSet drvPaths; + for (Strings::iterator i = opArgs.begin(); + i != opArgs.end(); i++) + if (isDerivation(*i)) + drvPaths.insert(*i); + buildDerivations(drvPaths); + } for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) - { - Derivation drv = derivationFromPath(*i); - cout << format("%1%\n") % findOutput(drv, "out"); - } + cout << format("%1%\n") % realisePath(*i); } @@ -54,8 +74,9 @@ static void opAdd(Strings opFlags, Strings opArgs) } -static Path maybeUseOutput(const Path & storePath, bool useOutput) +static Path maybeUseOutput(const Path & storePath, bool useOutput, bool forceRealise) { + if (forceRealise) realisePath(storePath); if (useOutput && isDerivation(storePath)) { Derivation drv = derivationFromPath(storePath); return findOutput(drv, "out"); @@ -78,6 +99,7 @@ static void opQuery(Strings opFlags, Strings opArgs) enum { qOutputs, qRequisites, qReferences, qReferers, qGraph } query = qOutputs; bool useOutput = false; bool includeOutputs = false; + bool forceRealise = false; for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); i++) @@ -87,6 +109,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (*i == "--referers") query = qReferers; else if (*i == "--graph") query = qGraph; else if (*i == "--use-output" || *i == "-u") useOutput = true; + else if (*i == "--force-realise" || *i == "-f") forceRealise = true; else if (*i == "--include-outputs") includeOutputs = true; else throw UsageError(format("unknown flag `%1%'") % *i); @@ -96,6 +119,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) { + if (forceRealise) realisePath(*i); Derivation drv = derivationFromPath(*i); cout << format("%1%\n") % findOutput(drv, "out"); } @@ -109,7 +133,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) { - Path path = maybeUseOutput(*i, useOutput); + Path path = maybeUseOutput(*i, useOutput, forceRealise); if (query == qRequisites) storePathRequisites(path, includeOutputs, paths); else if (query == qReferences) queryReferences(path, paths); @@ -340,8 +364,8 @@ void run(Strings args) Operation oldOp = op; - if (arg == "--build" || arg == "-b") - op = opBuild; + if (arg == "--realise" || arg == "-r") + op = opRealise; else if (arg == "--add" || arg == "-A") op = opAdd; else if (arg == "--query" || arg == "-q") diff --git a/tests/dependencies.sh b/tests/dependencies.sh index a9b9cd083f13..fe4aa0c3c960 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -2,16 +2,16 @@ storeExpr=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix) echo "store expr is $storeExpr" -outPath=$($TOP/src/nix-store/nix-store -bvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -rvv "$storeExpr") echo "output path is $outPath" text=$(cat "$outPath"/foobar) if test "$text" != "FOOBAR"; then exit 1; fi -deps=$($TOP/src/nix-store/nix-store -qnR "$storeExpr") +deps=$($TOP/src/nix-store/nix-store -quR "$storeExpr") -echo "output closures are $deps" +echo "output closure contains $deps" # The output path should be in the closure. echo "$deps" | grep -q "$outPath" diff --git a/tests/locking.sh b/tests/locking.sh index f89e1240bde8..5cafa2910e6f 100644 --- a/tests/locking.sh +++ b/tests/locking.sh @@ -4,12 +4,12 @@ echo "store expr is $storeExpr" for i in 1 2 3 4 5; do echo "WORKER $i" - $TOP/src/nix-store/nix-store -rvvvvvB "$storeExpr" & + $TOP/src/nix-store/nix-store -rvv "$storeExpr" & done sleep 5 -outPath=$($TOP/src/nix-store/nix-store -qnfvvvvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qvvf "$storeExpr") echo "output path is $outPath" diff --git a/tests/parallel.sh b/tests/parallel.sh index d16da3a7ac20..0044aaea6080 100644 --- a/tests/parallel.sh +++ b/tests/parallel.sh @@ -2,7 +2,7 @@ storeExpr=$($TOP/src/nix-instantiate/nix-instantiate parallel.nix) echo "store expr is $storeExpr" -outPath=$($TOP/src/nix-store/nix-store -qnfvvvv -j10000 "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qfvv -j10000 "$storeExpr") echo "output path is $outPath" diff --git a/tests/simple.sh b/tests/simple.sh index 4da1a7b53394..a5435a53bda4 100644 --- a/tests/simple.sh +++ b/tests/simple.sh @@ -2,7 +2,7 @@ storeExpr=$($TOP/src/nix-instantiate/nix-instantiate simple.nix) echo "store expr is $storeExpr" -outPath=$($TOP/src/nix-store/nix-store -bvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -rvv "$storeExpr") echo "output path is $outPath" |