From 8d8d47abd2a66898aa5d8999fcd75b29991e529d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Nov 2012 17:15:09 +0100 Subject: Only substitute wanted outputs of a derivation If a derivation has multiple outputs, then we only want to download those outputs that are actuallty needed. So if we do "nix-build -A openssl.man", then only the "man" output should be downloaded. Likewise if another package depends on ${openssl.man}. The tricky part is that different derivations can depend on different outputs of a given derivation, so we may need to restart the corresponding derivation goal if that happens. --- src/libstore/build.cc | 59 ++++++++++++++++++++++++++++++++++++++------- src/libstore/derivations.cc | 8 +++++- src/libstore/derivations.hh | 4 ++- src/libstore/misc.cc | 27 +++++++++++++-------- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5e5cd6b23bb9..ce87eaed2f31 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -240,7 +240,7 @@ public: ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath, bool repair = false); + GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false); GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); /* Remove a dead goal. */ @@ -756,6 +756,13 @@ private: /* The path of the derivation. */ Path drvPath; + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart; + /* The derivation stored at drvPath. */ Derivation drv; @@ -831,7 +838,7 @@ private: const static int childSetupFailed = 189; public: - DerivationGoal(const Path & drvPath, Worker & worker, bool repair = false); + DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false); ~DerivationGoal(); void cancel(); @@ -843,6 +850,9 @@ public: return drvPath; } + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet & outputs); + private: /* The states. */ void init(); @@ -897,8 +907,10 @@ private: }; -DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker, bool repair) +DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair) : Goal(worker) + , wantedOutputs(wantedOutputs) + , needRestart(false) , fLogFile(0) , bzLogFile(0) , useChroot(false) @@ -967,6 +979,23 @@ void DerivationGoal::work() } +void DerivationGoal::addWantedOutputs(const StringSet & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + foreach (StringSet::const_iterator, i, outputs) + if (wantedOutputs.find(*i) == wantedOutputs.end()) { + wantedOutputs.insert(*i); + needRestart = true; + } +} + + void DerivationGoal::init() { trace("init"); @@ -1043,6 +1072,12 @@ void DerivationGoal::outputsSubstituted() nrFailed = nrNoSubstituters = 0; + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + if (checkPathValidity(false, repair).size() == 0) { if (repair) repairClosure(); else amDone(ecSuccess); return; @@ -1051,9 +1086,13 @@ void DerivationGoal::outputsSubstituted() /* Otherwise, at least one of the output paths could not be produced using a substitute. So we have to build instead. */ + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = PathSet(); + /* The inputs must be built before we can build this goal. */ foreach (DerivationInputs::iterator, i, drv.inputDrvs) - addWaitee(worker.makeDerivationGoal(i->first, repair)); + addWaitee(worker.makeDerivationGoal(i->first, i->second, repair)); foreach (PathSet::iterator, i, drv.inputSrcs) addWaitee(worker.makeSubstitutionGoal(*i)); @@ -1103,7 +1142,7 @@ void DerivationGoal::repairClosure() if (drvPath2 == "") addWaitee(worker.makeSubstitutionGoal(*i, true)); else - addWaitee(worker.makeDerivationGoal(drvPath2, true)); + addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true)); } if (waitees.empty()) { @@ -2385,6 +2424,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { PathSet result; foreach (DerivationOutputs::iterator, i, drv.outputs) { + if (!wantOutput(i->first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i->second.path) && (!checkHash || worker.store.pathContentsGood(i->second.path)); @@ -2851,14 +2891,15 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const Path & path, bool repair) +GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair) { GoalPtr goal = derivationGoals[path].lock(); if (!goal) { - goal = GoalPtr(new DerivationGoal(path, *this, repair)); + goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair)); derivationGoals[path] = goal; wakeUp(goal); - } + } else + (dynamic_cast(goal.get()))->addWantedOutputs(wantedOutputs); return goal; } @@ -3200,7 +3241,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, bool repair) foreach (PathSet::const_iterator, i, drvPaths) { DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); if (isDerivation(i2.first)) - goals.insert(worker.makeDerivationGoal(i2.first, repair)); + goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair)); else goals.insert(worker.makeSubstitutionGoal(*i, repair)); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1551ae28a81a..d91e42784ca5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -261,7 +261,7 @@ DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) } -Path makeDrvPathWithOutputs(const Path & drvPath, std::set outputs) +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set & outputs) { return outputs.empty() ? drvPath @@ -269,4 +269,10 @@ Path makeDrvPathWithOutputs(const Path & drvPath, std::set outputs) } +bool wantOutput(const string & output, const std::set & wanted) +{ + return wanted.empty() || wanted.find(output) != wanted.end(); +} + + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6b7c3e6ab189..703410b92552 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -85,7 +85,9 @@ extern DrvHashes drvHashes; typedef std::pair > DrvPathWithOutputs; DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); -Path makeDrvPathWithOutputs(const Path & drvPath, std::set outputs); +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set & outputs); + +bool wantOutput(const string & output, const std::set & wanted); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index dacd1d3d7474..ecba0c419dd2 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -93,12 +93,13 @@ void queryMissing(StoreAPI & store, const PathSet & targets, Derivation drv = derivationFromPath(store, i2.first); PathSet invalid; - // FIXME: only fetch the desired outputs foreach (DerivationOutputs::iterator, j, drv.outputs) - if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path); + if (wantOutput(j->first, i2.second) + && !store.isValidPath(j->second.path)) + invalid.insert(j->second.path); if (invalid.empty()) continue; - todoDrv.insert(i2.first); + todoDrv.insert(*i); if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end()); } @@ -115,26 +116,32 @@ void queryMissing(StoreAPI & store, const PathSet & targets, store.querySubstitutablePathInfos(query, infos); foreach (PathSet::iterator, i, todoDrv) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + // FIXME: cache this - Derivation drv = derivationFromPath(store, *i); + Derivation drv = derivationFromPath(store, i2.first); + PathSet outputs; bool mustBuild = false; if (settings.useSubstitutes) { - foreach (DerivationOutputs::iterator, j, drv.outputs) + foreach (DerivationOutputs::iterator, j, drv.outputs) { + if (!wantOutput(j->first, i2.second)) continue; if (!store.isValidPath(j->second.path) && infos.find(j->second.path) == infos.end()) mustBuild = true; + else + outputs.insert(j->second.path); + } } else mustBuild = true; if (mustBuild) { - willBuild.insert(*i); + willBuild.insert(i2.first); todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - foreach (DerivationInputs::iterator, i, drv.inputDrvs) - todo.insert(i->first); + foreach (DerivationInputs::iterator, j, drv.inputDrvs) + todo.insert(makeDrvPathWithOutputs(j->first, j->second)); } else - foreach (DerivationOutputs::iterator, i, drv.outputs) - todoNonDrv.insert(i->second.path); + todoNonDrv.insert(outputs.begin(), outputs.end()); } foreach (PathSet::iterator, i, todoNonDrv) { -- cgit 1.4.1