Only substitute wanted outputs of a derivation

If a derivation has multiple outputs, then we only want to download
those outputs that are actuallty needed.  So if we do "nix-build -A
openssl.man", then only the "man" output should be downloaded.
Likewise if another package depends on ${openssl.man}.

The tricky part is that different derivations can depend on different
outputs of a given derivation, so we may need to restart the
corresponding derivation goal if that happens.

4 files changed, 77 insertions, 21 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5e5cd6b23bb9..ce87eaed2f31 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -240,7 +240,7 @@ public:
     ~Worker();
 
     /* Make a goal (with caching). */
-    GoalPtr makeDerivationGoal(const Path & drvPath, bool repair = false);
+    GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false);
     GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
 
     /* Remove a dead goal. */
@@ -756,6 +756,13 @@ private:
     /* The path of the derivation. */
     Path drvPath;
 
+    /* The specific outputs that we need to build.  Empty means all of
+       them. */
+    StringSet wantedOutputs;
+
+    /* Whether additional wanted outputs have been added. */
+    bool needRestart;
+
     /* The derivation stored at drvPath. */
     Derivation drv;
 
@@ -831,7 +838,7 @@ private:
     const static int childSetupFailed = 189;
 
 public:
-    DerivationGoal(const Path & drvPath, Worker & worker, bool repair = false);
+    DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false);
     ~DerivationGoal();
 
     void cancel();
@@ -843,6 +850,9 @@ public:
         return drvPath;
     }
 
+    /* Add wanted outputs to an already existing derivation goal. */
+    void addWantedOutputs(const StringSet & outputs);
+
 private:
     /* The states. */
     void init();
@@ -897,8 +907,10 @@ private:
 };
 
 
-DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker, bool repair)
+DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair)
     : Goal(worker)
+    , wantedOutputs(wantedOutputs)
+    , needRestart(false)
     , fLogFile(0)
     , bzLogFile(0)
     , useChroot(false)
@@ -967,6 +979,23 @@ void DerivationGoal::work()
 }
 
 
+void DerivationGoal::addWantedOutputs(const StringSet & outputs)
+{
+    /* If we already want all outputs, there is nothing to do. */
+    if (wantedOutputs.empty()) return;
+
+    if (outputs.empty()) {
+        wantedOutputs.clear();
+        needRestart = true;
+    } else
+        foreach (StringSet::const_iterator, i, outputs)
+            if (wantedOutputs.find(*i) == wantedOutputs.end()) {
+                wantedOutputs.insert(*i);
+                needRestart = true;
+            }
+}
+
+
 void DerivationGoal::init()
 {
     trace("init");
@@ -1043,6 +1072,12 @@ void DerivationGoal::outputsSubstituted()
 
     nrFailed = nrNoSubstituters = 0;
 
+    if (needRestart) {
+        needRestart = false;
+        haveDerivation();
+        return;
+    }
+
     if (checkPathValidity(false, repair).size() == 0) {
         if (repair) repairClosure(); else amDone(ecSuccess);
         return;
@@ -1051,9 +1086,13 @@ void DerivationGoal::outputsSubstituted()
     /* Otherwise, at least one of the output paths could not be
        produced using a substitute.  So we have to build instead. */
 
+    /* Make sure checkPathValidity() from now on checks all
+       outputs. */
+    wantedOutputs = PathSet();
+
     /* The inputs must be built before we can build this goal. */
     foreach (DerivationInputs::iterator, i, drv.inputDrvs)
-        addWaitee(worker.makeDerivationGoal(i->first, repair));
+        addWaitee(worker.makeDerivationGoal(i->first, i->second, repair));
 
     foreach (PathSet::iterator, i, drv.inputSrcs)
         addWaitee(worker.makeSubstitutionGoal(*i));
@@ -1103,7 +1142,7 @@ void DerivationGoal::repairClosure()
         if (drvPath2 == "")
             addWaitee(worker.makeSubstitutionGoal(*i, true));
         else
-            addWaitee(worker.makeDerivationGoal(drvPath2, true));
+            addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true));
     }
 
     if (waitees.empty()) {
@@ -2385,6 +2424,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
 {
     PathSet result;
     foreach (DerivationOutputs::iterator, i, drv.outputs) {
+        if (!wantOutput(i->first, wantedOutputs)) continue;
         bool good =
             worker.store.isValidPath(i->second.path) &&
             (!checkHash || worker.store.pathContentsGood(i->second.path));
@@ -2851,14 +2891,15 @@ Worker::~Worker()
 }
 
 
-GoalPtr Worker::makeDerivationGoal(const Path & path, bool repair)
+GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair)
 {
     GoalPtr goal = derivationGoals[path].lock();
     if (!goal) {
-        goal = GoalPtr(new DerivationGoal(path, *this, repair));
+        goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair));
         derivationGoals[path] = goal;
         wakeUp(goal);
-    }
+    } else
+        (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
     return goal;
 }
 
@@ -3200,7 +3241,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, bool repair)
     foreach (PathSet::const_iterator, i, drvPaths) {
         DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
         if (isDerivation(i2.first))
-            goals.insert(worker.makeDerivationGoal(i2.first, repair));
+            goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair));
         else
             goals.insert(worker.makeSubstitutionGoal(*i, repair));
     }
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1551ae28a81a..d91e42784ca5 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -261,7 +261,7 @@ DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
 }
 
 
-Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs)
+Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
 {
     return outputs.empty()
         ? drvPath
@@ -269,4 +269,10 @@ Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs)
 }
 
 
+bool wantOutput(const string & output, const std::set<string> & wanted)
+{
+    return wanted.empty() || wanted.find(output) != wanted.end();
+}
+
+
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 6b7c3e6ab189..703410b92552 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -85,7 +85,9 @@ extern DrvHashes drvHashes;
 typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
 DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
 
-Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs);
+Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
+
+bool wantOutput(const string & output, const std::set<string> & wanted);
 
 
 }
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index dacd1d3d7474..ecba0c419dd2 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -93,12 +93,13 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
                 Derivation drv = derivationFromPath(store, i2.first);
 
                 PathSet invalid;
-                // FIXME: only fetch the desired outputs
                 foreach (DerivationOutputs::iterator, j, drv.outputs)
-                    if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
+                    if (wantOutput(j->first, i2.second)
+                        && !store.isValidPath(j->second.path))
+                        invalid.insert(j->second.path);
                 if (invalid.empty()) continue;
 
-                todoDrv.insert(i2.first);
+                todoDrv.insert(*i);
                 if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
             }
 
@@ -115,26 +116,32 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
         store.querySubstitutablePathInfos(query, infos);
 
         foreach (PathSet::iterator, i, todoDrv) {
+            DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
+
             // FIXME: cache this
-            Derivation drv = derivationFromPath(store, *i);
+            Derivation drv = derivationFromPath(store, i2.first);
 
+            PathSet outputs;
             bool mustBuild = false;
             if (settings.useSubstitutes) {
-                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                foreach (DerivationOutputs::iterator, j, drv.outputs) {
+                    if (!wantOutput(j->first, i2.second)) continue;
                     if (!store.isValidPath(j->second.path) &&
                         infos.find(j->second.path) == infos.end())
                         mustBuild = true;
+                    else
+                        outputs.insert(j->second.path);
+                }
             } else
                 mustBuild = true;
 
             if (mustBuild) {
-                willBuild.insert(*i);
+                willBuild.insert(i2.first);
                 todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
-                foreach (DerivationInputs::iterator, i, drv.inputDrvs)
-                    todo.insert(i->first);
+                foreach (DerivationInputs::iterator, j, drv.inputDrvs)
+                    todo.insert(makeDrvPathWithOutputs(j->first, j->second));
             } else
-                foreach (DerivationOutputs::iterator, i, drv.outputs)
-                    todoNonDrv.insert(i->second.path);
+                todoNonDrv.insert(outputs.begin(), outputs.end());
         }
 
         foreach (PathSet::iterator, i, todoNonDrv) {