* Another change to low-level derivations. The last one this year, I

  promise :-) This allows derivations to specify on *what* output
  paths of input derivations they are dependent.  This helps to
  prevent unnecessary downloads.  For instance, a build might be
  dependent on the `devel' and `lib' outputs of some library
  component, but not the `docs' output.

7 files changed, 70 insertions, 32 deletions

diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9335ee3138a4..f10597557203 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -68,17 +68,17 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 
     /* For other derivations, replace the inputs paths with recursive
        calls to this function.*/
-    PathSet inputs2;
-    for (PathSet::iterator i = drv.inputDrvs.begin();
+    DerivationInputs inputs2;
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
          i != drv.inputDrvs.end(); ++i)
     {
-        Hash h = state.drvHashes[*i];
+        Hash h = state.drvHashes[i->first];
         if (h.type == htUnknown) {
-            Derivation drv2 = derivationFromPath(*i);
+            Derivation drv2 = derivationFromPath(i->first);
             h = hashDerivationModulo(state, drv2);
-            state.drvHashes[*i] = h;
+            state.drvHashes[i->first] = h;
         }
-        inputs2.insert(printHash(h));
+        inputs2[printHash(h)] = i->second;
     }
     drv.inputDrvs = inputs2;
     
@@ -119,7 +119,9 @@ static void processBinding(EvalState & state, Expr e, Derivation & drv,
             /* !!! supports only single output path */
             Path outPath = evalPath(state, a);
 
-            drv.inputDrvs.insert(drvPath);
+            StringSet ids;
+            ids.insert("out");
+            drv.inputDrvs[drvPath] = ids;
             ss.push_back(outPath);
         } else
             throw Error("invalid derivation attribute");
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 60e72c9dc195..9e984a5b3fd9 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -448,9 +448,10 @@ void DerivationGoal::haveStoreExpr()
     }
 
     /* Inputs must be built before we can build this goal. */
-    for (PathSet::iterator i = drv.inputDrvs.begin();
+    /* !!! but if possible, only install the paths that we need */
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
          i != drv.inputDrvs.end(); ++i)
-        addWaitee(worker.makeDerivationGoal(*i));
+        addWaitee(worker.makeDerivationGoal(i->first));
 
     for (PathSet::iterator i = drv.inputSrcs.begin();
          i != drv.inputSrcs.end(); ++i)
@@ -812,18 +813,23 @@ bool DerivationGoal::prepareBuild()
     /* Determine the full set of input paths. */
 
     /* First, the input derivations. */
-    for (PathSet::iterator i = drv.inputDrvs.begin();
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
          i != drv.inputDrvs.end(); ++i)
     {
-        /* Add all the output closures of the input derivation `*i' as
-           input paths.  !!!  there should be a way to indicate
-           specific outputs. */
+        /* Add the relevant output closures of the input derivation
+           `*i' as input paths.  Only add the closures of output paths
+           that are specified as inputs. */
         /* !!! is `*i' present? */
-        assert(isValidPath(*i));
-        Derivation inDrv = derivationFromPath(*i);
-        for (DerivationOutputs::iterator j = inDrv.outputs.begin();
-             j != inDrv.outputs.end(); ++j)
-            computeFSClosure(j->second.path, inputPaths);
+        assert(isValidPath(i->first));
+        Derivation inDrv = derivationFromPath(i->first);
+        for (StringSet::iterator j = i->second.begin();
+             j != i->second.begin(); ++j)
+            if (inDrv.outputs.find(*j) != inDrv.outputs.end())
+                computeFSClosure(inDrv.outputs[*j].path, inputPaths);
+            else
+                throw Error(
+                    format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
+                    % drvPath % *j % i->first);
     }
 
     for (PathSet::iterator i = inputPaths.begin(); i != inputPaths.end(); ++i)
diff --git a/src/libstore/derivations-ast.def b/src/libstore/derivations-ast.def
index 2dce8de12c01..38689845883b 100644
--- a/src/libstore/derivations-ast.def
+++ b/src/libstore/derivations-ast.def
@@ -3,4 +3,5 @@ init initDerivationsHelpers
 Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
 
 | string string | ATerm | EnvBinding |
+| string ATermList | ATerm | DerivationInput |
 | string string string string | ATerm | DerivationOutput |
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 2f37c66fb60d..9008be43981c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -26,13 +26,13 @@ static void checkPath(const string & s)
 }
     
 
-static void parsePaths(ATermList paths, PathSet & out)
+static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
 {
     for (ATermIterator i(paths); i; ++i) {
         if (ATgetType(*i) != AT_APPL)
             throw badTerm("not a path", *i);
         string s = aterm2String(*i);
-        checkPath(s);
+        if (arePaths) checkPath(s);
         out.insert(s);
     }
 }
@@ -65,8 +65,19 @@ Derivation parseDerivation(ATerm t)
         drv.outputs[aterm2String(id)] = out;
     }
 
-    parsePaths(inDrvs, drv.inputDrvs);
-    parsePaths(inSrcs, drv.inputSrcs);
+    for (ATermIterator i(inDrvs); i; ++i) {
+        ATerm drvPath;
+        ATermList ids;
+        if (!matchDerivationInput(*i, drvPath, ids))
+            throwBadDrv(t);
+        Path drvPath2 = aterm2String(drvPath);
+        checkPath(drvPath2);
+        StringSet ids2;
+        parseStrings(ids, ids2, false);
+        drv.inputDrvs[drvPath2] = ids2;
+    }
+    
+    parseStrings(inSrcs, drv.inputSrcs, true);
 
     drv.builder = aterm2String(builder);
     drv.platform = aterm2String(platform);
@@ -88,11 +99,11 @@ Derivation parseDerivation(ATerm t)
 }
 
 
-static ATermList unparsePaths(const PathSet & paths)
+static ATermList unparseStrings(const StringSet & paths)
 {
     ATermList l = ATempty;
     for (PathSet::const_iterator i = paths.begin();
-         i != paths.end(); i++)
+         i != paths.end(); ++i)
         l = ATinsert(l, toATerm(*i));
     return ATreverse(l);
 }
@@ -102,7 +113,7 @@ ATerm unparseDerivation(const Derivation & drv)
 {
     ATermList outputs = ATempty;
     for (DerivationOutputs::const_iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); i++)
+         i != drv.outputs.end(); ++i)
         outputs = ATinsert(outputs,
             makeDerivationOutput(
                 toATerm(i->first),
@@ -110,14 +121,22 @@ ATerm unparseDerivation(const Derivation & drv)
                 toATerm(i->second.hashAlgo),
                 toATerm(i->second.hash)));
 
+    ATermList inDrvs = ATempty;
+    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+        inDrvs = ATinsert(inDrvs,
+            makeDerivationInput(
+                toATerm(i->first),
+                unparseStrings(i->second)));
+    
     ATermList args = ATempty;
     for (Strings::const_iterator i = drv.args.begin();
-         i != drv.args.end(); i++)
+         i != drv.args.end(); ++i)
         args = ATinsert(args, toATerm(*i));
 
     ATermList env = ATempty;
     for (StringPairs::const_iterator i = drv.env.begin();
-         i != drv.env.end(); i++)
+         i != drv.env.end(); ++i)
         env = ATinsert(env,
             makeEnvBinding(
                 toATerm(i->first),
@@ -125,8 +144,8 @@ ATerm unparseDerivation(const Derivation & drv)
 
     return makeDerive(
         ATreverse(outputs),
-        unparsePaths(drv.inputDrvs),
-        unparsePaths(drv.inputSrcs),
+        ATreverse(inDrvs),
+        unparseStrings(drv.inputSrcs),
         toATerm(drv.platform),
         toATerm(drv.builder),
         ATreverse(args),
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index c264981d265c..9358db2edae0 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -28,12 +28,17 @@ struct DerivationOutput
 };
 
 typedef map<string, DerivationOutput> DerivationOutputs;
+
+/* For inputs that are sub-derivations, we specify exactly which
+   output IDs we are interested in. */
+typedef map<Path, StringSet> DerivationInputs;
+
 typedef map<string, string> StringPairs;
 
 struct Derivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
-    PathSet inputDrvs; /* inputs that are sub-derivations */
+    DerivationInputs inputDrvs; /* inputs that are sub-derivations */
     PathSet inputSrcs; /* inputs that are sources */
     string platform;
     Path builder;
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index c585a9aeac5c..dbb9273f7955 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -39,9 +39,12 @@ void storePathRequisites(const Path & storePath,
         
         Derivation drv = derivationFromPath(storePath);
 
-        for (PathSet::iterator i = drv.inputDrvs.begin();
+        for (DerivationInputs::iterator i = drv.inputDrvs.begin();
              i != drv.inputDrvs.end(); ++i)
-            storePathRequisites(*i, includeOutputs, paths);
+            /* !!! Maybe this is too strict, since it will include
+               *all* output paths of the input derivation, not just
+               the ones needed by this derivation. */
+            storePathRequisites(i->first, includeOutputs, paths);
 
         for (PathSet::iterator i = drv.inputSrcs.begin();
              i != drv.inputSrcs.end(); ++i)
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 266ab67f04ee..e77009321d8d 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -587,6 +587,8 @@ Strings unpackStrings(const string & s)
         len |= ((unsigned char) *i++) << 8;
         len |= ((unsigned char) *i++) << 16;
         len |= ((unsigned char) *i++) << 24;
+
+        if (len == 0xffffffff) return strings; /* explicit end-of-list */
         
         if (i + len > s.end())
             throw Error(format("short db entry: `%1%'") % s);