* Merge the multiple-outputs-sandbox branch (svn merge --reintegrate

  ^/nix/branches/multiple-outputs-sandbox).  Multiple output support
  still isn't complete, but it wasn't complete in the trunk either, so
  it doesn't hurt.

13 files changed, 190 insertions, 104 deletions

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 2b97b76fb776..bba14bc35491 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -148,8 +148,6 @@ EvalState::EvalState()
     nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
     deepestStack = (char *) -1;
 
-    createBaseEnv();
-    
     allowUnsafeEquality = getEnv("NIX_NO_UNSAFE_EQ", "") == "";
 
 #if HAVE_BOEHMGC
@@ -188,6 +186,8 @@ EvalState::EvalState()
     foreach (Strings::iterator, i, paths) addToSearchPath(*i);
     addToSearchPath("nix=" + nixDataDir + "/nix/corepkgs");
     searchPathInsertionPoint = searchPath.begin();
+
+    createBaseEnv();
 }
 
 
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 66173cdaf05f..21c9fb351d98 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -356,27 +356,31 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
            inputs to ensure that they are available when the builder
            runs. */
         if (path.at(0) == '=') {
-            path = string(path, 1);
-            PathSet refs; computeFSClosure(*store, path, refs);
+            /* !!! This doesn't work if readOnlyMode is set. */
+            PathSet refs; computeFSClosure(*store, string(path, 1), refs);
             foreach (PathSet::iterator, j, refs) {
                 drv.inputSrcs.insert(*j);
                 if (isDerivation(*j))
-                    drv.inputDrvs[*j] = singleton<StringSet>("out");
+                    drv.inputDrvs[*j] = store->queryDerivationOutputNames(*j);
             }
         }
 
         /* See prim_unsafeDiscardOutputDependency. */
-        bool useDrvAsSrc = false;
-        if (path.at(0) == '~') {
-            path = string(path, 1);
-            useDrvAsSrc = true;
+        else if (path.at(0) == '~')
+            drv.inputSrcs.insert(string(path, 1));
+
+        /* Handle derivation outputs of the form ‘!<name>!<path>’. */
+        else if (path.at(0) == '!') {
+            size_t index = path.find("!", 1);
+            drv.inputDrvs[string(path, index + 1)].insert(string(path, 1, index - 1));
         }
 
-        assert(isStorePath(path));
+        /* Handle derivation contexts returned by
+           ‘builtins.storePath’. */
+        else if (isDerivation(path))
+            drv.inputDrvs[path] = store->queryDerivationOutputNames(path);
 
-        debug(format("derivation uses `%1%'") % path);
-        if (!useDrvAsSrc && isDerivation(path))
-            drv.inputDrvs[path] = singleton<StringSet>("out");
+        /* Otherwise it's a source file. */
         else
             drv.inputSrcs.insert(path);
     }
@@ -447,10 +451,8 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
     state.mkAttrs(v, 1 + drv.outputs.size());
     mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton<PathSet>("=" + drvPath));
     foreach (DerivationOutputs::iterator, i, drv.outputs) {
-        /* The output path of an output X is ‘<X>Path’,
-           e.g. ‘outPath’. */
-        mkString(*state.allocAttr(v, state.symbols.create(i->first + "Path")),
-            i->second.path, singleton<PathSet>(drvPath));
+        mkString(*state.allocAttr(v, state.symbols.create(i->first)),
+            i->second.path, singleton<PathSet>("!" + i->first + "!" + drvPath));
     }
     v.attrs->sort();
 }
@@ -1042,15 +1044,6 @@ void EvalState::createBaseEnv()
     addPrimOp("__getEnv", 1, prim_getEnv);
     addPrimOp("__trace", 2, prim_trace);
 
-    // Derivations
-    addPrimOp("derivationStrict", 1, prim_derivationStrict);
-
-    /* Add a wrapper around the derivation primop that computes the
-       `drvPath' and `outPath' attributes lazily. */
-    string s = "attrs: let res = derivationStrict attrs; in attrs // { drvPath = res.drvPath; outPath = res.outPath; type = \"derivation\"; }";
-    mkThunk_(v, parseExprFromString(s, "/"));
-    addConstant("derivation", v);
-
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
     addPrimOp("__storePath", 1, prim_storePath);
@@ -1099,6 +1092,14 @@ void EvalState::createBaseEnv()
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);
     addPrimOp("__compareVersions", 2, prim_compareVersions);
 
+    // Derivations
+    addPrimOp("derivationStrict", 1, prim_derivationStrict);
+
+    /* Add a wrapper around the derivation primop that computes the
+       `drvPath' and `outPath' attributes lazily. */
+    mkThunk_(v, parseExprFromFile(findFile("nix/derivation.nix")));
+    addConstant("derivation", v);
+
     /* Now that we've added all primops, sort the `builtins' attribute
        set, because attribute lookups expect it to be sorted. */
     baseEnv.values[0]->attrs->sort();
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 149cd8b09783..d8f8826e1990 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -278,10 +278,6 @@ public:
 };
 
 
-MakeError(SubstError, Error)
-MakeError(BuildError, Error) /* denotes a permanent build failure */
-
-
 //////////////////////////////////////////////////////////////////////
 
 
@@ -1982,7 +1978,8 @@ void DerivationGoal::computeClosure()
     }
 
     /* Register each output path as valid, and register the sets of
-       paths referenced by each of them. */
+       paths referenced by each of them.  If there are cycles in the
+       outputs, this will fail. */
     ValidPathInfos infos;
     foreach (DerivationOutputs::iterator, i, drv.outputs) {
         ValidPathInfo info;
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index feaab573ef30..14c8ba0bfe90 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -371,36 +371,6 @@ static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
 }
 
 
-static void dfsVisit(StoreAPI & store, const PathSet & paths,
-    const Path & path, PathSet & visited, Paths & sorted)
-{
-    if (visited.find(path) != visited.end()) return;
-    visited.insert(path);
-    
-    PathSet references;
-    if (store.isValidPath(path))
-        store.queryReferences(path, references);
-    
-    foreach (PathSet::iterator, i, references)
-        /* Don't traverse into paths that don't exist.  That can
-           happen due to substitutes for non-existent paths. */
-        if (*i != path && paths.find(*i) != paths.end())
-            dfsVisit(store, paths, *i, visited, sorted);
-
-    sorted.push_front(path);
-}
-
-
-Paths topoSortPaths(StoreAPI & store, const PathSet & paths)
-{
-    Paths sorted;
-    PathSet visited;
-    foreach (PathSet::const_iterator, i, paths)
-        dfsVisit(store, paths, *i, visited, sorted);
-    return sorted;
-}
-
-
 struct GCLimitReached { };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index cf0e2ad1b13b..771776f6a2c7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -820,6 +820,28 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path)
 }
 
 
+StringSet LocalStore::queryDerivationOutputNames(const Path & path)
+{
+    SQLiteTxn txn(db);
+    
+    SQLiteStmtUse use(stmtQueryDerivationOutputs);
+    stmtQueryDerivationOutputs.bind(queryValidPathId(path));
+    
+    StringSet outputNames;
+    int r;
+    while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
+        const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0);
+        assert(s);
+        outputNames.insert(s);
+    }
+    
+    if (r != SQLITE_DONE)
+        throwSQLiteError(db, format("error getting output names of `%1%'") % path);
+
+    return outputNames;
+}
+
+
 void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
 {
     if (run.pid != -1) return;
@@ -944,12 +966,14 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
     while (1) {
         try {
             SQLiteTxn txn(db);
+            PathSet paths;
     
             foreach (ValidPathInfos::const_iterator, i, infos) {
                 assert(i->hash.type == htSHA256);
                 /* !!! Maybe the registration info should be updated if the
                    path is already valid. */
                 if (!isValidPath(i->path)) addValidPath(*i);
+                paths.insert(i->path);
             }
 
             foreach (ValidPathInfos::const_iterator, i, infos) {
@@ -958,6 +982,12 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
                     addReference(referrer, queryValidPathId(*j));
             }
 
+            /* Do a topological sort of the paths.  This will throw an
+               error if a cycle is detected and roll back the
+               transaction.  Cycles can only occur when a derivation
+               has multiple outputs. */
+            topoSortPaths(*this, paths);
+
             txn.commit();
             break;
         } catch (SQLiteBusy & e) {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 4cb905f67231..2739c4eea69d 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -118,6 +118,8 @@ public:
     PathSet queryValidDerivers(const Path & path);
 
     PathSet queryDerivationOutputs(const Path & path);
+
+    StringSet queryDerivationOutputNames(const Path & path);
     
     PathSet querySubstitutablePaths();
     
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index d4bbccd111ba..4ac0afe844b6 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -97,4 +97,40 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 }
 
  
+static void dfsVisit(StoreAPI & store, const PathSet & paths,
+    const Path & path, PathSet & visited, Paths & sorted,
+    PathSet & parents)
+{
+    if (parents.find(path) != parents.end())
+        throw BuildError(format("cycle detected in the references of `%1%'") % path);
+    
+    if (visited.find(path) != visited.end()) return;
+    visited.insert(path);
+    parents.insert(path);
+    
+    PathSet references;
+    if (store.isValidPath(path))
+        store.queryReferences(path, references);
+    
+    foreach (PathSet::iterator, i, references)
+        /* Don't traverse into paths that don't exist.  That can
+           happen due to substitutes for non-existent paths. */
+        if (*i != path && paths.find(*i) != paths.end())
+            dfsVisit(store, paths, *i, visited, sorted, parents);
+
+    sorted.push_front(path);
+    parents.erase(path);
+}
+
+
+Paths topoSortPaths(StoreAPI & store, const PathSet & paths)
+{
+    Paths sorted;
+    PathSet visited, parents;
+    foreach (PathSet::const_iterator, i, paths)
+        dfsVisit(store, paths, *i, visited, sorted, parents);
+    return sorted;
+}
+
+
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 942c5bcf1c7c..6e9921ede79f 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -326,6 +326,16 @@ PathSet RemoteStore::queryDerivationOutputs(const Path & path)
 }
 
 
+PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
+{
+    openConnection();
+    writeInt(wopQueryDerivationOutputNames, to);
+    writeString(path, to);
+    processStderr();
+    return readStrings<PathSet>(from);
+}
+
+
 Path RemoteStore::addToStore(const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter)
 {
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 34a2d91dfcd1..c5853ef53646 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -41,6 +41,8 @@ public:
     
     PathSet queryDerivationOutputs(const Path & path);
     
+    StringSet queryDerivationOutputNames(const Path & path);
+
     bool hasSubstitutes(const Path & path);
     
     bool querySubstitutablePathInfo(const Path & path,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d4997c886207..61bcaf50507f 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -140,6 +140,9 @@ public:
 
     /* Query the outputs of the derivation denoted by `path'. */
     virtual PathSet queryDerivationOutputs(const Path & path) = 0;
+
+    /* Query the output names of the derivation denoted by `path'. */
+    virtual StringSet queryDerivationOutputNames(const Path & path) = 0;
     
     /* Query whether a path has substitutes. */
     virtual bool hasSubstitutes(const Path & path) = 0;
@@ -346,6 +349,10 @@ void exportPaths(StoreAPI & store, const Paths & paths,
     bool sign, Sink & sink);
 
 
+MakeError(SubstError, Error)
+MakeError(BuildError, Error) /* denotes a permanent build failure */
+
+
 }
 
 
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index 760d08a747ae..ef1e0993df28 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -39,6 +39,7 @@ typedef enum {
     wopClearFailedPaths = 25,
     wopQueryPathInfo = 26,
     wopImportPaths = 27,
+    wopQueryDerivationOutputNames = 28,
 } WorkerOp;
 
 
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index e92ccb153d12..b9f8d927a3a6 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -50,26 +50,30 @@ static Path useDeriver(Path path)
 }
 
 
-/* Realisation the given path.  For a derivation that means build it;
-   for other paths it means ensure their validity. */
-static Path realisePath(const Path & path)
+/* Realise the given path.  For a derivation that means build it; for
+   other paths it means ensure their validity. */
+static PathSet realisePath(const Path & path)
 {
     if (isDerivation(path)) {
-        PathSet paths;
-        paths.insert(path);
-        store->buildDerivations(paths);
-        Path outPath = findOutput(derivationFromPath(*store, path), "out");
-        
-        if (gcRoot == "")
-            printGCWarning();
-        else
-            outPath = addPermRoot(*store, outPath,
-                makeRootName(gcRoot, rootNr), indirectRoot);
-        
-        return outPath;
-    } else {
+        store->buildDerivations(singleton<PathSet>(path));
+        Derivation drv = derivationFromPath(*store, path);
+
+        PathSet outputs;
+        foreach (DerivationOutputs::iterator, i, drv.outputs) {
+            Path outPath = i->second.path;
+            if (gcRoot == "")
+                printGCWarning();
+            else
+                outPath = addPermRoot(*store, outPath,
+                    makeRootName(gcRoot, rootNr), indirectRoot);
+            outputs.insert(outPath);
+        }
+        return outputs;
+    }
+
+    else {
         store->ensurePath(path);
-        return path;
+        return singleton<PathSet>(path);
     }
 }
 
@@ -96,8 +100,11 @@ static void opRealise(Strings opFlags, Strings opArgs)
         if (isDerivation(*i)) drvPaths.insert(*i);
     store->buildDerivations(drvPaths);
 
-    foreach (Strings::iterator, i, opArgs)
-        cout << format("%1%\n") % realisePath(*i);
+    foreach (Strings::iterator, i, opArgs) {
+        PathSet paths = realisePath(*i);
+        foreach (PathSet::iterator, j, paths)
+            cout << format("%1%\n") % *j;
+    }
 }
 
 
@@ -157,14 +164,17 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 }
 
 
-static Path maybeUseOutput(const Path & storePath, bool useOutput, bool forceRealise)
+static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise)
 {
     if (forceRealise) realisePath(storePath);
     if (useOutput && isDerivation(storePath)) {
         Derivation drv = derivationFromPath(*store, storePath);
-        return findOutput(drv, "out");
+        PathSet outputs;
+        foreach (DerivationOutputs::iterator, i, drv.outputs)
+            outputs.insert(i->second.path);
+        return outputs;
     }
-    else return storePath;
+    else return singleton<PathSet>(storePath);
 }
 
 
@@ -257,7 +267,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 *i = followLinksToStorePath(*i);
                 if (forceRealise) realisePath(*i);
                 Derivation drv = derivationFromPath(*store, *i);
-                cout << format("%1%\n") % findOutput(drv, "out");
+                foreach (DerivationOutputs::iterator, j, drv.outputs)
+                    cout << format("%1%\n") % j->second.path;
             }
             break;
         }
@@ -268,11 +279,13 @@ static void opQuery(Strings opFlags, Strings opArgs)
         case qReferrersClosure: {
             PathSet paths;
             foreach (Strings::iterator, i, opArgs) {
-                Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise);
-                if (query == qRequisites) computeFSClosure(*store, path, paths, false, includeOutputs);
-                else if (query == qReferences) store->queryReferences(path, paths);
-                else if (query == qReferrers) store->queryReferrers(path, paths);
-                else if (query == qReferrersClosure) computeFSClosure(*store, path, paths, true);
+                PathSet ps = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
+                foreach (PathSet::iterator, j, ps) {
+                    if (query == qRequisites) computeFSClosure(*store, *j, paths, false, includeOutputs);
+                    else if (query == qReferences) store->queryReferences(*j, paths);
+                    else if (query == qReferrers) store->queryReferrers(*j, paths);
+                    else if (query == qReferrersClosure) computeFSClosure(*store, *j, paths, true);
+                }
             }
             Paths sorted = topoSortPaths(*store, paths);
             for (Paths::reverse_iterator i = sorted.rbegin(); 
@@ -304,13 +317,15 @@ static void opQuery(Strings opFlags, Strings opArgs)
         case qHash:
         case qSize:
             foreach (Strings::iterator, i, opArgs) {
-                Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise);
-                ValidPathInfo info = store->queryPathInfo(path);
-                if (query == qHash) {
-                    assert(info.hash.type == htSHA256);
-                    cout << format("sha256:%1%\n") % printHash32(info.hash);
-                } else if (query == qSize) 
-                    cout << format("%1%\n") % info.narSize;
+                PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
+                foreach (PathSet::iterator, j, paths) {
+                    ValidPathInfo info = store->queryPathInfo(*j);
+                    if (query == qHash) {
+                        assert(info.hash.type == htSHA256);
+                        cout << format("sha256:%1%\n") % printHash32(info.hash);
+                    } else if (query == qSize) 
+                        cout << format("%1%\n") % info.narSize;
+                }
             }
             break;
 
@@ -323,16 +338,20 @@ static void opQuery(Strings opFlags, Strings opArgs)
             
         case qGraph: {
             PathSet roots;
-            foreach (Strings::iterator, i, opArgs)
-                roots.insert(maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise));
+            foreach (Strings::iterator, i, opArgs) {
+                PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
+                roots.insert(paths.begin(), paths.end());
+            }
             printDotGraph(roots);
             break;
         }
 
         case qXml: {
             PathSet roots;
-            foreach (Strings::iterator, i, opArgs)
-                roots.insert(maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise));
+            foreach (Strings::iterator, i, opArgs) {
+                PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
+                roots.insert(paths.begin(), paths.end());
+            }
             printXmlGraph(roots);
             break;
         }
@@ -345,10 +364,11 @@ static void opQuery(Strings opFlags, Strings opArgs)
             
         case qRoots: {
             PathSet referrers;
-            foreach (Strings::iterator, i, opArgs)
-                computeFSClosure(*store,
-                    maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise),
-                    referrers, true);
+            foreach (Strings::iterator, i, opArgs) {
+                PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise);
+                foreach (PathSet::iterator, j, paths)
+                    computeFSClosure(*store, *j, referrers, true);
+            }
             Roots roots = store->findRoots();
             foreach (Roots::iterator, i, roots)
                 if (referrers.find(i->second) != referrers.end())
diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc
index 5f57b2981d9a..f28905a24b8d 100644
--- a/src/nix-worker/nix-worker.cc
+++ b/src/nix-worker/nix-worker.cc
@@ -331,6 +331,16 @@ static void performOp(unsigned int clientVersion,
         break;
     }
 
+    case wopQueryDerivationOutputNames: {
+        Path path = readStorePath(from);
+        startWork();
+        StringSet names;
+        names = store->queryDerivationOutputNames(path);
+        stopWork();
+        writeStrings(names, to);
+        break;
+    }
+
     case wopQueryDeriver: {
         Path path = readStorePath(from);
         startWork();