* Substitutes and nix-pull now work again.

* Fixed a segfault caused by the buffering of stderr.
* Fix now allows the specification of the full output path.  This
  should be used with great care, since it by-passes the normal hash
  generation.
* Incremented the version number to 0.4 (prerelease).

12 files changed, 174 insertions, 94 deletions

diff --git a/configure.ac b/configure.ac
index 8e4a9d12b2ed..8c878f2e30a5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT(nix, "0.3")
+AC_INIT(nix, "0.4")
 AC_CONFIG_SRCDIR(src/nix.cc)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE
diff --git a/corepkgs/nar/unnar.fix b/corepkgs/nar/unnar.fix
index 315be5873552..cd5079e50a05 100644
--- a/corepkgs/nar/unnar.fix
+++ b/corepkgs/nar/unnar.fix
@@ -1,9 +1,9 @@
-Function(["nar", "name"],
+Function(["nar", "outPath"],
   Package(
-    [ ("name", Var("name"))
+    [ ("name", "unnar")
+    , ("outPath", Var("outPath"))
     , ("build", Relative("nar/unnar.sh"))
     , ("nar", Var("nar"))
-    , ("id", Var("id"))
     ]
   )
 )
\ No newline at end of file
diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in
index a3d23ea167af..8cd276801ee6 100644
--- a/scripts/nix-pull.in
+++ b/scripts/nix-pull.in
@@ -2,11 +2,18 @@
 
 use strict;
 use IPC::Open2;
+use POSIX qw(tmpnam);
 
-my $tmpfile = "@localstatedir@/nix/pull.tmp";
+my $tmpdir;
+do { $tmpdir = tmpnam(); }
+until mkdir $tmpdir, 0777;
+
+my $manifest = "$tmpdir/manifest";
 my $conffile = "@sysconfdir@/nix/prebuilts.conf";
 
-my @ids;
+#END { unlink $manifest; rmdir $tmpdir; }
+
+my @srcpaths;
 my @subs;
 my @sucs;
 
@@ -20,70 +27,89 @@ while (<CONFFILE>) {
     chomp;
     if (/^\s*(\S+)\s*(\#.*)?$/) {
         my $url = $1;
+	$url =~ s/\/$//;
 
         print "obtaining list of Nix archives at $url...\n";
 
-        system "wget '$url' -O '$tmpfile' 2> /dev/null"; # !!! escape
+        system "wget '$url'/MANIFEST -O '$manifest' 2> /dev/null"; # !!! escape
         if ($?) { die "`wget' failed"; }
         
-        open INDEX, "<$tmpfile";
-
-        while (<INDEX>) {
-            # Get all links to prebuilts, that is, file names of the
-            # form foo-HASH-HASH.tar.bz2.
-            next unless (/HREF=\"([^\"]*)\"/);
-            my $fn = $1;
-            next if $fn =~ /\.\./;
-            next if $fn =~ /\//;
-            next unless $fn =~ /^([0-9a-z]{32})-([0-9a-z]{32})(.*)\.nar\.bz2$/;
-            my $hash = $1;
-            my $id = $2;
-            my $outname = $3;
-            my $fsid;
-            if ($outname =~ /^-/) {
-                next unless $outname =~ /^-((s-([0-9a-z]{32}))?.*)$/;
-                $outname = $1;
-                $fsid = $3;
-            } else {
-                $outname = "unnamed";
-            }
-
-            print STDERR "$id ($outname)\n";
-
-            # Construct a Fix expression that fetches and unpacks a
-            # Nix archive from the network.
-            my $fetch =
-              "App(IncludeFix(\"fetchurl/fetchurl.fix\"), " .
-              "[(\"url\", \"$url/$fn\"), (\"md5\", \"$hash\")])";
-            my $fixexpr = 
-                "App(IncludeFix(\"nar/unnar.fix\"), " .
-                "[ (\"nar\", $fetch)" .
-                ", (\"name\", \"$outname\")" .
-                ", (\"id\", \"$id\")" .
-                "])";
-            
-	    if (!$first) { $fullexpr .= "," };
-	    $first = 0;
-	    $fullexpr .= $fixexpr; # !!! O(n^2)?
-
-	    push @ids, $id;
-
-            # Does the name encode a successor relation?
-            if (defined $fsid) {
-                push @sucs, $fsid;
-                push @sucs, $id;
-            }
+        open MANIFEST, "<$manifest";
+
+	my $inside = 0;
+
+	my $storepath;
+	my $narname;
+	my $hash;
+	my @preds;
+
+        while (<MANIFEST>) {
+	    chomp;
+	    s/\#.*$//g;
+	    next if (/^$/);
+
+	    if (!$inside) {
+		if (/^\{$/) { 
+		    $inside = 1;
+		    undef $storepath;
+		    undef $narname;
+		    undef $hash;
+		    @preds = ();
+		}
+		else { die "bad line: $_"; }
+	    } else {
+		if (/^\}$/) {
+		    $inside = 0;
+		    my $fullurl = "$url/$narname";
+		    print "$storepath\n";
+
+		    # Construct a Fix expression that fetches and unpacks a
+		    # Nix archive from the network.
+		    my $fetch =
+			"App(IncludeFix(\"fetchurl/fetchurl.fix\"), " .
+			"[(\"url\", \"$fullurl\"), (\"md5\", \"$hash\")])";
+		    my $fixexpr = 
+			"App(IncludeFix(\"nar/unnar.fix\"), " .
+			"[ (\"nar\", $fetch)" .
+			", (\"outPath\", \"$storepath\")" .
+			"])";
+		    
+		    if (!$first) { $fullexpr .= "," };
+		    $first = 0;
+		    $fullexpr .= $fixexpr; # !!! O(n^2)?
+
+		    push @srcpaths, $storepath;
+
+		    foreach my $p (@preds) {
+			push @sucs, $p;
+			push @sucs, $storepath;
+		    }
+
+		}
+		elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
+		    $storepath = $1;
+		}
+		elsif (/^\s*NarName:\s*(\S+)\s*$/) {
+		    $narname = $1;
+		}
+		elsif (/^\s*MD5:\s*(\S+)\s*$/) {
+		    $hash = $1;
+		}
+		elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
+		    push @preds, $1;
+		}
+		else { die "bad line: $_"; }
+	    }
         }
 
-        close INDEX;
-
-        unlink $tmpfile;
+        close MANIFEST;
     }
 
 }
 
 $fullexpr .= "]";
 
+
 # Instantiate Nix expressions from the Fix expressions we created above.
 print STDERR "running fix...\n";
 my $pid = open2(\*READ, \*WRITE, "fix -") or die "cannot run fix";
@@ -93,23 +119,28 @@ close WRITE;
 my $i = 0;
 while (<READ>) {
     chomp;
-    die unless /^([0-9a-z]{32})$/;
-    my $nid = $1;
-    die unless ($i < scalar @ids);
-    my $id = $ids[$i++];
-    push @subs, $id;
-    push @subs, $nid;
+    die unless /^\//;
+    my $subpath = $_;
+    die unless ($i < scalar @srcpaths);
+    my $srcpath = $srcpaths[$i++];
+    push @subs, $srcpath;
+    push @subs, $subpath;
+    print "$srcpath $subpath\n";
 }
 
 waitpid $pid, 0;
 $? == 0 or die "fix failed";
 
+
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
+print "@subs\n";
 system "nix --substitute @subs";
 if ($?) { die "`nix --substitute' failed"; }
 
+
 # Register all successors.
 print STDERR "registering successors...\n";
-system "nix --successor @sucs";
+print "@sucs\n";
+system "nix --successor -vvvv @sucs";
 if ($?) { die "`nix --successor' failed"; }
diff --git a/src/dotgraph.cc b/src/dotgraph.cc
index 514fda325914..36daf7f9966d 100644
--- a/src/dotgraph.cc
+++ b/src/dotgraph.cc
@@ -1,4 +1,5 @@
 #include "dotgraph.hh"
+#include "normalise.hh"
 
 
 static string dotQuote(const string & s)
@@ -98,8 +99,8 @@ void printDotGraph(const PathSet & roots)
 
 	if (doneSet.find(nePath) == doneSet.end()) {
 	    doneSet.insert(nePath);
-                    
-	    NixExpr ne = parseNixExpr(termFromPath(nePath));
+
+	    NixExpr ne = exprFromPath(nePath);
 
 	    string label, colour;
                     
diff --git a/src/expr.cc b/src/expr.cc
index cfc4af1f39a8..cead803425ba 100644
--- a/src/expr.cc
+++ b/src/expr.cc
@@ -22,14 +22,6 @@ Hash hashTerm(ATerm t)
 }
 
 
-ATerm termFromPath(const Path & path)
-{
-    ATerm t = ATreadFromNamedFile(path.c_str());
-    if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
-    return t;
-}
-
-
 Path writeTerm(ATerm t, const string & suffix)
 {
     /* The id of a term is its hash. */
diff --git a/src/expr.hh b/src/expr.hh
index b345643223cc..7d0420935f9d 100644
--- a/src/expr.hh
+++ b/src/expr.hh
@@ -53,9 +53,6 @@ Error badTerm(const format & f, ATerm t);
 /* Hash an aterm. */
 Hash hashTerm(ATerm t);
 
-/* Read an aterm from disk. */
-ATerm termFromPath(const Path & path);
-
 /* Write an aterm to the Nix store directory, and return its path. */
 Path writeTerm(ATerm t, const string & suffix);
 
diff --git a/src/fix.cc b/src/fix.cc
index cbf759b31763..c1f9c1ad68d0 100644
--- a/src/fix.cc
+++ b/src/fix.cc
@@ -299,6 +299,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
         ne.type = NixExpr::neDerivation;
         ne.derivation.platform = SYSTEM;
         string name;
+        Path outPath;
         Hash outHash;
         bool outHashGiven = false;
         bnds = ATempty;
@@ -327,6 +328,7 @@ static Expr evalExpr2(EvalState & state, Expr e)
 
                 if (key == "build") ne.derivation.builder = s;
                 if (key == "name") name = s;
+                if (key == "outPath") outPath = s;
                 if (key == "id") { 
                     outHash = parseHash(s);
                     outHashGiven = true;
@@ -343,11 +345,13 @@ static Expr evalExpr2(EvalState & state, Expr e)
         if (name == "")
             throw badTerm("no package name specified", e);
         
-        /* Hash the Nix expression with no outputs to produce a
-           unique but deterministic path name for this package. */
+        /* Determine the output path. */
         if (!outHashGiven) outHash = hashPackage(state, ne);
-        Path outPath = 
-            canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name);
+        if (outPath == "")
+            /* Hash the Nix expression with no outputs to produce a
+               unique but deterministic path name for this package. */
+            outPath = 
+                canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name);
         ne.derivation.env["out"] = outPath;
         ne.derivation.outputs.insert(outPath);
 
diff --git a/src/normalise.cc b/src/normalise.cc
index 160130d96634..be71081ffb25 100644
--- a/src/normalise.cc
+++ b/src/normalise.cc
@@ -122,7 +122,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
     Path nePath = useSuccessor(_nePath);
 
     /* Get the Nix expression. */
-    NixExpr ne = parseNixExpr(termFromPath(nePath));
+    NixExpr ne = exprFromPath(nePath, pending);
 
     /* If this is a normal form (i.e., a closure) we are done. */
     if (ne.type == NixExpr::neClosure) return nePath;
@@ -172,7 +172,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
     {
         Path nePath2 = useSuccessor(nePath);
         if (nePath != nePath2) {
-            NixExpr ne = parseNixExpr(termFromPath(nePath2));
+            NixExpr ne = exprFromPath(nePath2, pending);
             debug(format("skipping build of expression `%1%', someone beat us to it")
 		  % (string) nePath);
             if (ne.type != NixExpr::neClosure) abort();
@@ -193,7 +193,7 @@ Path normaliseNixExpr(const Path & _nePath, PathSet pending)
         realiseClosure(nfPath, pending);
         /* !!! nfPath should be a root of the garbage collector while
            we are building */
-        NixExpr ne = parseNixExpr(termFromPath(nfPath));
+        NixExpr ne = exprFromPath(nfPath, pending);
         if (ne.type != NixExpr::neClosure) abort();
         for (ClosureElems::iterator j = ne.closure.elems.begin();
              j != ne.closure.elems.end(); j++)
@@ -364,16 +364,49 @@ void realiseClosure(const Path & nePath, PathSet pending)
 {
     Nest nest(lvlDebug, format("realising closure `%1%'") % nePath);
 
-    NixExpr ne = parseNixExpr(termFromPath(nePath));
+    NixExpr ne = exprFromPath(nePath, pending);
     if (ne.type != NixExpr::neClosure)
         throw Error(format("expected closure in `%1%'") % nePath);
     
     for (ClosureElems::const_iterator i = ne.closure.elems.begin();
          i != ne.closure.elems.end(); i++)
-        assert(isValidPath(i->first));
-#if 0
-        expandId(i->second.id, i->first, "/", pending);
-#endif
+        ensurePath(i->first, pending);
+}
+
+
+void ensurePath(const Path & path, PathSet pending)
+{
+    /* If the path is already valid, we're done. */
+    if (isValidPath(path)) return;
+    
+    /* Otherwise, try the substitutes. */
+    Paths subPaths = querySubstitutes(path);
+
+    for (Paths::iterator i = subPaths.begin(); 
+         i != subPaths.end(); i++)
+    {
+        try {
+            normaliseNixExpr(*i, pending);
+            if (isValidPath(path)) return;
+            throw Error(format("substitute failed to produce expected output path"));
+        } catch (Error & e) {
+            msg(lvlTalkative, 
+                format("building of substitute `%1%' for `%2%' failed: %3%")
+                % *i % path % e.what());
+        }
+    }
+
+    throw Error(format("path `%1%' is required, "
+        "but there are no (successful) substitutes") % path);
+}
+
+
+NixExpr exprFromPath(const Path & path, PathSet pending)
+{
+    ensurePath(path, pending);
+    ATerm t = ATreadFromNamedFile(path.c_str());
+    if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
+    return parseNixExpr(t);
 }
 
 
@@ -381,7 +414,7 @@ PathSet nixExprRoots(const Path & nePath)
 {
     PathSet paths;
 
-    NixExpr ne = parseNixExpr(termFromPath(nePath));
+    NixExpr ne = exprFromPath(nePath);
 
     if (ne.type == NixExpr::neClosure)
         paths.insert(ne.closure.roots.begin(), ne.closure.roots.end());
@@ -401,7 +434,7 @@ static void requisitesWorker(const Path & nePath,
     if (doneSet.find(nePath) != doneSet.end()) return;
     doneSet.insert(nePath);
 
-    NixExpr ne = parseNixExpr(termFromPath(nePath));
+    NixExpr ne = exprFromPath(nePath);
 
     if (ne.type == NixExpr::neClosure)
         for (ClosureElems::iterator i = ne.closure.elems.begin();
diff --git a/src/normalise.hh b/src/normalise.hh
index e8e72f5bc857..bbe846404cc0 100644
--- a/src/normalise.hh
+++ b/src/normalise.hh
@@ -18,6 +18,14 @@ Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet());
    its output paths through substitutes... kaboom!). */
 void realiseClosure(const Path & nePath, PathSet pending = PathSet());
 
+/* Ensure that a path exists, possibly by instantiating it by
+   realising a substitute. */
+void ensurePath(const Path & path, PathSet pending = PathSet());
+
+/* Read a Nix expression, after ensuring its existence through
+   ensurePath(). */
+NixExpr exprFromPath(const Path & path, PathSet pending = PathSet());
+
 /* Get the list of root (output) paths of the given Nix expression. */
 PathSet nixExprRoots(const Path & nePath);
 
diff --git a/src/shared.cc b/src/shared.cc
index dcda0b50abe6..80463308a3f0 100644
--- a/src/shared.cc
+++ b/src/shared.cc
@@ -47,6 +47,8 @@ static void initAndRun(int argc, char * * argv)
 }
 
 
+static char buf[1024];
+
 int main(int argc, char * * argv)
 {
     /* ATerm setup. */
@@ -54,7 +56,6 @@ int main(int argc, char * * argv)
     ATinit(argc, argv, &bottomOfStack);
 
     /* Turn on buffering for cerr. */
-    char buf[1024];
     cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 
     try {
diff --git a/src/store.cc b/src/store.cc
index 7f10c6377dc6..2d223313b612 100644
--- a/src/store.cc
+++ b/src/store.cc
@@ -175,7 +175,8 @@ void registerSuccessor(const Transaction & txn,
 
     Paths revs;
     nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
-    revs.push_back(srcPath);
+    if (find(revs.begin(), revs.end(), srcPath) == revs.end())
+        revs.push_back(srcPath);
 
     nixDB.setString(txn, dbSuccessors, srcPath, sucPath);
     nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
@@ -212,7 +213,8 @@ void registerSubstitute(const Path & srcPath, const Path & subPath)
 
     Paths revs;
     nixDB.queryStrings(txn, dbSubstitutesRev, subPath, revs);
-    revs.push_back(srcPath);
+    if (find(revs.begin(), revs.end(), srcPath) == revs.end())
+        revs.push_back(srcPath);
     
     nixDB.setStrings(txn, dbSubstitutes, srcPath, subs);
     nixDB.setStrings(txn, dbSubstitutesRev, subPath, revs);
@@ -221,6 +223,14 @@ void registerSubstitute(const Path & srcPath, const Path & subPath)
 }
 
 
+Paths querySubstitutes(const Path & srcPath)
+{
+    Paths subPaths;
+    nixDB.queryStrings(noTxn, dbSubstitutes, srcPath, subPaths);
+    return subPaths;
+}
+
+
 void registerValidPath(const Transaction & txn, const Path & _path)
 {
     Path path(canonPath(_path));
diff --git a/src/store.hh b/src/store.hh
index 3d7575c3e86c..dab3d603f802 100644
--- a/src/store.hh
+++ b/src/store.hh
@@ -42,6 +42,9 @@ Paths queryPredecessors(const Path & sucPath);
 /* Register a substitute. */
 void registerSubstitute(const Path & srcPath, const Path & subPath);
 
+/* Return the substitutes expression for the given path. */
+Paths querySubstitutes(const Path & srcPath);
+
 /* Register the validity of a path. */
 void registerValidPath(const Transaction & txn, const Path & path);