* Really fix the substitute mechanism, i.e., ensure the closure

  invariant by registering references through the manifest.
* Added a test for nix-pull.

16 files changed, 255 insertions, 89 deletions

diff --git a/corepkgs/nar/Makefile.am b/corepkgs/nar/Makefile.am
index 8fb879ae1b08..741d7e9b3768 100644
--- a/corepkgs/nar/Makefile.am
+++ b/corepkgs/nar/Makefile.am
@@ -1,13 +1,11 @@
-all-local: nar.sh unnar.sh
+all-local: nar.sh
 
 install-exec-local:
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_DATA) nar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_PROGRAM) nar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_DATA) unnar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_PROGRAM) unnar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
 
 include ../../substitute.mk
 
-EXTRA_DIST = nar.nix nar.sh.in unnar.nix unnar.sh.in
+EXTRA_DIST = nar.nix nar.sh.in
diff --git a/corepkgs/nar/nar.sh.in b/corepkgs/nar/nar.sh.in
index 11598e6a98f0..ccf5b9e90a6a 100644
--- a/corepkgs/nar/nar.sh.in
+++ b/corepkgs/nar/nar.sh.in
@@ -10,10 +10,6 @@ dst=$out/$(basename $path).nar.bz2
 
 @bzip2@ < tmp > $dst
 
-narHash=$(md5sum -b tmp | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $narHash > $out/nar-hash
+@bindir@/nix-hash -vvvvv --flat --type sha1 --base32 tmp > $out/nar-hash
 
-narbz2Hash=$(md5sum -b $dst | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $narbz2Hash > $out/narbz2-hash
+@bindir@/nix-hash --flat --type sha1 --base32 $dst > $out/narbz2-hash
diff --git a/corepkgs/nar/unnar.nix b/corepkgs/nar/unnar.nix
deleted file mode 100644
index a18e499b24f7..000000000000
--- a/corepkgs/nar/unnar.nix
+++ /dev/null
@@ -1,7 +0,0 @@
-{system, narFile, outPath}: derivation {
-  name = "unnar";
-  builder = ./unnar.sh;
-  system = system;
-  narFile = narFile;
-  outPath = outPath;
-}
diff --git a/corepkgs/nar/unnar.sh.in b/corepkgs/nar/unnar.sh.in
deleted file mode 100644
index 6fab350a1782..000000000000
--- a/corepkgs/nar/unnar.sh.in
+++ /dev/null
@@ -1,4 +0,0 @@
-#! @shell@ -e
-
-echo "unpacking $narFile to $out..."
-@bunzip2@ < $narFile | @bindir@/nix-store --restore "$out"
diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in
index 5d4193590b75..5698f49ae4aa 100644
--- a/scripts/download-using-manifests.pl.in
+++ b/scripts/download-using-manifests.pl.in
@@ -13,7 +13,7 @@ open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
 die unless scalar @ARGV == 1;
 my $targetPath = $ARGV[0];
 
-my $date = `date`;
+my $date = `date` or die;
 chomp $date;
 print LOGFILE "$$ get $targetPath $date\n";
 
@@ -180,7 +180,7 @@ sub downloadFile {
     my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
     chomp $hash2;
     chomp $path;
-    die "hash mismatch" if $hash ne $hash2;
+    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
     return $path;
 }
 
@@ -236,7 +236,7 @@ while (scalar @path > 0) {
 
         # Unpack the archive into the target path.
         print "  unpacking archive...\n";
-        system "@bunzip2@ < '$narFilePath' | nix-store --restore '$v'";
+        system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
         die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
     }
 }
diff --git a/scripts/nix-prefetch-url.in b/scripts/nix-prefetch-url.in
index c4731f3f627b..409c3f992d91 100644
--- a/scripts/nix-prefetch-url.in
+++ b/scripts/nix-prefetch-url.in
@@ -46,10 +46,8 @@ storeExpr=$( \
         {url = $url; outputHashAlgo = \"$hashType\"; outputHash = \"$hash\"; system = \"@system@\";}" \
     | @bindir@/nix-instantiate -)
 
-echo "$storeExpr"    
-
 # Realise it.
-finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
+finalPath=$(@bindir@/nix-store -q --force-realise $storeExpr)
     
 if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
 
diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in
index 1a8e2a91127c..ee90a06e27e7 100644
--- a/scripts/nix-pull.in
+++ b/scripts/nix-pull.in
@@ -13,6 +13,15 @@ my $manifest = "$tmpdir/manifest";
 
 END { unlink $manifest; rmdir $tmpdir; }
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
+$libexecDir = "@libexecdir@" unless defined $libexecDir;
+
+my $localStateDir = $ENV{"NIX_LOCALSTATE_DIR"};
+$localStateDir = "@localstatedir@" unless defined $localStateDir;
+
 
 # Obtain URLs either from the command line or from a configuration file.
 my %narFiles;
@@ -36,11 +45,11 @@ sub processURL {
         $baseName = $1;
     }
 
-    my $hash = `@bindir@/nix-hash --flat '$manifest'`
+    my $hash = `$binDir/nix-hash --flat '$manifest'`
         or die "cannot hash `$manifest'";
     chomp $hash;
     
-    my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
+    my $finalPath = "$localStateDir/nix/manifests/$baseName-$hash.nixmanifest";
     
     system("mv '$manifest' '$finalPath'") == 0
         or die "cannot move `$manifest' to `$finalPath";
@@ -59,7 +68,7 @@ print "$size store paths in manifest\n";
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
 
-my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --substitute")
     or die "cannot run nix-store";
 
 close READ;
@@ -68,8 +77,14 @@ foreach my $storePath (keys %narFiles) {
     my $narFileList = $narFiles{$storePath};
     foreach my $narFile (@{$narFileList}) {
         print WRITE "$storePath\n";
-        print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
+        print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
         print WRITE "0\n";
+        my @references = split " ", $narFile->{references};
+        my $count = scalar @references;
+        print WRITE "$count\n";
+        foreach my $reference (@references) {
+            print WRITE "$reference\n";
+        }
     }
 }
 
@@ -77,16 +92,3 @@ close WRITE;
 
 waitpid $pid, 0;
 $? == 0 or die "nix-store failed";
-
-
-# Register all successors.
-print STDERR "registering successors...\n";
-my @sucs = %successors;
-while (scalar @sucs > 0) {
-    my $n = scalar @sucs;
-    if ($n > 256) { $n = 256 };
-    my @sucs2 = @sucs[0..$n - 1];
-    @sucs = @sucs[$n..scalar @sucs - 1];
-    system "@bindir@/nix-store --successor @sucs2";
-    if ($?) { die "`nix-store --successor' failed"; }
-}
diff --git a/scripts/nix-push.in b/scripts/nix-push.in
index 1c51778f1850..60ccce4edacf 100644
--- a/scripts/nix-push.in
+++ b/scripts/nix-push.in
@@ -1,6 +1,7 @@
 #! @perl@ -w -I@libexecdir@/nix
 
 use strict;
+use IPC::Open2;
 use POSIX qw(tmpnam);
 use readmanifest;
 
@@ -17,32 +18,62 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $dataDir = $ENV{"NIX_DATA_DIR"};
+$dataDir = "@datadir@" unless defined $dataDir;
+
 
 # Parse the command line.
-my $archives_put_url = shift @ARGV;
-my $archives_get_url = shift @ARGV;
-my $manifest_put_url = shift @ARGV;
+my $localCopy;
+my $localArchivesDir;
+my $localManifestFile;
+
+my $archives_put_url;
+my $archives_get_url;
+my $manifest_put_url;
+
+if ($ARGV[0] eq "--copy") {
+    die "syntax: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 1;
+    shift @ARGV;
+    $localArchivesDir = shift @ARGV;
+    $localManifestFile = shift @ARGV;
+}
+else {
+    die "syntax: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL " .
+        "MANIFEST_PUT_URL PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 0;
+    $archives_put_url = shift @ARGV;
+    $archives_get_url = shift @ARGV;
+    $manifest_put_url = shift @ARGV;
+}
 
 
-# From the given store expressions, determine the requisite store
-# paths.
+# From the given store paths, determine the set of requisite store
+# paths, i.e, the paths required to realise them.
 my %storePaths;
 
-foreach my $storeexpr (@ARGV) {
-    die unless $storeexpr =~ /^\//;
+foreach my $path (@ARGV) {
+    die unless $path =~ /^\//;
 
     # Get all paths referenced by the normalisation of the given 
     # Nix expression.
-    system "@bindir@/nix-store --realise $storeexpr > /dev/null";
-    die if ($?);
-
-    open PATHS, "@bindir@/nix-store --query --requisites --include-successors $storeexpr 2> /dev/null |" or die;
-    while (<PATHS>) {
+    my $pid = open2(\*READ, \*WRITE,
+        "$binDir/nix-store --query --requisites --force-realise " .
+        "--include-outputs '$path'") or die;
+    close WRITE;
+    
+    while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
         $storePaths{$_} = "";
     }
-    close PATHS;
+    close READ;
+    
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 my @storePaths = keys %storePaths;
@@ -58,8 +89,7 @@ foreach my $storePath (@storePaths) {
 
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
-        "((import @datadir@/nix/corepkgs/nar/nar.nix) " .
-        # !!! $storePath should be represented as a closure
+        "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
         "{path = \"$storePath\"; system = \"@system@\";}) ";
     
     print NIX $nixexpr;
@@ -72,7 +102,7 @@ close NIX;
 # Instantiate store expressions from the Nix expression.
 my @storeexprs;
 print STDERR "instantiating store expressions...\n";
-open STOREEXPRS, "@bindir@/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
+open STOREEXPRS, "$binDir/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
 while (<STOREEXPRS>) {
     chomp;
     die unless /^\//;
@@ -93,10 +123,7 @@ while (scalar @tmp > 0) {
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    system "@bindir@/nix-store --realise @tmp2 > /dev/null";
-    if ($?) { die "`nix-store --realise' failed"; }
-
-    open NARPATHS, "@bindir@/nix-store --query --list @tmp2 |" or die "cannot run nix";
+    open NARPATHS, "$binDir/nix-store --realise @tmp2 |" or die "cannot run nix-store";
     while (<NARPATHS>) {
         chomp;
         die unless (/^\//);
@@ -142,16 +169,26 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
     
     my $narbz2Size = (stat $narfile)[7];
 
+    my $references = join(" ", split(" ", `$binDir/nix-store --query --references '$storePath'`));
+
+    my $url;
+    if ($localCopy) {
+        $url = "file://$localArchivesDir/$narname";
+    } else {
+        $url = "$archives_get_url/$narname";
+    }
     $narFiles{$storePath} = [
-        { url => $archives_get_url/$narname
+        { url => $url
         , hash => $narbz2Hash
         , size => $narbz2Size
         , narHash => $narHash
+        , hashAlgo => "sha1"
+        , references => $references
         }
     ];
                             
     if ($storePath =~ /\.store$/) {
-        open PREDS, "@bindir@/nix-store --query --predecessors $storePath |" or die "cannot run nix";
+        open PREDS, "$binDir/nix-store --query --predecessors $storePath |" or die "cannot run nix";
         while (<PREDS>) {
             chomp;
             die unless (/^\//);
@@ -170,6 +207,14 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
 writeManifest $manifest, \%narFiles, \%patches, \%successors;
 
 
+sub copyFile {
+    my $src = shift;
+    my $dst = shift;
+    system("cp '$src' '$dst.tmp'") == 0 or die "cannot copy file";
+    rename("$dst.tmp", "$dst") or die "cannot rename file";
+}
+
+
 # Upload the archives.
 print STDERR "uploading archives...\n";
 foreach my $nararchive (@nararchives) {
@@ -177,17 +222,29 @@ foreach my $nararchive (@nararchives) {
     $nararchive =~ /\/([^\/]*)$/;
     my $basename = $1;
 
-    if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
-        print STDERR "  $nararchive\n";
-        system("$curl --show-error --upload-file " .
-               "'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
-            die "curl failed on $nararchive: $?";
+    if ($localCopy) {
+        if (! -f "$localArchivesDir/$basename") {
+            print STDERR "  $nararchive\n";
+            copyFile $nararchive, "$localArchivesDir/$basename";
+        }
+    }
+    else {
+        if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
+            print STDERR "  $nararchive\n";
+            system("$curl --show-error --upload-file " .
+                   "'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
+                   die "curl failed on $nararchive: $?";
+        }
     }
 }
 
 
 # Upload the manifest.
 print STDERR "uploading manifest...\n";
-system("$curl  --show-error --upload-file " .
-       "'$manifest' '$manifest_put_url' > /dev/null") == 0 or
-    die "curl failed on $manifest: $?";
+if ($localCopy) {
+    copyFile $manifest, $localManifestFile;
+} else {
+    system("$curl  --show-error --upload-file " .
+           "'$manifest' '$manifest_put_url' > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
+}
diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in
index 8b9920b798d5..3e74c88f98b7 100644
--- a/scripts/readmanifest.pm.in
+++ b/scripts/readmanifest.pm.in
@@ -49,6 +49,7 @@ sub readManifest {
     my $baseHash;
     my $patchType;
     my $narHash;
+    my $references;
 
     while (<MANIFEST>) {
         chomp;
@@ -70,6 +71,7 @@ sub readManifest {
                 undef $basePath;
                 undef $baseHash;
                 undef $patchType;
+                $references = "";
 	    }
 
         } else {
@@ -98,7 +100,7 @@ sub readManifest {
                     if (!$found) {
                         push @{$narFileList},
                             { url => $url, hash => $hash, size => $size
-                            , narHash => $narHash
+                            , narHash => $narHash, references => $references
                             };
                     }
                 
@@ -127,6 +129,7 @@ sub readManifest {
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
             elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
 
             # Compatibility;
             elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
@@ -153,10 +156,13 @@ sub writeManifest
         foreach my $narFile (@{$narFileList}) {
             print MANIFEST "{\n";
             print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  HashAlgo: $narFile->{hashAlgo}\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
             print MANIFEST "  MD5: $narFile->{hash}\n";
             print MANIFEST "  NarHash: $narFile->{narHash}\n";
             print MANIFEST "  Size: $narFile->{size}\n";
+            print MANIFEST "  References: $narFile->{references}\n"
+                if defined $narFile->{references} && $narFile->{references} ne "";
             foreach my $p (keys %{$successors}) { # !!! quadratic
                 if ($$successors{$p} eq $storePath) {
                     print MANIFEST "  SuccOf: $p\n";
@@ -171,6 +177,7 @@ sub writeManifest
         foreach my $patch (@{$patchList}) {
             print MANIFEST "patch {\n";
             print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  HashAlgo: $patch->{hashAlgo}\n";
             print MANIFEST "  NarURL: $patch->{url}\n";
             print MANIFEST "  MD5: $patch->{hash}\n";
             print MANIFEST "  NarHash: $patch->{narHash}\n";
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 026721f3b9d8..7dd7412e9f21 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -472,9 +472,8 @@ void DerivationGoal::outputsSubstituted()
 {
     trace("all outputs substituted (maybe)");
 
-    if (nrFailed > 0 && !tryFallback) {
+    if (nrFailed > 0 && !tryFallback)
         throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
-    }
 
     nrFailed = 0;
 
@@ -1252,6 +1251,9 @@ private:
     /* The current substitute. */
     Substitute sub;
 
+    /* Outgoing references for this path. */
+    PathSet references;
+
     /* Pipe for the substitute's standard output/error. */
     Pipe logPipe;
 
@@ -1272,6 +1274,7 @@ public:
 
     /* The states. */
     void init();
+    void referencesValid();
     void tryNext();
     void tryToRun();
     void finished();
@@ -1313,13 +1316,35 @@ void SubstitutionGoal::init()
         return;
     }
 
-    /* !!! build the outgoing references of this path first to
-       maintain the closure invariant! */
-
-    /* Otherwise, get the substitutes. */
+    /* Read the substitutes. */
     subs = querySubstitutes(storePath);
 
-    /* Try the first one. */
+    /* To maintain the closure invairant, we first have to realise the
+       paths referenced by this one. */
+    queryReferences(storePath, references);
+
+    for (PathSet::iterator i = references.begin();
+         i != references.end(); ++i)
+        addWaitee(worker.makeSubstitutionGoal(*i));
+
+    if (waitees.empty()) /* to prevent hang (no wake-up event) */
+        referencesValid();
+    else
+        state = &SubstitutionGoal::referencesValid;
+}
+
+
+void SubstitutionGoal::referencesValid()
+{
+    trace("all referenced realised");
+
+    if (nrFailed > 0)
+        throw Error(format("some references of path `%1%' could not be realised") % storePath);
+
+    for (PathSet::iterator i = references.begin();
+         i != references.end(); ++i)
+        assert(isValidPath(*i));
+    
     tryNext();
 }
 
diff --git a/src/libstore/store.cc b/src/libstore/store.cc
index 97b7f5fe8ab4..02bf9803b29c 100644
--- a/src/libstore/store.cc
+++ b/src/libstore/store.cc
@@ -265,8 +265,8 @@ void setReferences(const Transaction & txn, const Path & storePath,
 void queryReferences(const Path & storePath, PathSet & references)
 {
     Paths references2;
-    if (!isValidPath(storePath))
-        throw Error(format("path `%1%' is not valid") % storePath);
+    //    if (!isValidPath(storePath))
+    //        throw Error(format("path `%1%' is not valid") % storePath);
     nixDB.queryStrings(noTxn, dbReferences, storePath, references2);
     references.insert(references2.begin(), references2.end());
 }
@@ -275,8 +275,8 @@ void queryReferences(const Path & storePath, PathSet & references)
 void queryReferers(const Path & storePath, PathSet & referers)
 {
     Paths referers2;
-    if (!isValidPath(storePath))
-        throw Error(format("path `%1%' is not valid") % storePath);
+    //    if (!isValidPath(storePath))
+    //        throw Error(format("path `%1%' is not valid") % storePath);
     nixDB.queryStrings(noTxn, dbReferers, storePath, referers2);
     referers.insert(referers2.begin(), referers2.end());
 }
diff --git a/src/nix-store/main.cc b/src/nix-store/main.cc
index dc7a6de8b8ec..51e587255003 100644
--- a/src/nix-store/main.cc
+++ b/src/nix-store/main.cc
@@ -177,19 +177,27 @@ static void opSubstitute(Strings opFlags, Strings opArgs)
     while (1) {
         Path srcPath;
         Substitute sub;
+        PathSet references;
         getline(cin, srcPath);
         if (cin.eof()) break;
         getline(cin, sub.program);
         string s;
-        getline(cin, s);
         int n;
+        getline(cin, s);
         if (!string2Int(s, n)) throw Error("number expected");
         while (n--) {
             getline(cin, s);
             sub.args.push_back(s);
         }
+        getline(cin, s);
+        if (!string2Int(s, n)) throw Error("number expected");
+        while (n--) {
+            getline(cin, s);
+            references.insert(s);
+        }
         if (!cin || cin.eof()) throw Error("missing input");
         subPairs.push_back(pair<Path, Substitute>(srcPath, sub));
+        setReferences(txn, srcPath, references);
     }
 
     registerSubstitutes(txn, subPairs);
diff --git a/tests/Makefile.am b/tests/Makefile.am
index d91b021cacf6..9094d39f5041 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -8,7 +8,16 @@ TESTS_ENVIRONMENT = TEST_ROOT=$(TEST_ROOT) \
   NIX_LOG_DIR=$(TEST_ROOT)/log \
   NIX_STATE_DIR=$(TEST_ROOT)/state \
   NIX_DB_DIR=$(TEST_ROOT)/db \
+  NIX_BIN_DIR=$(TEST_ROOT)/bin \
+  NIX_LIBEXEC_DIR=$(TEST_ROOT)/bin \
+  NIX_LOCALSTATE_DIR=$(TEST_ROOT)/state \
+  REAL_BIN_DIR=$(bindir) \
+  REAL_LIBEXEC_DIR=$(libexecdir) \
+  REAL_LOCALSTATE_DIR=$(localstatedir) \
+  REAL_DATA_DIR=$(datadir) \
+  REAL_STORE_DIR=$(storedir) \
   NIX_BUILD_HOOK= \
+  PERL=$(perl) \
   TOP=$(shell pwd)/.. \
   SHARED=$(extra1) \
   aterm_bin=$(aterm_bin) \
@@ -22,9 +31,14 @@ build-hook.sh: build-hook.nix
 substitutes.sh: substitutes.nix
 substitutes2.sh: substitutes2.nix
 fallback.sh: fallback.nix
+nix-push.sh: dependencies.nix
+nix-pull.sh: dependencies.nix
 
-TESTS = init.sh hash.sh lang.sh simple.sh dependencies.sh locking.sh parallel.sh \
-  build-hook.sh substitutes.sh substitutes2.sh fallback.sh verify.sh
+#TESTS = init.sh hash.sh lang.sh simple.sh dependencies.sh locking.sh parallel.sh \
+#  build-hook.sh substitutes.sh substitutes2.sh fallback.sh nix-push.sh verify.sh \
+#  nix-pull.sh
+
+TESTS = init.sh nix-push.sh nix-pull.sh
 
 XFAIL_TESTS =
 
diff --git a/tests/init.sh b/tests/init.sh
index 627cdae1752c..504cb14f12dc 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -13,6 +13,36 @@ mkdir "$NIX_LOG_DIR"
 mkdir "$NIX_STATE_DIR"
 mkdir "$NIX_DB_DIR"
 
+mkdir $NIX_BIN_DIR
+ln -s $TOP/src/nix-store/nix-store $NIX_BIN_DIR/
+ln -s $TOP/src/nix-instantiate/nix-instantiate $NIX_BIN_DIR/
+ln -s $TOP/src/nix-hash/nix-hash $NIX_BIN_DIR/
+ln -s $TOP/scripts/nix-prefetch-url $NIX_BIN_DIR/
+mkdir $NIX_BIN_DIR/nix
+ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/
+ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/
+
+mkdir -p "$NIX_LOCALSTATE_DIR"/nix/manifests
+mkdir -p "$NIX_LOCALSTATE_DIR"/log/nix
+
+mkdir $NIX_DATA_DIR/nix
+cp -prd $TOP/corepkgs $NIX_DATA_DIR/nix/
+# Bah, script has the prefix hard-coded.
+for i in $NIX_DATA_DIR/nix/corepkgs/nar/nar.sh \
+    $NIX_BIN_DIR/nix/download-using-manifests.pl \
+    $NIX_BIN_DIR/nix-prefetch-url \
+    ; do
+    echo "$REAL_BIN_DIR"
+    sed < $i > $i.tmp \
+        -e "s^$REAL_BIN_DIR^$NIX_BIN_DIR^" \
+        -e "s^$REAL_LIBEXEC_DIR^$NIX_LIBEXEC_DIR^" \
+        -e "s^$REAL_LOCALSTATE_DIR^$NIX_LOCALSTATE_DIR^" \
+        -e "s^$REAL_DATA_DIR^$NIX_DATA_DIR^" \
+        -e "s^$REAL_STORE_DIR^$NIX_STORE_DIR^"
+    mv $i.tmp $i
+    chmod +x $i
+done
+
 # Initialise the database.
 $TOP/src/nix-store/nix-store --init
 
diff --git a/tests/nix-pull.sh b/tests/nix-pull.sh
new file mode 100644
index 000000000000..9a17f5bf6026
--- /dev/null
+++ b/tests/nix-pull.sh
@@ -0,0 +1,33 @@
+clearStore () {
+    echo "clearing store..."
+    chmod -R +w "$NIX_STORE_DIR"
+    rm -rf "$NIX_STORE_DIR"
+    mkdir "$NIX_STORE_DIR"
+    rm -rf "$NIX_DB_DIR"
+    mkdir "$NIX_DB_DIR"
+    $TOP/src/nix-store/nix-store --init
+}
+
+pullCache () {
+    echo "pulling cache..."
+    $PERL -w -I$TOP/scripts $TOP/scripts/nix-pull file://$TEST_ROOT/manifest
+}
+
+clearStore
+pullCache
+
+drvPath=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix)
+outPath=$($TOP/src/nix-store/nix-store -q $drvPath)
+
+echo "building $outPath using substitutes..."
+$TOP/src/nix-store/nix-store -r $outPath
+
+cat $outPath/input-2/bar
+
+clearStore
+pullCache
+
+echo "building $drvPath using substitutes..."
+$TOP/src/nix-store/nix-store -r $drvPath
+
+cat $outPath/input-2/bar
diff --git a/tests/nix-push.sh b/tests/nix-push.sh
new file mode 100644
index 000000000000..73499f1e9e35
--- /dev/null
+++ b/tests/nix-push.sh
@@ -0,0 +1,9 @@
+drvPath=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix)
+outPath=$($TOP/src/nix-store/nix-store -r $drvPath)
+
+echo "pushing $drvPath"
+
+mkdir $TEST_ROOT/cache
+
+$PERL -w -I$TOP/scripts $TOP/scripts/nix-push \
+    --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath