* Place manifests in /nix/var/nix/manifests.

* Use the new patch downloader.

7 files changed, 250 insertions, 23 deletions

diff --git a/scripts/Makefile.am b/scripts/Makefile.am
index 8a933fe95bbe..20e42e74c5d0 100644
--- a/scripts/Makefile.am
+++ b/scripts/Makefile.am
@@ -4,13 +4,14 @@ bin_SCRIPTS = nix-collect-garbage \
 
 noinst_SCRIPTS = nix-profile.sh
 
-nix-pull nix-push: readmanifest.pm
+nix-pull nix-push: readmanifest.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm
+install-exec-local: readmanifest.pm download-using-manifests.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -20,4 +21,5 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-prefetch-url.in nix-install-package.in \
   nix-channel.in \
   readmanifest.pm.in \
-  nix-build.in
+  nix-build.in \
+  download-using-manifests.pl.in
diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in
new file mode 100644
index 000000000000..058fa9965a33
--- /dev/null
+++ b/scripts/download-using-manifests.pl.in
@@ -0,0 +1,218 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use readmanifest;
+
+my $manifestDir = "@localstatedir@/nix/manifests";
+
+
+# Check the arguments.
+die unless scalar @ARGV == 1;
+my $targetPath = $ARGV[0];
+
+
+# Load all manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+for my $manifest (glob "$manifestDir/*.nixmanifest") {
+    print STDERR "reading $manifest\n";
+    readManifest $manifest, \%narFiles, \%patches, \%successors;
+}
+
+
+# Build a graph of all store paths that might contribute to the
+# construction of $targetPath, and the special node "start".  The
+# edges are either patch operations, or downloads of full NAR files.
+# The latter edges only occur between "start" and a store path.
+
+my %graph;
+
+$graph{"start"} = {d => 0, pred => undef, edges => []};
+
+my @queue = ();
+my $queueFront = 0;
+my %done;
+
+sub addToQueue {
+    my $v = shift;
+    return if defined $done{$v};
+    $done{$v} = 1;
+    push @queue, $v;
+}
+
+sub addNode {
+    my $u = shift;
+    $graph{$u} = {d => 999999999999, pred => undef, edges => []}
+        unless defined $graph{$u};
+}
+
+sub addEdge {
+    my $u = shift;
+    my $v = shift;
+    my $w = shift;
+    my $type = shift;
+    my $info = shift;
+    addNode $u;
+    push @{$graph{$u}->{edges}},
+        {weight => $w, start => $u, end => $v, type => $type, info => $info};
+    my $n = scalar @{$graph{$u}->{edges}};
+}
+
+addToQueue $targetPath;
+
+while ($queueFront < scalar @queue) {
+    my $u = $queue[$queueFront++];
+    print "$u\n";
+
+    addNode $u;
+
+    # If the path already exists, it has distance 0 from the "start"
+    # node.
+    system "nix-store --isvalid '$u' 2> /dev/null";
+    if ($? == 0) {
+        addEdge "start", $u, 0, "present", undef;
+    }
+
+    else {
+
+        # Add patch edges.
+        my $patchList = $patches{$u};
+        foreach my $patch (@{$patchList}) {
+            # !!! this should be cached
+            my $hash = `nix-hash "$patch->{basePath}"`;
+            chomp $hash;
+            print "  MY HASH is $hash\n";
+            if ($hash ne $patch->{baseHash}) {
+                print "  REJECTING PATCH from $patch->{basePath}\n";
+                next;
+            }
+            print "  PATCH from $patch->{basePath}\n";
+            addToQueue $patch->{basePath};
+            addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
+        }
+
+        # Add NAR file edges to the start node.
+        my $narFileList = $narFiles{$u};
+        foreach my $narFile (@{$narFileList}) {
+            print "  NAR from $narFile->{url}\n";
+            addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
+        }
+
+    }
+}
+
+
+# Run Dijkstra's shortest path algorithm to determine the shortest
+# sequence of download and/or patch actions that will produce
+# $targetPath.
+
+sub byDistance { # sort by distance, reversed
+    return -($graph{$a}->{d} <=> $graph{$b}->{d});
+}
+
+my @todo = keys %graph;
+
+while (scalar @todo > 0) {
+
+    # Remove the closest element from the todo list.
+    @todo = sort byDistance @todo;
+    my $u = pop @todo;
+
+    my $u_ = $graph{$u};
+
+    print "IN $u $u_->{d}\n";
+
+    foreach my $edge (@{$u_->{edges}}) {
+        my $v_ = $graph{$edge->{end}};
+        if ($v_->{d} > $u_->{d} + $edge->{weight}) {
+            $v_->{d} = $u_->{d} + $edge->{weight};
+            # Store the edge; to edge->start is actually the
+            # predecessor.
+            $v_->{pred} = $edge; 
+            print "  RELAX $edge->{end} $v_->{d}\n";
+        }
+    }
+}
+
+
+# Retrieve the shortest path from "start" to $targetPath.
+my @path = ();
+my $cur = $targetPath;
+die "don't know how to produce $targetPath\n"
+    unless defined $graph{$targetPath}->{pred};
+while ($cur ne "start") {
+    push @path, $graph{$cur}->{pred};
+    $cur = $graph{$cur}->{pred}->{start};
+}
+
+
+# Traverse the shortest path, perform the actions described by the
+# edges.
+my $curStep = 1;
+my $maxStep = scalar @path;
+
+sub downloadFile {
+    my $url = shift;
+    my $hash = shift;
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`;
+    chomp $hash2;
+    chomp $path;
+    die "hash mismatch" if $hash ne $hash2;
+    return $path;
+}
+
+while (scalar @path > 0) {
+    my $edge = pop @path;
+    my $u = $edge->{start};
+    my $v = $edge->{end};
+
+    print "\n*** Step $curStep/$maxStep: ";
+    $curStep++;
+
+    if ($edge->{type} eq "present") {
+        print "using already present path `$v'\n";
+    }
+
+    elsif ($edge->{type} eq "patch") {
+        my $patch = $edge->{info};
+        print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
+
+        # Download the patch.
+        print "  downloading patch...\n";
+        my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
+
+        # Turn the base path into a NAR archive, to which we can
+        # actually apply the patch.
+        print "  packing base path...\n";
+        system "nix-store --dump $patch->{basePath} > /tmp/nar";
+        die "cannot dump `$patch->{basePath}'" if ($? != 0);
+
+        # Apply the patch.
+        print "  applying patch...\n";
+        system "bspatch /tmp/nar /tmp/nar2 $patchPath";
+        die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
+
+        # Unpack the resulting NAR archive into the target path.
+        print "  unpacking patched archive...\n";
+        system "nix-store --restore $targetPath < /tmp/nar2";
+        die "cannot unpack /tmp/nar2 into `$targetPath'" if ($? != 0);
+    }
+
+    elsif ($edge->{type} eq "narfile") {
+        my $narFile = $edge->{info};
+        print "downloading `$narFile->{url}' into `$v'\n";
+
+        # Download the archive.
+        print "  downloading archive...\n";
+        my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
+
+        # Unpack the archive into the target path.
+        print "  unpacking archive...\n";
+        system "bunzip2 < '$narFilePath' | nix-store --restore '$targetPath'";
+        die "cannot unpack `$narFilePath' into `$targetPath'" if ($? != 0);
+    }
+}
diff --git a/scripts/nix-build.in b/scripts/nix-build.in
index 43d818274b33..43d818274b33 100755..100644
--- a/scripts/nix-build.in
+++ b/scripts/nix-build.in
diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in
index 3c99f454c6f8..3c99f454c6f8 100755..100644
--- a/scripts/nix-channel.in
+++ b/scripts/nix-channel.in
diff --git a/scripts/nix-collect-garbage.in b/scripts/nix-collect-garbage.in
index 44bcc16bbca0..44bcc16bbca0 100755..100644
--- a/scripts/nix-collect-garbage.in
+++ b/scripts/nix-collect-garbage.in
diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in
index d2c38d0ca12e..1a8e2a91127c 100644
--- a/scripts/nix-pull.in
+++ b/scripts/nix-pull.in
@@ -11,7 +11,7 @@ until mkdir $tmpdir, 0777;
 
 my $manifest = "$tmpdir/manifest";
 
-#END { unlink $manifest; rmdir $tmpdir; }
+END { unlink $manifest; rmdir $tmpdir; }
 
 
 # Obtain URLs either from the command line or from a configuration file.
@@ -28,8 +28,22 @@ sub processURL {
     system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
            "'$url' > '$manifest'") == 0
            or die "curl failed: $?";
-    
+
     readManifest $manifest, \%narFiles, \%patches, \%successors;
+
+    my $baseName = "unnamed";
+    if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
+        $baseName = $1;
+    }
+
+    my $hash = `@bindir@/nix-hash --flat '$manifest'`
+        or die "cannot hash `$manifest'";
+    chomp $hash;
+    
+    my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
+    
+    system("mv '$manifest' '$finalPath'") == 0
+        or die "cannot move `$manifest' to `$finalPath";
 }
 
 while (@ARGV) {
@@ -42,18 +56,6 @@ my $size = scalar (keys %narFiles);
 print "$size store paths in manifest\n";
 
 
-# Instantiate a store expression that builds the substitute program
-# (the program that fetches URLs and unpacks them into the store).
-my $nixExpr =
-    "(import @datadir@/nix/corepkgs/nix-pull) " .
-    "{system = \"@system@\";}";
-
-print STDERR "building downloader...\n";
-my $substProgram = `echo '$nixExpr' | @bindir@/nix-store -qnf \$(@bindir@/nix-instantiate -)`
-    or die "cannot instantiate Nix expression";
-chomp $substProgram;
-
-
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
 
@@ -66,10 +68,8 @@ foreach my $storePath (keys %narFiles) {
     my $narFileList = $narFiles{$storePath};
     foreach my $narFile (@{$narFileList}) {
         print WRITE "$storePath\n";
-        print WRITE "$substProgram/fetch\n";
-        print WRITE "2\n";
-        print WRITE "$narFile->{url}\n";
-        print WRITE "$narFile->{hash}\n";
+        print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
+        print WRITE "0\n";
     }
 }
 
diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in
index 8d6694ff2e54..7f41bd55f267 100644
--- a/scripts/readmanifest.pm.in
+++ b/scripts/readmanifest.pm.in
@@ -19,6 +19,7 @@ sub readManifest {
     my $basePath;
     my $baseHash;
     my $patchType;
+    my $narHash;
 
     while (<MANIFEST>) {
         chomp;
@@ -34,6 +35,7 @@ sub readManifest {
                 undef $hash;
                 $size = 999999999;
                 @preds = ();
+                undef $narHash;
 	    }
             elsif (/^patch \{$/) {
                 $type = "patch";
@@ -44,6 +46,7 @@ sub readManifest {
                 undef $basePath;
                 undef $baseHash;
                 undef $patchType;
+                undef $narHash;
             }
             else { die "bad line: $_"; }
         } else {
@@ -71,7 +74,9 @@ sub readManifest {
                     }
                     if (!$found) {
                         push @{$narFileList},
-                            {url => $url, hash => $hash, size => $size};
+                            { url => $url, hash => $hash, size => $size
+                            , narHash => $narHash
+                            };
                     }
                 
                     foreach my $p (@preds) {
@@ -102,6 +107,7 @@ sub readManifest {
                         push @{$patchList},
                             { url => $url, hash => $hash, size => $size
                             , basePath => $basePath, baseHash => $baseHash
+                            , narHash => $narHash
                             };
                     }
                     
@@ -117,11 +123,12 @@ sub readManifest {
             elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
+            elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
 
             # Compatibility;
             elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
             elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; }
-            
+
             else { die "bad line: $_"; }
         }
     }