diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/Makefile.am | 8 | ||||
| -rw-r--r-- | scripts/download-using-manifests.pl.in | 218 | ||||
| -rw-r--r--[-rwxr-xr-x] | scripts/nix-build.in | 0 | ||||
| -rw-r--r--[-rwxr-xr-x] | scripts/nix-channel.in | 0 | ||||
| -rw-r--r--[-rwxr-xr-x] | scripts/nix-collect-garbage.in | 0 | ||||
| -rw-r--r-- | scripts/nix-pull.in | 36 | ||||
| -rw-r--r-- | scripts/readmanifest.pm.in | 11 |
7 files changed, 250 insertions, 23 deletions
diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 8a933fe95bbe..20e42e74c5d0 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -4,13 +4,14 @@ bin_SCRIPTS = nix-collect-garbage \ noinst_SCRIPTS = nix-profile.sh -nix-pull nix-push: readmanifest.pm +nix-pull nix-push: readmanifest.pm download-using-manifests.pl -install-exec-local: readmanifest.pm +install-exec-local: readmanifest.pm download-using-manifests.pl $(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d $(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh $(INSTALL) -d $(DESTDIR)$(libexecdir)/nix $(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix + $(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix $(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix include ../substitute.mk @@ -20,4 +21,5 @@ EXTRA_DIST = nix-collect-garbage.in \ nix-prefetch-url.in nix-install-package.in \ nix-channel.in \ readmanifest.pm.in \ - nix-build.in + nix-build.in \ + download-using-manifests.pl.in diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in new file mode 100644 index 000000000000..058fa9965a33 --- /dev/null +++ b/scripts/download-using-manifests.pl.in @@ -0,0 +1,218 @@ +#! @perl@ -w -I@libexecdir@/nix + +use strict; +use readmanifest; + +my $manifestDir = "@localstatedir@/nix/manifests"; + + +# Check the arguments. +die unless scalar @ARGV == 1; +my $targetPath = $ARGV[0]; + + +# Load all manifests. +my %narFiles; +my %patches; +my %successors; + +for my $manifest (glob "$manifestDir/*.nixmanifest") { + print STDERR "reading $manifest\n"; + readManifest $manifest, \%narFiles, \%patches, \%successors; +} + + +# Build a graph of all store paths that might contribute to the +# construction of $targetPath, and the special node "start". The +# edges are either patch operations, or downloads of full NAR files. +# The latter edges only occur between "start" and a store path. + +my %graph; + +$graph{"start"} = {d => 0, pred => undef, edges => []}; + +my @queue = (); +my $queueFront = 0; +my %done; + +sub addToQueue { + my $v = shift; + return if defined $done{$v}; + $done{$v} = 1; + push @queue, $v; +} + +sub addNode { + my $u = shift; + $graph{$u} = {d => 999999999999, pred => undef, edges => []} + unless defined $graph{$u}; +} + +sub addEdge { + my $u = shift; + my $v = shift; + my $w = shift; + my $type = shift; + my $info = shift; + addNode $u; + push @{$graph{$u}->{edges}}, + {weight => $w, start => $u, end => $v, type => $type, info => $info}; + my $n = scalar @{$graph{$u}->{edges}}; +} + +addToQueue $targetPath; + +while ($queueFront < scalar @queue) { + my $u = $queue[$queueFront++]; + print "$u\n"; + + addNode $u; + + # If the path already exists, it has distance 0 from the "start" + # node. + system "nix-store --isvalid '$u' 2> /dev/null"; + if ($? == 0) { + addEdge "start", $u, 0, "present", undef; + } + + else { + + # Add patch edges. + my $patchList = $patches{$u}; + foreach my $patch (@{$patchList}) { + # !!! this should be cached + my $hash = `nix-hash "$patch->{basePath}"`; + chomp $hash; + print " MY HASH is $hash\n"; + if ($hash ne $patch->{baseHash}) { + print " REJECTING PATCH from $patch->{basePath}\n"; + next; + } + print " PATCH from $patch->{basePath}\n"; + addToQueue $patch->{basePath}; + addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch; + } + + # Add NAR file edges to the start node. + my $narFileList = $narFiles{$u}; + foreach my $narFile (@{$narFileList}) { + print " NAR from $narFile->{url}\n"; + addEdge "start", $u, $narFile->{size}, "narfile", $narFile; + } + + } +} + + +# Run Dijkstra's shortest path algorithm to determine the shortest +# sequence of download and/or patch actions that will produce +# $targetPath. + +sub byDistance { # sort by distance, reversed + return -($graph{$a}->{d} <=> $graph{$b}->{d}); +} + +my @todo = keys %graph; + +while (scalar @todo > 0) { + + # Remove the closest element from the todo list. + @todo = sort byDistance @todo; + my $u = pop @todo; + + my $u_ = $graph{$u}; + + print "IN $u $u_->{d}\n"; + + foreach my $edge (@{$u_->{edges}}) { + my $v_ = $graph{$edge->{end}}; + if ($v_->{d} > $u_->{d} + $edge->{weight}) { + $v_->{d} = $u_->{d} + $edge->{weight}; + # Store the edge; to edge->start is actually the + # predecessor. + $v_->{pred} = $edge; + print " RELAX $edge->{end} $v_->{d}\n"; + } + } +} + + +# Retrieve the shortest path from "start" to $targetPath. +my @path = (); +my $cur = $targetPath; +die "don't know how to produce $targetPath\n" + unless defined $graph{$targetPath}->{pred}; +while ($cur ne "start") { + push @path, $graph{$cur}->{pred}; + $cur = $graph{$cur}->{pred}->{start}; +} + + +# Traverse the shortest path, perform the actions described by the +# edges. +my $curStep = 1; +my $maxStep = scalar @path; + +sub downloadFile { + my $url = shift; + my $hash = shift; + $ENV{"PRINT_PATH"} = 1; + $ENV{"QUIET"} = 1; + my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`; + chomp $hash2; + chomp $path; + die "hash mismatch" if $hash ne $hash2; + return $path; +} + +while (scalar @path > 0) { + my $edge = pop @path; + my $u = $edge->{start}; + my $v = $edge->{end}; + + print "\n*** Step $curStep/$maxStep: "; + $curStep++; + + if ($edge->{type} eq "present") { + print "using already present path `$v'\n"; + } + + elsif ($edge->{type} eq "patch") { + my $patch = $edge->{info}; + print "applying patch `$patch->{url}' to `$u' to create `$v'\n"; + + # Download the patch. + print " downloading patch...\n"; + my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}"; + + # Turn the base path into a NAR archive, to which we can + # actually apply the patch. + print " packing base path...\n"; + system "nix-store --dump $patch->{basePath} > /tmp/nar"; + die "cannot dump `$patch->{basePath}'" if ($? != 0); + + # Apply the patch. + print " applying patch...\n"; + system "bspatch /tmp/nar /tmp/nar2 $patchPath"; + die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0); + + # Unpack the resulting NAR archive into the target path. + print " unpacking patched archive...\n"; + system "nix-store --restore $targetPath < /tmp/nar2"; + die "cannot unpack /tmp/nar2 into `$targetPath'" if ($? != 0); + } + + elsif ($edge->{type} eq "narfile") { + my $narFile = $edge->{info}; + print "downloading `$narFile->{url}' into `$v'\n"; + + # Download the archive. + print " downloading archive...\n"; + my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}"; + + # Unpack the archive into the target path. + print " unpacking archive...\n"; + system "bunzip2 < '$narFilePath' | nix-store --restore '$targetPath'"; + die "cannot unpack `$narFilePath' into `$targetPath'" if ($? != 0); + } +} diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 43d818274b33..43d818274b33 100755..100644 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index 3c99f454c6f8..3c99f454c6f8 100755..100644 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in diff --git a/scripts/nix-collect-garbage.in b/scripts/nix-collect-garbage.in index 44bcc16bbca0..44bcc16bbca0 100755..100644 --- a/scripts/nix-collect-garbage.in +++ b/scripts/nix-collect-garbage.in diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index d2c38d0ca12e..1a8e2a91127c 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -11,7 +11,7 @@ until mkdir $tmpdir, 0777; my $manifest = "$tmpdir/manifest"; -#END { unlink $manifest; rmdir $tmpdir; } +END { unlink $manifest; rmdir $tmpdir; } # Obtain URLs either from the command line or from a configuration file. @@ -28,8 +28,22 @@ sub processURL { system("@curl@ --fail --silent --show-error --location --max-redirs 20 " . "'$url' > '$manifest'") == 0 or die "curl failed: $?"; - + readManifest $manifest, \%narFiles, \%patches, \%successors; + + my $baseName = "unnamed"; + if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component + $baseName = $1; + } + + my $hash = `@bindir@/nix-hash --flat '$manifest'` + or die "cannot hash `$manifest'"; + chomp $hash; + + my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest"; + + system("mv '$manifest' '$finalPath'") == 0 + or die "cannot move `$manifest' to `$finalPath"; } while (@ARGV) { @@ -42,18 +56,6 @@ my $size = scalar (keys %narFiles); print "$size store paths in manifest\n"; -# Instantiate a store expression that builds the substitute program -# (the program that fetches URLs and unpacks them into the store). -my $nixExpr = - "(import @datadir@/nix/corepkgs/nix-pull) " . - "{system = \"@system@\";}"; - -print STDERR "building downloader...\n"; -my $substProgram = `echo '$nixExpr' | @bindir@/nix-store -qnf \$(@bindir@/nix-instantiate -)` - or die "cannot instantiate Nix expression"; -chomp $substProgram; - - # Register all substitutes. print STDERR "registering substitutes...\n"; @@ -66,10 +68,8 @@ foreach my $storePath (keys %narFiles) { my $narFileList = $narFiles{$storePath}; foreach my $narFile (@{$narFileList}) { print WRITE "$storePath\n"; - print WRITE "$substProgram/fetch\n"; - print WRITE "2\n"; - print WRITE "$narFile->{url}\n"; - print WRITE "$narFile->{hash}\n"; + print WRITE "@libexecdir@/nix/download-using-manifests.pl\n"; + print WRITE "0\n"; } } diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index 8d6694ff2e54..7f41bd55f267 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -19,6 +19,7 @@ sub readManifest { my $basePath; my $baseHash; my $patchType; + my $narHash; while (<MANIFEST>) { chomp; @@ -34,6 +35,7 @@ sub readManifest { undef $hash; $size = 999999999; @preds = (); + undef $narHash; } elsif (/^patch \{$/) { $type = "patch"; @@ -44,6 +46,7 @@ sub readManifest { undef $basePath; undef $baseHash; undef $patchType; + undef $narHash; } else { die "bad line: $_"; } } else { @@ -71,7 +74,9 @@ sub readManifest { } if (!$found) { push @{$narFileList}, - {url => $url, hash => $hash, size => $size}; + { url => $url, hash => $hash, size => $size + , narHash => $narHash + }; } foreach my $p (@preds) { @@ -102,6 +107,7 @@ sub readManifest { push @{$patchList}, { url => $url, hash => $hash, size => $size , basePath => $basePath, baseHash => $baseHash + , narHash => $narHash }; } @@ -117,11 +123,12 @@ sub readManifest { elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } + elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } # Compatibility; elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; } - + else { die "bad line: $_"; } } } |