From 49cd7387ad6546571ca31a41f208091b482defaa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Jun 2012 17:19:32 -0400 Subject: nix-push: create a manifest-less binary cache Manifests are a huge pain, since users need to run nix-pull directly or indirectly to obtain them. They tend to be large and lag behind the available binaries; also, the downloaded manifests in /nix/var/nix/manifest need to be in sync with the Nixpkgs sources. So we want to get rid of them. The idea of manifest-free operation works as follows. Nix is configured with a set of URIs of binary caches, e.g. http://nixos.org/binary-cache Whenever Nix needs a store path X, it checks each binary cache for the existence of a file /.narinfo, e.g. http://nixos.org/binary-cache/bi1gh9...ia17.narinfo The .narinfo file contains the necessary information about the store path that was formerly kept in the manifest, i.e., (relative) URI of the compressed NAR, references, size, hash, etc. For example: StorePath: /nix/store/xqp4l88cr9bxv01jinkz861mnc9p7qfi-neon-0.29.6 URL: 1bjxbg52l32wj8ww47sw9f4qz0r8n5vs71l93lcbgk2506v3cpfd.nar.bz2 CompressedHash: sha256:1bjxbg52l32wj8ww47sw9f4qz0r8n5vs71l93lcbgk2506v3cpfd CompressedSize: 202542 NarHash: sha256:1af26536781e6134ab84201b33408759fc59b36cc5530f57c0663f67b588e15f NarSize: 700440 References: 043zrsanirjh8nbc5vqpjn93hhrf107f-bash-4.2-p24 cj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 ... Deriver: 4idz1bgi58h3pazxr3akrw4fsr6zrf3r-neon-0.29.6.drv System: x86_64-linux Nix then knows that it needs to download http://nixos.org/binary-cache/1bjxbg52l32wj8ww47sw9f4qz0r8n5vs71l93lcbgk2506v3cpfd.nar.bz2 to substitute the store path. Note that the store directory is omitted from the References and Deriver fields to save space, and that the URL can be relative to the binary cache prefix. This patch just makes nix-push create binary caches in this format. The next step is to make a substituter that supports them. --- scripts/nix-push.in | 252 +++++++++++++++++++--------------------------------- 1 file changed, 93 insertions(+), 159 deletions(-) (limited to 'scripts') diff --git a/scripts/nix-push.in b/scripts/nix-push.in index a1c02190bd6c..db94b51fd3b2 100755 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -1,10 +1,13 @@ #! @perl@ -w @perlFlags@ use strict; +use File::Basename; use File::Temp qw(tempdir); +use File::Path qw(mkpath); use File::stat; +use File::Copy; use Nix::Config; -use Nix::Manifest; +use Nix::Store; my $hashAlgo = "sha256"; @@ -12,7 +15,6 @@ my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; my $nixExpr = "$tmpDir/create-nars.nix"; -my $manifest = "$tmpDir/MANIFEST"; my $curl = "$Nix::Config::curl --fail --silent"; my $extraCurlFlags = ${ENV{'CURL_FLAGS'}}; @@ -22,18 +24,14 @@ $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags; # Parse the command line. my $localCopy; my $localArchivesDir; -my $localManifestFile; - -my $targetArchivesUrl; my $archivesPutURL; my $archivesGetURL; -my $manifestPutURL; sub showSyntax { print STDERR <$nixExpr"; print NIX "["; @@ -112,172 +101,117 @@ print NIX "]"; close NIX; -# Instantiate store derivations from the Nix expression. -my @storeExprs; -print STDERR "instantiating store derivations...\n"; -my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|") - or die "cannot run nix-instantiate"; +# Build the Nix expression. +print STDERR "building compressed archives...\n"; +my @narPaths; +my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr|") + or die "cannot run nix-build"; while () { chomp; die unless /^\//; - push @storeExprs, $_; -} -close READ or die "nix-instantiate failed: $?"; - - -# Build the derivations. -print STDERR "creating archives...\n"; - -my @narPaths; - -my @tmp = @storeExprs; -while (scalar @tmp > 0) { - my $n = scalar @tmp; - if ($n > 256) { $n = 256 }; - my @tmp2 = @tmp[0..$n - 1]; - @tmp = @tmp[$n..scalar @tmp - 1]; - - my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|") - or die "cannot run nix-store"; - while () { - chomp; - die unless (/^\//); - push @narPaths, "$_"; - } - close READ or die "nix-store failed: $?"; + push @narPaths, $_; } +close READ or die "nix-build failed: $?"; -# Create the manifest. -print STDERR "creating manifest...\n"; +# Upload the archives and the corresponding info files. +print STDERR "uploading/copying archives...\n"; -my %narFiles; -my %patches; +my $totalNarSize = 0; +my $totalNarBz2Size = 0; -my @narArchives; for (my $n = 0; $n < scalar @storePaths; $n++) { my $storePath = $storePaths[$n]; my $narDir = $narPaths[$n]; - - $storePath =~ /\/([^\/]*)$/; - my $basename = $1; - defined $basename or die; - - open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash"; - my $narbz2Hash = ; - chomp $narbz2Hash; - $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash"; - close HASH; - - my $narName = "$narbz2Hash.nar.bz2"; - - my $narFile = "$narDir/$narName"; - (-f $narFile) or die "narfile for $storePath not found"; - push @narArchives, $narFile; - - my $narbz2Size = stat($narFile)->size; - - my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`; - die "cannot query references for `$storePath'" if $? != 0; - $references = join(" ", split(" ", $references)); - - my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`; - die "cannot query deriver for `$storePath'" if $? != 0; - chomp $deriver; - $deriver = "" if $deriver eq "unknown-deriver"; + my $baseName = basename $storePath; - my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`; - die "cannot query hash for `$storePath'" if $? != 0; - chomp $narHash; + # Get info about the store path. + my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath); # In some exceptional cases (such as VM tests that use the Nix # store of the host), the database doesn't contain the hash. So # compute it. if ($narHash =~ /^sha256:0*$/) { - $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`; - die "cannot hash `$storePath'" if $? != 0; + my $nar = "$tmpDir/nar"; + system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0 + or die "cannot dump $storePath\n"; + $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --flat $nar`; + die "cannot hash `$nar'" if $? != 0; chomp $narHash; $narHash = "sha256:$narHash"; + $narSize = stat("$nar")->size; + unlink $nar or die; } - my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`; - die "cannot query size for `$storePath'" if $? != 0; - chomp $narSize; - - my $url; - if ($localCopy) { - $url = "$targetArchivesUrl/$narName"; - } else { - $url = "$archivesGetURL/$narName"; - } - $narFiles{$storePath} = [ - { url => $url - , hash => "$hashAlgo:$narbz2Hash" - , size => $narbz2Size - , narHash => "$narHash" - , narSize => $narSize - , references => $references - , deriver => $deriver - } - ]; -} - -writeManifest $manifest, \%narFiles, \%patches; - - -sub copyFile { - my $src = shift; - my $dst = shift; - my $tmp = "$dst.tmp.$$"; - system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file"; - rename($tmp, $dst) or die "cannot rename file: $!"; -} - + $totalNarSize += $narSize; + + # Get info about the compressed NAR. + open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash"; + my $narBz2Hash = ; + chomp $narBz2Hash; + $narBz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash"; + close HASH; -# Upload/copy the archives. -print STDERR "uploading/copying archives...\n"; + my $narName = "$narBz2Hash.nar.bz2"; -sub archiveExists { - my $name = shift; - print STDERR " HEAD on $archivesGetURL/$name\n"; - return system("$curl --head $archivesGetURL/$name > /dev/null") == 0; -} + my $narFile = "$narDir/$narName"; + (-f $narFile) or die "NAR file for $storePath not found"; -foreach my $narArchive (@narArchives) { + my $narBz2Size = stat($narFile)->size; + $totalNarBz2Size += $narBz2Size; - $narArchive =~ /\/([^\/]*)$/; - my $basename = $1; + printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath, + $narBz2Size / (1024 * 1024), $narBz2Size / $narSize * 100; + # Upload the compressed NAR. if ($localCopy) { - # Since nix-push creates $dst atomically, if it exists we - # don't have to copy again. - my $dst = "$localArchivesDir/$basename"; - if (! -f "$localArchivesDir/$basename") { - print STDERR " $narArchive\n"; - copyFile $narArchive, $dst; + my $dst = "$localArchivesDir/$narName"; + if (! -f $dst) { + my $tmp = "$localArchivesDir/.tmp.$$.$narName"; + copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n"; + rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; } + } else { + die "unimplemented"; + #if (!archiveExists("$basename")) { + # system("$curl --show-error --upload-file " . + # "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or + # die "curl failed on $narArchive: $?"; + #} } - else { - if (!archiveExists("$basename")) { - print STDERR " $narArchive\n"; - system("$curl --show-error --upload-file " . - "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or - die "curl failed on $narArchive: $?"; + + # Upload the info file. + my $info; + $info .= "StorePath: $storePath\n"; + $info .= "URL: $narName\n"; + $info .= "CompressedHash: sha256:$narBz2Hash\n"; + $info .= "CompressedSize: $narBz2Size\n"; + $info .= "NarHash: $narHash\n"; + $info .= "NarSize: $narSize\n"; + $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; + if (defined $deriver) { + $info .= "Deriver: " . basename $deriver, "\n"; + if (isValidPath($deriver)) { + my $drv = derivationFromPath($deriver); + $info .= "System: $drv->{platform}\n"; } } -} - -# Upload the manifest. -print STDERR "uploading manifest...\n"; -if ($localCopy) { - copyFile $manifest, $localManifestFile; - copyFile "$manifest.bz2", "$localManifestFile.bz2"; -} else { - system("$curl --show-error --upload-file " . - "'$manifest' '$manifestPutURL' > /dev/null") == 0 or - die "curl failed on $manifest: $?"; - system("$curl --show-error --upload-file " . - "'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or - die "curl failed on $manifest: $?"; + my $infoName = hashString("sha256", 1, $storePath); + + if ($localCopy) { + my $dst = "$localArchivesDir/$infoName.narinfo"; + if (! -f $dst) { + my $tmp = "$localArchivesDir/.tmp.$$.$infoName"; + open INFO, ">$tmp" or die; + print INFO "$info" or die; + close INFO or die; + rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; + } + } else { + die "unimplemented"; + } } + +printf STDERR "total compressed size %.2f MiB, %.1f%%\n", + $totalNarBz2Size / (1024 * 1024), $totalNarBz2Size / $totalNarSize * 100; -- cgit 1.4.1