diff options
| -rwxr-xr-x | scripts/nix-push.in | 252 |
1 files changed, 93 insertions, 159 deletions
diff --git a/scripts/nix-push.in b/scripts/nix-push.in index a1c02190bd6c..db94b51fd3b2 100755 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -1,10 +1,13 @@ #! @perl@ -w @perlFlags@ use strict; +use File::Basename; use File::Temp qw(tempdir); +use File::Path qw(mkpath); use File::stat; +use File::Copy; use Nix::Config; -use Nix::Manifest; +use Nix::Store; my $hashAlgo = "sha256"; @@ -12,7 +15,6 @@ my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; my $nixExpr = "$tmpDir/create-nars.nix"; -my $manifest = "$tmpDir/MANIFEST"; my $curl = "$Nix::Config::curl --fail --silent"; my $extraCurlFlags = ${ENV{'CURL_FLAGS'}}; @@ -22,18 +24,14 @@ $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags; # Parse the command line. my $localCopy; my $localArchivesDir; -my $localManifestFile; - -my $targetArchivesUrl; my $archivesPutURL; my $archivesGetURL; -my $manifestPutURL; sub showSyntax { print STDERR <<EOF -Usage: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS... - or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL MANIFEST_PUT_URL PATHS... +Usage: nix-push --copy ARCHIVES_DIR PATHS... + or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL PATHS... `nix-push' copies or uploads the closure of PATHS to the given destination. @@ -45,25 +43,16 @@ EOF showSyntax if scalar @ARGV < 1; if ($ARGV[0] eq "--copy") { - showSyntax if scalar @ARGV < 3; + showSyntax if scalar @ARGV < 2; $localCopy = 1; shift @ARGV; $localArchivesDir = shift @ARGV; - $localManifestFile = shift @ARGV; - if ($ARGV[0] eq "--target") { - shift @ARGV; - $targetArchivesUrl = shift @ARGV; - } - else { - $targetArchivesUrl = "file://$localArchivesDir"; - } -} -else { - showSyntax if scalar @ARGV < 3; + mkpath($localArchivesDir, 0, 0755); +} else { + showSyntax if scalar @ARGV < 2; $localCopy = 0; $archivesPutURL = shift @ARGV; $archivesGetURL = shift @ARGV; - $manifestPutURL = shift @ARGV; } @@ -92,8 +81,8 @@ foreach my $path (@ARGV) { my @storePaths = keys %storePaths; -# For each path, create a Nix expression that turns the path into -# a Nix archive. +# Create a list of Nix derivations that turn each path into a Nix +# archive. open NIX, ">$nixExpr"; print NIX "["; @@ -112,172 +101,117 @@ print NIX "]"; close NIX; -# Instantiate store derivations from the Nix expression. -my @storeExprs; -print STDERR "instantiating store derivations...\n"; -my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|") - or die "cannot run nix-instantiate"; +# Build the Nix expression. +print STDERR "building compressed archives...\n"; +my @narPaths; +my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr|") + or die "cannot run nix-build"; while (<READ>) { chomp; die unless /^\//; - push @storeExprs, $_; -} -close READ or die "nix-instantiate failed: $?"; - - -# Build the derivations. -print STDERR "creating archives...\n"; - -my @narPaths; - -my @tmp = @storeExprs; -while (scalar @tmp > 0) { - my $n = scalar @tmp; - if ($n > 256) { $n = 256 }; - my @tmp2 = @tmp[0..$n - 1]; - @tmp = @tmp[$n..scalar @tmp - 1]; - - my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|") - or die "cannot run nix-store"; - while (<READ>) { - chomp; - die unless (/^\//); - push @narPaths, "$_"; - } - close READ or die "nix-store failed: $?"; + push @narPaths, $_; } +close READ or die "nix-build failed: $?"; -# Create the manifest. -print STDERR "creating manifest...\n"; +# Upload the archives and the corresponding info files. +print STDERR "uploading/copying archives...\n"; -my %narFiles; -my %patches; +my $totalNarSize = 0; +my $totalNarBz2Size = 0; -my @narArchives; for (my $n = 0; $n < scalar @storePaths; $n++) { my $storePath = $storePaths[$n]; my $narDir = $narPaths[$n]; - - $storePath =~ /\/([^\/]*)$/; - my $basename = $1; - defined $basename or die; - - open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash"; - my $narbz2Hash = <HASH>; - chomp $narbz2Hash; - $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash"; - close HASH; - - my $narName = "$narbz2Hash.nar.bz2"; - - my $narFile = "$narDir/$narName"; - (-f $narFile) or die "narfile for $storePath not found"; - push @narArchives, $narFile; - - my $narbz2Size = stat($narFile)->size; - - my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`; - die "cannot query references for `$storePath'" if $? != 0; - $references = join(" ", split(" ", $references)); - - my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`; - die "cannot query deriver for `$storePath'" if $? != 0; - chomp $deriver; - $deriver = "" if $deriver eq "unknown-deriver"; + my $baseName = basename $storePath; - my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`; - die "cannot query hash for `$storePath'" if $? != 0; - chomp $narHash; + # Get info about the store path. + my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath); # In some exceptional cases (such as VM tests that use the Nix # store of the host), the database doesn't contain the hash. So # compute it. if ($narHash =~ /^sha256:0*$/) { - $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`; - die "cannot hash `$storePath'" if $? != 0; + my $nar = "$tmpDir/nar"; + system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0 + or die "cannot dump $storePath\n"; + $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --flat $nar`; + die "cannot hash `$nar'" if $? != 0; chomp $narHash; $narHash = "sha256:$narHash"; + $narSize = stat("$nar")->size; + unlink $nar or die; } - my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`; - die "cannot query size for `$storePath'" if $? != 0; - chomp $narSize; - - my $url; - if ($localCopy) { - $url = "$targetArchivesUrl/$narName"; - } else { - $url = "$archivesGetURL/$narName"; - } - $narFiles{$storePath} = [ - { url => $url - , hash => "$hashAlgo:$narbz2Hash" - , size => $narbz2Size - , narHash => "$narHash" - , narSize => $narSize - , references => $references - , deriver => $deriver - } - ]; -} - -writeManifest $manifest, \%narFiles, \%patches; - - -sub copyFile { - my $src = shift; - my $dst = shift; - my $tmp = "$dst.tmp.$$"; - system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file"; - rename($tmp, $dst) or die "cannot rename file: $!"; -} - + $totalNarSize += $narSize; + + # Get info about the compressed NAR. + open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash"; + my $narBz2Hash = <HASH>; + chomp $narBz2Hash; + $narBz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash"; + close HASH; -# Upload/copy the archives. -print STDERR "uploading/copying archives...\n"; + my $narName = "$narBz2Hash.nar.bz2"; -sub archiveExists { - my $name = shift; - print STDERR " HEAD on $archivesGetURL/$name\n"; - return system("$curl --head $archivesGetURL/$name > /dev/null") == 0; -} + my $narFile = "$narDir/$narName"; + (-f $narFile) or die "NAR file for $storePath not found"; -foreach my $narArchive (@narArchives) { + my $narBz2Size = stat($narFile)->size; + $totalNarBz2Size += $narBz2Size; - $narArchive =~ /\/([^\/]*)$/; - my $basename = $1; + printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath, + $narBz2Size / (1024 * 1024), $narBz2Size / $narSize * 100; + # Upload the compressed NAR. if ($localCopy) { - # Since nix-push creates $dst atomically, if it exists we - # don't have to copy again. - my $dst = "$localArchivesDir/$basename"; - if (! -f "$localArchivesDir/$basename") { - print STDERR " $narArchive\n"; - copyFile $narArchive, $dst; + my $dst = "$localArchivesDir/$narName"; + if (! -f $dst) { + my $tmp = "$localArchivesDir/.tmp.$$.$narName"; + copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n"; + rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; } + } else { + die "unimplemented"; + #if (!archiveExists("$basename")) { + # system("$curl --show-error --upload-file " . + # "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or + # die "curl failed on $narArchive: $?"; + #} } - else { - if (!archiveExists("$basename")) { - print STDERR " $narArchive\n"; - system("$curl --show-error --upload-file " . - "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or - die "curl failed on $narArchive: $?"; + + # Upload the info file. + my $info; + $info .= "StorePath: $storePath\n"; + $info .= "URL: $narName\n"; + $info .= "CompressedHash: sha256:$narBz2Hash\n"; + $info .= "CompressedSize: $narBz2Size\n"; + $info .= "NarHash: $narHash\n"; + $info .= "NarSize: $narSize\n"; + $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; + if (defined $deriver) { + $info .= "Deriver: " . basename $deriver, "\n"; + if (isValidPath($deriver)) { + my $drv = derivationFromPath($deriver); + $info .= "System: $drv->{platform}\n"; } } -} - -# Upload the manifest. -print STDERR "uploading manifest...\n"; -if ($localCopy) { - copyFile $manifest, $localManifestFile; - copyFile "$manifest.bz2", "$localManifestFile.bz2"; -} else { - system("$curl --show-error --upload-file " . - "'$manifest' '$manifestPutURL' > /dev/null") == 0 or - die "curl failed on $manifest: $?"; - system("$curl --show-error --upload-file " . - "'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or - die "curl failed on $manifest: $?"; + my $infoName = hashString("sha256", 1, $storePath); + + if ($localCopy) { + my $dst = "$localArchivesDir/$infoName.narinfo"; + if (! -f $dst) { + my $tmp = "$localArchivesDir/.tmp.$$.$infoName"; + open INFO, ">$tmp" or die; + print INFO "$info" or die; + close INFO or die; + rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; + } + } else { + die "unimplemented"; + } } + +printf STDERR "total compressed size %.2f MiB, %.1f%%\n", + $totalNarBz2Size / (1024 * 1024), $totalNarBz2Size / $totalNarSize * 100; |