diff options
Diffstat (limited to 'scripts/nix-push.in')
| -rwxr-xr-x | scripts/nix-push.in | 293 |
1 files changed, 293 insertions, 0 deletions
diff --git a/scripts/nix-push.in b/scripts/nix-push.in new file mode 100755 index 000000000000..bdd128a6f5c2 --- /dev/null +++ b/scripts/nix-push.in @@ -0,0 +1,293 @@ +#! @perl@ -w @perlFlags@ + +use strict; +use File::Basename; +use File::Temp qw(tempdir); +use File::Path qw(mkpath); +use File::stat; +use File::Copy; +use Nix::Config; +use Nix::Store; +use Nix::Manifest; +use Nix::Utils; +use Nix::Crypto; + +my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1) + or die "cannot create a temporary directory"; + +my $nixExpr = "$tmpDir/create-nars.nix"; + + +# Parse the command line. +my $compressionType = "xz"; +my $force = 0; +my $destDir; +my $writeManifest = 0; +my $manifestPath; +my $archivesURL; +my $link = 0; +my $privateKeyFile; +my $keyName; +my @roots; + +for (my $n = 0; $n < scalar @ARGV; $n++) { + my $arg = $ARGV[$n]; + + if ($arg eq "--help") { + exec "man nix-push" or die; + } elsif ($arg eq "--bzip2") { + $compressionType = "bzip2"; + } elsif ($arg eq "--none") { + $compressionType = "none"; + } elsif ($arg eq "--force") { + $force = 1; + } elsif ($arg eq "--dest") { + $n++; + die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; + $destDir = $ARGV[$n]; + mkpath($destDir, 0, 0755); + } elsif ($arg eq "--manifest") { + $writeManifest = 1; + } elsif ($arg eq "--manifest-path") { + $n++; + die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; + $manifestPath = $ARGV[$n]; + $writeManifest = 1; + mkpath(dirname($manifestPath), 0, 0755); + } elsif ($arg eq "--url-prefix") { + $n++; + die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; + $archivesURL = $ARGV[$n]; + } elsif ($arg eq "--link") { + $link = 1; + } elsif ($arg eq "--key") { + $n++; + die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; + $privateKeyFile = $ARGV[$n]; + } elsif ($arg eq "--key-name") { + $n++; + die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; + $keyName = $ARGV[$n]; + } elsif (substr($arg, 0, 1) eq "-") { + die "$0: unknown flag `$arg'\n"; + } else { + push @roots, $arg; + } +} + +die "$0: please specify a destination directory\n" if !defined $destDir; + +$archivesURL = "file://$destDir" unless defined $archivesURL; + + +# From the given store paths, determine the set of requisite store +# paths, i.e, the paths required to realise them. +my %storePaths; + +foreach my $path (@roots) { + # Get all paths referenced by the normalisation of the given + # Nix expression. + my $pid = open(READ, + "$Nix::Config::binDir/nix-store --query --requisites --force-realise " . + "--include-outputs '$path'|") or die; + + while (<READ>) { + chomp; + die "bad: $_" unless /^\//; + $storePaths{$_} = ""; + } + + close READ or die "nix-store failed: $?"; +} + +my @storePaths = keys %storePaths; + + +# Don't create archives for files that are already in the binary cache. +my @storePaths2; +my %narFiles; +foreach my $storePath (@storePaths) { + my $pathHash = substr(basename($storePath), 0, 32); + my $narInfoFile = "$destDir/$pathHash.narinfo"; + if (-e $narInfoFile) { + my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read `$narInfoFile'\n"; + my $narFile = "$destDir/$narInfo->{url}"; + if (-e $narFile) { + print STDERR "skipping existing $storePath\n"; + # Add the NAR info to $narFiles if we're writing a + # manifest. + $narFiles{$storePath} = [ + { url => ("$archivesURL/" . basename $narInfo->{url}) + , hash => $narInfo->{fileHash} + , size => $narInfo->{fileSize} + , compressionType => $narInfo->{compression} + , narHash => $narInfo->{narHash} + , narSize => $narInfo->{narSize} + , references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}}) + , deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef + } + ] if $writeManifest; + next; + } + } + push @storePaths2, $storePath; +} + + +# Create a list of Nix derivations that turn each path into a Nix +# archive. +open NIX, ">$nixExpr"; +print NIX "["; + +foreach my $storePath (@storePaths2) { + die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/); + + # Construct a Nix expression that creates a Nix archive. + my $nixexpr = + "(import <nix/nar.nix> " . + "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) "; + + print NIX $nixexpr; +} + +print NIX "]"; +close NIX; + + +# Build the Nix expression. +print STDERR "building compressed archives...\n"; +my @narPaths; +my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr -o $tmpDir/result |") + or die "cannot run nix-build"; +while (<READ>) { + chomp; + die unless /^\//; + push @narPaths, $_; +} +close READ or die "nix-build failed: $?"; + + +# Write the cache info file. +my $cacheInfoFile = "$destDir/nix-cache-info"; +if (! -e $cacheInfoFile) { + open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!"; + print FILE "StoreDir: $Nix::Config::storeDir\n"; + print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas" + close FILE; +} + + +# Copy the archives and the corresponding NAR info files. +print STDERR "copying archives...\n"; + +my $totalNarSize = 0; +my $totalCompressedSize = 0; + +for (my $n = 0; $n < scalar @storePaths2; $n++) { + my $storePath = $storePaths2[$n]; + my $narDir = $narPaths[$n]; + my $baseName = basename $storePath; + + # Get info about the store path. + my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1); + + # In some exceptional cases (such as VM tests that use the Nix + # store of the host), the database doesn't contain the hash. So + # compute it. + if ($narHash =~ /^sha256:0*$/) { + my $nar = "$tmpDir/nar"; + system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0 + or die "cannot dump $storePath\n"; + $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`; + die "cannot hash `$nar'" if $? != 0; + chomp $narHash; + $narHash = "sha256:$narHash"; + $narSize = stat("$nar")->size; + unlink $nar or die; + } + + $totalNarSize += $narSize; + + # Get info about the compressed NAR. + open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash"; + my $compressedHash = <HASH>; + chomp $compressedHash; + $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash"; + close HASH; + + my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : ""); + + my $narFile = "$narDir/$narName"; + (-f $narFile) or die "NAR file for $storePath not found"; + + my $compressedSize = stat($narFile)->size; + $totalCompressedSize += $compressedSize; + + printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath, + $compressedSize / (1024 * 1024), $compressedSize / $narSize * 100; + + # Copy the compressed NAR. + my $dst = "$destDir/$narName"; + if (! -f $dst) { + my $tmp = "$destDir/.tmp.$$.$narName"; + if ($link) { + link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n"; + } else { + copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n"; + } + rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; + } + + # Write the info file. + my $info; + $info .= "StorePath: $storePath\n"; + $info .= "URL: $narName\n"; + $info .= "Compression: $compressionType\n"; + $info .= "FileHash: sha256:$compressedHash\n"; + $info .= "FileSize: $compressedSize\n"; + $info .= "NarHash: $narHash\n"; + $info .= "NarSize: $narSize\n"; + $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; + if (defined $deriver) { + $info .= "Deriver: " . basename $deriver . "\n"; + if (isValidPath($deriver)) { + my $drv = derivationFromPath($deriver); + $info .= "System: $drv->{platform}\n"; + } + } + + if (defined $privateKeyFile && defined $keyName) { + my $sig = signString($privateKeyFile, $info); + $info .= "Signature: 1;$keyName;$sig\n"; + } + + my $pathHash = substr(basename($storePath), 0, 32); + + $dst = "$destDir/$pathHash.narinfo"; + if ($force || ! -f $dst) { + my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo"; + open INFO, ">$tmp" or die; + print INFO "$info" or die; + close INFO or die; + rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; + } + + $narFiles{$storePath} = [ + { url => "$archivesURL/$narName" + , hash => "sha256:$compressedHash" + , size => $compressedSize + , compressionType => $compressionType + , narHash => "$narHash" + , narSize => $narSize + , references => join(" ", @{$refs}) + , deriver => $deriver + } + ] if $writeManifest; +} + +printf STDERR "total compressed size %.2f MiB, %.1f%%\n", + $totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100; + + +# Optionally write a manifest. +writeManifest($manifestPath // "$destDir/MANIFEST", \%narFiles, \()) if $writeManifest; |