about summary refs log tree commit diff
path: root/scripts/nix-push.in
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/nix-push.in')
-rwxr-xr-xscripts/nix-push.in252
1 files changed, 93 insertions, 159 deletions
diff --git a/scripts/nix-push.in b/scripts/nix-push.in
index a1c02190bd6c..db94b51fd3b2 100755
--- a/scripts/nix-push.in
+++ b/scripts/nix-push.in
@@ -1,10 +1,13 @@
 #! @perl@ -w @perlFlags@
 
 use strict;
+use File::Basename;
 use File::Temp qw(tempdir);
+use File::Path qw(mkpath);
 use File::stat;
+use File::Copy;
 use Nix::Config;
-use Nix::Manifest;
+use Nix::Store;
 
 my $hashAlgo = "sha256";
 
@@ -12,7 +15,6 @@ my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
 my $nixExpr = "$tmpDir/create-nars.nix";
-my $manifest = "$tmpDir/MANIFEST";
 
 my $curl = "$Nix::Config::curl --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
@@ -22,18 +24,14 @@ $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 # Parse the command line.
 my $localCopy;
 my $localArchivesDir;
-my $localManifestFile;
-
-my $targetArchivesUrl;
 
 my $archivesPutURL;
 my $archivesGetURL;
-my $manifestPutURL;
 
 sub showSyntax {
     print STDERR <<EOF
-Usage: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...
-   or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL MANIFEST_PUT_URL PATHS...
+Usage: nix-push --copy ARCHIVES_DIR PATHS...
+   or: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL PATHS...
 
 `nix-push' copies or uploads the closure of PATHS to the given
 destination.
@@ -45,25 +43,16 @@ EOF
 showSyntax if scalar @ARGV < 1;
 
 if ($ARGV[0] eq "--copy") {
-    showSyntax if scalar @ARGV < 3;
+    showSyntax if scalar @ARGV < 2;
     $localCopy = 1;
     shift @ARGV;
     $localArchivesDir = shift @ARGV;
-    $localManifestFile = shift @ARGV;
-    if ($ARGV[0] eq "--target") {
-       shift @ARGV;
-       $targetArchivesUrl = shift @ARGV;
-    }
-    else {
-       $targetArchivesUrl = "file://$localArchivesDir";
-    }
-}
-else {
-    showSyntax if scalar @ARGV < 3;
+    mkpath($localArchivesDir, 0, 0755);
+} else {
+    showSyntax if scalar @ARGV < 2;
     $localCopy = 0;
     $archivesPutURL = shift @ARGV;
     $archivesGetURL = shift @ARGV;
-    $manifestPutURL = shift @ARGV;
 }
 
 
@@ -92,8 +81,8 @@ foreach my $path (@ARGV) {
 my @storePaths = keys %storePaths;
 
 
-# For each path, create a Nix expression that turns the path into
-# a Nix archive.
+# Create a list of Nix derivations that turn each path into a Nix
+# archive.
 open NIX, ">$nixExpr";
 print NIX "[";
 
@@ -112,172 +101,117 @@ print NIX "]";
 close NIX;
 
 
-# Instantiate store derivations from the Nix expression.
-my @storeExprs;
-print STDERR "instantiating store derivations...\n";
-my $pid = open(READ, "$Nix::Config::binDir/nix-instantiate $nixExpr|")
-    or die "cannot run nix-instantiate";
+# Build the Nix expression.
+print STDERR "building compressed archives...\n";
+my @narPaths;
+my $pid = open(READ, "$Nix::Config::binDir/nix-build $nixExpr|")
+    or die "cannot run nix-build";
 while (<READ>) {
     chomp;
     die unless /^\//;
-    push @storeExprs, $_;
-}
-close READ or die "nix-instantiate failed: $?";
-
-
-# Build the derivations.
-print STDERR "creating archives...\n";
-
-my @narPaths;
-
-my @tmp = @storeExprs;
-while (scalar @tmp > 0) {
-    my $n = scalar @tmp;
-    if ($n > 256) { $n = 256 };
-    my @tmp2 = @tmp[0..$n - 1];
-    @tmp = @tmp[$n..scalar @tmp - 1];
-
-    my $pid = open(READ, "$Nix::Config::binDir/nix-store --realise @tmp2|")
-        or die "cannot run nix-store";
-    while (<READ>) {
-        chomp;
-        die unless (/^\//);
-        push @narPaths, "$_";
-    }
-    close READ or die "nix-store failed: $?";
+    push @narPaths, $_;
 }
+close READ or die "nix-build failed: $?";
 
 
-# Create the manifest.
-print STDERR "creating manifest...\n";
+# Upload the archives and the corresponding info files.
+print STDERR "uploading/copying archives...\n";
 
-my %narFiles;
-my %patches;
+my $totalNarSize = 0;
+my $totalNarBz2Size = 0;
 
-my @narArchives;
 for (my $n = 0; $n < scalar @storePaths; $n++) {
     my $storePath = $storePaths[$n];
     my $narDir = $narPaths[$n];
-    
-    $storePath =~ /\/([^\/]*)$/;
-    my $basename = $1;
-    defined $basename or die;
-
-    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
-    my $narbz2Hash = <HASH>;
-    chomp $narbz2Hash;
-    $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
-    close HASH;
-
-    my $narName = "$narbz2Hash.nar.bz2";
-
-    my $narFile = "$narDir/$narName";
-    (-f $narFile) or die "narfile for $storePath not found";
-    push @narArchives, $narFile;
-
-    my $narbz2Size = stat($narFile)->size;
-
-    my $references = `$Nix::Config::binDir/nix-store --query --references '$storePath'`;
-    die "cannot query references for `$storePath'" if $? != 0;
-    $references = join(" ", split(" ", $references));
-
-    my $deriver = `$Nix::Config::binDir/nix-store --query --deriver '$storePath'`;
-    die "cannot query deriver for `$storePath'" if $? != 0;
-    chomp $deriver;
-    $deriver = "" if $deriver eq "unknown-deriver";
+    my $baseName = basename $storePath;
 
-    my $narHash = `$Nix::Config::binDir/nix-store --query --hash '$storePath'`;
-    die "cannot query hash for `$storePath'" if $? != 0;
-    chomp $narHash;
+    # Get info about the store path.
+    my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath);
 
     # In some exceptional cases (such as VM tests that use the Nix
     # store of the host), the database doesn't contain the hash.  So
     # compute it.
     if ($narHash =~ /^sha256:0*$/) {
-        $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 '$storePath'`;
-        die "cannot hash `$storePath'" if $? != 0;
+        my $nar = "$tmpDir/nar";
+        system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
+            or die "cannot dump $storePath\n";
+        $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --flat $nar`;
+        die "cannot hash `$nar'" if $? != 0;
         chomp $narHash;
         $narHash = "sha256:$narHash";
+        $narSize = stat("$nar")->size;
+        unlink $nar or die;
     }
 
-    my $narSize = `$Nix::Config::binDir/nix-store --query --size '$storePath'`;
-    die "cannot query size for `$storePath'" if $? != 0;
-    chomp $narSize;
-
-    my $url;
-    if ($localCopy) {
-        $url = "$targetArchivesUrl/$narName";
-    } else {
-        $url = "$archivesGetURL/$narName";
-    }
-    $narFiles{$storePath} = [
-        { url => $url
-        , hash => "$hashAlgo:$narbz2Hash"
-        , size => $narbz2Size
-        , narHash => "$narHash"
-        , narSize => $narSize
-        , references => $references
-        , deriver => $deriver
-        }
-    ];
-}
-
-writeManifest $manifest, \%narFiles, \%patches;
-
-
-sub copyFile {
-    my $src = shift;
-    my $dst = shift;
-    my $tmp = "$dst.tmp.$$";
-    system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
-    rename($tmp, $dst) or die "cannot rename file: $!";
-}
-
+    $totalNarSize += $narSize;
+    
+    # Get info about the compressed NAR.
+    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
+    my $narBz2Hash = <HASH>;
+    chomp $narBz2Hash;
+    $narBz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
 
-# Upload/copy the archives.
-print STDERR "uploading/copying archives...\n";
+    my $narName = "$narBz2Hash.nar.bz2";
 
-sub archiveExists {
-    my $name = shift;
-    print STDERR "  HEAD on $archivesGetURL/$name\n";
-    return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
-}
+    my $narFile = "$narDir/$narName";
+    (-f $narFile) or die "NAR file for $storePath not found";
 
-foreach my $narArchive (@narArchives) {
+    my $narBz2Size = stat($narFile)->size;    
+    $totalNarBz2Size += $narBz2Size;
 
-    $narArchive =~ /\/([^\/]*)$/;
-    my $basename = $1;
+    printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
+        $narBz2Size / (1024 * 1024), $narBz2Size / $narSize * 100;
 
+    # Upload the compressed NAR.
     if ($localCopy) {
-        # Since nix-push creates $dst atomically, if it exists we
-        # don't have to copy again.
-        my $dst = "$localArchivesDir/$basename";
-        if (! -f "$localArchivesDir/$basename") {
-            print STDERR "  $narArchive\n";
-            copyFile $narArchive, $dst;
+        my $dst = "$localArchivesDir/$narName";
+        if (! -f $dst) {
+            my $tmp = "$localArchivesDir/.tmp.$$.$narName";
+            copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
+            rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
         }
+    } else {
+        die "unimplemented";
+        #if (!archiveExists("$basename")) {
+        #    system("$curl --show-error --upload-file " .
+        #           "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
+        #           die "curl failed on $narArchive: $?";
+        #}
     }
-    else {
-        if (!archiveExists("$basename")) {
-            print STDERR "  $narArchive\n";
-            system("$curl --show-error --upload-file " .
-                   "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
-                   die "curl failed on $narArchive: $?";
+
+    # Upload the info file.
+    my $info;
+    $info .= "StorePath: $storePath\n";
+    $info .= "URL: $narName\n";
+    $info .= "CompressedHash: sha256:$narBz2Hash\n";
+    $info .= "CompressedSize: $narBz2Size\n";
+    $info .= "NarHash: $narHash\n";
+    $info .= "NarSize: $narSize\n";
+    $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
+    if (defined $deriver) {
+        $info .= "Deriver: " . basename $deriver, "\n";
+        if (isValidPath($deriver)) {
+            my $drv = derivationFromPath($deriver);
+            $info .= "System: $drv->{platform}\n";
         }
     }
-}
-
 
-# Upload the manifest.
-print STDERR "uploading manifest...\n";
-if ($localCopy) {
-    copyFile $manifest, $localManifestFile;
-    copyFile "$manifest.bz2", "$localManifestFile.bz2";
-} else {
-    system("$curl --show-error --upload-file " .
-           "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
-           die "curl failed on $manifest: $?";
-    system("$curl --show-error --upload-file " .
-           "'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
-           die "curl failed on $manifest: $?";
+    my $infoName = hashString("sha256", 1, $storePath);
+    
+    if ($localCopy) {
+        my $dst = "$localArchivesDir/$infoName.narinfo";
+        if (! -f $dst) {
+            my $tmp = "$localArchivesDir/.tmp.$$.$infoName";
+            open INFO, ">$tmp" or die;
+            print INFO "$info" or die;
+            close INFO or die;
+            rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
+        }
+    } else {
+        die "unimplemented";
+    }
 }
+
+printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
+    $totalNarBz2Size / (1024 * 1024), $totalNarBz2Size / $totalNarSize * 100;