diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | doc/manual/command-ref/nix-channel.xml | 4 | ||||
| -rw-r--r-- | doc/manual/command-ref/nix-push.xml | 449 | ||||
| -rw-r--r-- | doc/manual/command-ref/nix-store.xml | 3 | ||||
| -rw-r--r-- | doc/manual/command-ref/utilities.xml | 1 | ||||
| -rw-r--r-- | doc/manual/local.mk | 2 | ||||
| -rw-r--r-- | scripts/local.mk | 1 | ||||
| -rwxr-xr-x | scripts/nix-push.in | 296 | ||||
| -rw-r--r-- | tests/binary-cache.sh | 6 | ||||
| -rw-r--r-- | tests/local.mk | 2 | ||||
| -rw-r--r-- | tests/nix-channel.sh | 2 | ||||
| -rw-r--r-- | tests/nix-push.sh | 12 | ||||
| -rw-r--r-- | tests/repair.sh | 2 |
13 files changed, 9 insertions, 772 deletions
diff --git a/.gitignore b/.gitignore index 245b89f3cbc2..2d1695d474cb 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-push /scripts/nix-switch /scripts/nix-collect-garbage /scripts/nix-prefetch-url diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index 0a1f2a8b722d..9acf44e52984 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -180,9 +180,7 @@ following files:</para> sufficient rights to add binary caches. For instance, in a multi-user Nix setup, the binary caches provided by the channels of the root user are used automatically, but caches corresponding - to the channels of non-root users are ignored. Binary caches can - be created and maintained using - <command>nix-push</command>.</para></listitem> + to the channels of non-root users are ignored.</para></listitem> </varlistentry> diff --git a/doc/manual/command-ref/nix-push.xml b/doc/manual/command-ref/nix-push.xml deleted file mode 100644 index 0749824a0ad4..000000000000 --- a/doc/manual/command-ref/nix-push.xml +++ /dev/null @@ -1,449 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-push"> - -<refmeta> - <refentrytitle>nix-push</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-push</refname> - <refpurpose>generate a binary cache</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-push</command> - <arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg> - <arg><option>--bzip2</option></arg> - <arg><option>--none</option></arg> - <arg><option>--force</option></arg> - <arg><option>--link</option></arg> - <arg><option>--manifest</option></arg> - <arg><option>--manifest-path</option> <replaceable>filename</replaceable></arg> - <arg><option>--url-prefix</option> <replaceable>url</replaceable></arg> - <arg><option>--key-file</option> <replaceable>path</replaceable></arg> - <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<para>The command <command>nix-push</command> produces a -<emphasis>binary cache</emphasis>, a directory containing compressed -Nix archives (NARs) plus some metadata of the closure of the specified -store paths. This directory can then be made available through a web -server to other Nix installations, allowing them to skip building from -source and instead download binaries from the cache -automatically.</para> - -<para><command>nix-push</command> performs the following actions. - -<orderedlist> - - <listitem><para>Each path in <replaceable>paths</replaceable> is - built (using <link - linkend='rsec-nix-store-realise'><command>nix-store - --realise</command></link>).</para></listitem> - - <listitem><para>All paths in the closure of - <replaceable>paths</replaceable> are determined (using - <command>nix-store --query --requisites - --include-outputs</command>). Note that since the - <option>--include-outputs</option> flag is used, if - <replaceable>paths</replaceable> includes a store derivation, you - get a combined source/binary distribution (e.g., source tarballs - will be included).</para></listitem> - - <listitem><para>All store paths determined in the previous step are - packaged into a NAR (using <command>nix-store --dump</command>) and - compressed using <command>xz</command> or <command>bzip2</command>. - The resulting files have the extension <filename>.nar.xz</filename> - or <filename>.nar.bz2</filename>. Also for each store path, Nix - generates a file with extension <filename>.narinfo</filename> - containing metadata such as the references, cryptographic hash and - size of each path.</para></listitem> - - <listitem><para>Optionally, a single <emphasis>manifest</emphasis> - file is created that contains the same metadata as the - <filename>.narinfo</filename> files. This is for compatibility with - Nix versions prior to 1.2.</para></listitem> - - <listitem><para>A file named <option>nix-cache-info</option> is - placed in the destination directory. The existence of this file - marks the directory as a binary cache.</para></listitem> - -</orderedlist> - -</para> - -</refsection> - - -<refsection><title>Options</title> - -<variablelist> - - <varlistentry><term><option>--dest</option> <replaceable>dest-dir</replaceable></term> - - <listitem><para>Set the destination directory to - <replaceable>dir</replaceable>, which is created if it does not - exist. This flag is required.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--bzip2</option></term> - - <listitem><para>Compress NARs using <command>bzip2</command> - instead of <command>xz</command>. The latter compresses about 30% - better on typical archives, decompresses about twice as fast, but - compresses a lot slower and is not supported by Nix prior to - version 1.2.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--none</option></term> - - <listitem><para>Do not compress NARs.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--force</option></term> - - <listitem><para>Overwrite <filename>.narinfo</filename> files if - they already exist.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--link</option></term> - - <listitem><para>By default, NARs are generated in the Nix store - and then copied to <replaceable>dest-dir</replaceable>. If this - option is given, hard links are used instead. This only works if - <replaceable>dest-dir</replaceable> is on the same filesystem as - the Nix store.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--manifest</option></term> - - <listitem><para>Force the generation of a manifest suitable for - use by old versions of Nix. The manifest is stored as - <filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--manifest-path</option> <replaceable>filename</replaceable></term> - - <listitem><para>Like <option>--manifest</option>, but store the - manifest in <replaceable>filename</replaceable>.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--url-prefix</option> <replaceable>url</replaceable></term> - - <listitem><para>Manifests are expected to contain the absolute - URLs of NARs. For generating these URLs, the prefix - <replaceable>url</replaceable> is used. It defaults to - <uri>file://<replaceable>dest-dir</replaceable></uri>.</para></listitem> - - </varlistentry> - - <varlistentry><term><option>--key-file</option> <replaceable>path</replaceable></term> - - <listitem><para>Sign the binary cache using the secret key stored - in <replaceable>path</replaceable>. This secret key must have been - created using <command - linkend="rsec-nix-store-generate-binary-cache-key">nix-store - --generate-binary-cache-key</command>. Users of this binary cache - should add the corresponding public key to the option - <option>binary-cache-public-keys</option> in - <filename>nix.conf</filename>.</para></listitem> - - </varlistentry> - -</variablelist> - -</refsection> - - -<refsection><title>Examples</title> - -<para>To add the closure of Thunderbird to a binary cache: - -<screen> -$ nix-push --dest /tmp/cache $(nix-build -A thunderbird) -</screen> - -Assuming that <filename>/tmp/cache</filename> is exported by a web -server as <uri>http://example.org/cache</uri>, you can then use this -cache on another machine to speed up the installation of Thunderbird: - -<screen> -$ nix-build -A thunderbird --option binary-caches http://example.org/cache -</screen> - -Alternatively, you could add <literal>binary-caches = -http://example.org/cache</literal> to -<filename>nix.conf</filename>.</para> - -<para>To also include build-time dependencies (such as source -tarballs): - -<screen> -$ nix-push --dest /tmp/cache $(nix-instantiate -A thunderbird) -</screen> - -</para> - -<para>To generate a signed binary cache, you must first generate a key -pair, in this example called <literal>cache.example.org-1</literal>, -storing the secret key in <filename>./sk</filename> and the public key -in <filename>./pk</filename>: - -<screen> -$ nix-store --generate-binary-cache-key cache.example.org-1 sk pk - -$ cat sk -cache.example.org-1:jcMRQYFo8pQKzTtimpQLIPeHkMYZjfhB24hGfwF+u9PuX8H8FO7q564+X3G/JDlqqIqGar3OXRRwS9N3Wh3vbw== - -$ cat pk -cache.example.org-1:7l/B/BTu6ueuPl9xvyQ5aqiKhmq9zl0UcEvTd1od728= -</screen> - -You can then generate a binary cache signed with the secret key: - -<screen> -$ nix-push --dest /tmp/cache --key-file ./sk $(type -p firefox) -</screen> - -Users who wish to verify the integrity of binaries downloaded from -your cache would add the following to their -<filename>nix.conf</filename>: - -<programlisting> -binary-caches = http://cache.example.org -signed-binary-caches = * -binary-cache-public-keys = cache.example.org-1:7l/B/BTu6ueuPl9xvyQ5aqiKhmq9zl0UcEvTd1od728= -</programlisting> - -Nix will then ignore any binary that has a missing, incorrect or -unrecognised signature.</para> - -</refsection> - - -<refsection><title>Binary cache format and operation</title> - -<para>A binary cache with URL <replaceable>url</replaceable> only -denotes a valid binary cache if the file -<uri><replaceable>url</replaceable>/nix-cache-info</uri> exists. If -this file does not exist (or cannot be downloaded), the cache is -ignored. If it does exist, it must be a text file containing cache -properties. Here’s an example: - -<screen> -StoreDir: /nix/store -WantMassQuery: 1 -Priority: 10 -</screen> - -The properties that are currently supported are: - -<variablelist> - - <varlistentry><term><literal>StoreDir</literal></term> - - <listitem><para>The path of the Nix store to which this binary - cache applies. Binaries are not relocatable — a binary built for - <filename>/nix/store</filename> won’t generally work in - <filename>/home/alice/store</filename> — so to prevent binaries - from being used in a wrong store, a binary cache is only used if - its <literal>StoreDir</literal> matches the local Nix - configuration. The default is - <filename>/nix/store</filename>.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>WantMassQuery</literal></term> - - <listitem><para>Query operations such as <command>nix-env - -qas</command> can cause thousands of cache queries, and thus - thousands of HTTP requests, to determine which packages are - available in binary form. While these requests are small, not - every server may appreciate a potential onslaught of queries. If - <literal>WantMassQuery</literal> is set to <literal>0</literal> - (default), “mass queries” such as <command>nix-env -qas</command> - will skip this cache. Thus a package may appear not to have a - binary substitute. However, the binary will still be used when - you actually install the package. If - <literal>WantMassQuery</literal> is set to <literal>1</literal>, - mass queries will use this cache.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>Priority</literal></term> - - <listitem><para>Each binary cache has a priority (defaulting to - 50). Binary caches are checked for binaries in order of ascending - priority; thus a higher number denotes a lower priority. The - binary cache <uri>https://cache.nixos.org</uri> has priority - 40.</para></listitem> - - </varlistentry> - -</variablelist> - -</para> - -<para>Every time Nix needs to build some store path -<replaceable>p</replaceable>, it will check each configured binary -cache to see if it has a NAR file for <replaceable>p</replaceable>, -until it finds one. If no cache has a NAR, Nix will fall back to -building the path from source (if applicable). To see if a cache with -URL <replaceable>url</replaceable> has a binary for -<replaceable>p</replaceable>, Nix fetches -<replaceable>url/h</replaceable>, where <replaceable>h</replaceable> -is the hash part of <replaceable>p</replaceable>. Thus, if we have a -cache <uri>https://cache.nixos.org</uri> and we want to obtain the -store path -<screen> -/nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7 -</screen> -then Nix will attempt to fetch -<screen> -https://cache.nixos.org/a8922c0h87iilxzzvwn2hmv8x210aqb9.narinfo -</screen> -(Commands such as <command>nix-env -qas</command> will issue an HTTP -HEAD request, since it only needs to know if the -<filename>.narinfo</filename> file exists.) The -<filename>.narinfo</filename> file is a simple text file that looks -like this: - -<screen> -StorePath: /nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7 -URL: nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2 -Compression: bzip2 -FileHash: sha256:0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70 -FileSize: 24473768 -NarHash: sha256:0s491y1h9hxj5ghiizlxk7ax6jwbha00zwn7lpyd5xg5bhf60vzg -NarSize: 109521136 -References: 2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16 ... -Deriver: 7akyyc87ka32xwmqza9dvyg5pwx3j212-glibc-2.7.drv -Sig: cache.example.org-1:WepnSp2UT0odDpR3NRjPVhJBHmdBgSBSTbHpdh4SCz92nGXwFY82bkPEmISoC0hGqBXDXEmB6y3Ohgna3mMgDg== -</screen> - -The fields are as follows: - -<variablelist> - - <varlistentry><term><literal>StorePath</literal></term> - - <listitem><para>The full store path, including the name part - (e.g., <literal>glibc-2.7</literal>). It must match the - requested store path.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>URL</literal></term> - - <listitem><para>The URL of the NAR, relative to the binary cache - URL.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>Compression</literal></term> - - <listitem><para>The compression method; either - <literal>xz</literal> or - <literal>bzip2</literal>.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>FileHash</literal></term> - - <listitem><para>The SHA-256 hash of the compressed - NAR.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>FileSize</literal></term> - - <listitem><para>The size of the compressed NAR.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>NarHash</literal></term> - - <listitem><para>The SHA-256 hash of the uncompressed NAR. This is - equal to the hash of the store path as returned by - <command>nix-store -q --hash - <replaceable>p</replaceable></command>.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>NarSize</literal></term> - - <listitem><para>The size of the uncompressed NAR.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>References</literal></term> - - <listitem><para>The references of the store path, without the Nix - store prefix.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>Deriver</literal></term> - - <listitem><para>The deriver of the store path, without the Nix - store prefix. This field is optional.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>System</literal></term> - - <listitem><para>The Nix platform type of this binary, if known. - This field is optional.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>Sig</literal></term> - - <listitem><para>A signature of the the form - <literal><replaceable>key-name</replaceable>:<replaceable>sig</replaceable></literal>, - where <replaceable>key-name</replaceable> is the symbolic name of - the key pair used to sign and verify the cache - (e.g. <literal>cache.example.org-1</literal>), and - <replaceable>sig</replaceable> is the actual signature, computed - over the <varname>StorePath</varname>, <varname>NarHash</varname>, - <varname>NarSize</varname> and <varname>References</varname> - fields using the <link - xlink:href="http://ed25519.cr.yp.to/">Ed25519 public-key signature - system</link>.</para></listitem> - - </varlistentry> - -</variablelist> - -</para> - -<para>Thus, in our example, after recursively ensuring that the -references exist (e.g., -<filename>/nix/store/2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16</filename>), -Nix will fetch <screen> -https://cache.nixos.org/nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2 -</screen> and decompress and unpack it to -<filename>/nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7</filename>.</para> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 346718588b77..0f6172defb38 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1390,8 +1390,7 @@ parameters: </orderedlist> -For an example, see the manual page for <command -linkend="sec-nix-push">nix-push</command>.</para> +</para> </refsection> diff --git a/doc/manual/command-ref/utilities.xml b/doc/manual/command-ref/utilities.xml index 0800b5380d41..893f5b5b5260 100644 --- a/doc/manual/command-ref/utilities.xml +++ b/doc/manual/command-ref/utilities.xml @@ -16,6 +16,5 @@ work with Nix.</para> <xi:include href="nix-hash.xml" /> <xi:include href="nix-instantiate.xml" /> <xi:include href="nix-prefetch-url.xml" /> -<xi:include href="nix-push.xml" /> </chapter> diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 9a1123638ebb..d89555899a70 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -39,7 +39,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ - nix-collect-garbage.1 nix-push.1 \ + nix-collect-garbage.1 \ nix-prefetch-url.1 nix-channel.1 \ nix-hash.1 nix-copy-closure.1 \ nix.conf.5 nix-daemon.8, \ diff --git a/scripts/local.mk b/scripts/local.mk index 46b3fe3cf781..fef256451d8b 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -2,7 +2,6 @@ nix_bin_scripts := \ $(d)/nix-build \ $(d)/nix-channel \ $(d)/nix-copy-closure \ - $(d)/nix-push bin-scripts += $(nix_bin_scripts) diff --git a/scripts/nix-push.in b/scripts/nix-push.in deleted file mode 100755 index 54456ac9512e..000000000000 --- a/scripts/nix-push.in +++ /dev/null @@ -1,296 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use File::Basename; -use File::Path qw(mkpath); -use File::stat; -use File::Copy; -use MIME::Base64; -use Nix::Config; -use Nix::Store; -use Nix::Manifest; -use Nix::Utils; - -binmode STDERR, ":encoding(utf8)"; - -my $tmpDir = mkTempDir("nix-push"); - -my $nixExpr = "$tmpDir/create-nars.nix"; - - -# Parse the command line. -my $compressionType = "xz"; -my $force = 0; -my $destDir; -my $writeManifest = 0; -my $manifestPath; -my $archivesURL; -my $link = 0; -my $secretKeyFile; -my @roots; -my @buildArgs; - -for (my $n = 0; $n < scalar @ARGV; $n++) { - my $arg = $ARGV[$n]; - - if ($arg eq "--help") { - exec "man nix-push" or die; - } elsif ($arg eq "--bzip2") { - $compressionType = "bzip2"; - } elsif ($arg eq "--none") { - $compressionType = "none"; - } elsif ($arg eq "--force") { - $force = 1; - } elsif ($arg eq "--dest") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - $destDir = $ARGV[$n]; - mkpath($destDir, 0, 0755); - } elsif ($arg eq "--manifest") { - $writeManifest = 1; - } elsif ($arg eq "--manifest-path") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - $manifestPath = $ARGV[$n]; - $writeManifest = 1; - mkpath(dirname($manifestPath), 0, 0755); - } elsif ($arg eq "--url-prefix") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - $archivesURL = $ARGV[$n]; - } elsif ($arg eq "--link") { - $link = 1; - } elsif ($arg eq "--key-file") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - $secretKeyFile = $ARGV[$n]; - } elsif ($arg eq "--max-jobs" || $arg eq "-j") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - push @buildArgs, ($arg, $ARGV[$n]); - } elsif (substr($arg, 0, 1) eq "-") { - die "$0: unknown flag ‘$arg’\n"; - } else { - push @roots, $arg; - } -} - -die "$0: please specify a destination directory\n" if !defined $destDir; - -$archivesURL = "file://$destDir" unless defined $archivesURL; - - -# From the given store paths, determine the set of requisite store -# paths, i.e, the paths required to realise them. -my %storePaths; - -foreach my $path (@roots) { - # Get all paths referenced by the normalisation of the given - # Nix expression. - my $pid = open(READ, - "$Nix::Config::binDir/nix-store --query --requisites --force-realise " . - "--include-outputs '$path'|") or die; - - while (<READ>) { - chomp; - die "bad: $_" unless /^\//; - $storePaths{$_} = ""; - } - - close READ or die "nix-store failed: $?"; -} - -my @storePaths = keys %storePaths; - - -# Don't create archives for files that are already in the binary cache. -my @storePaths2; -my %narFiles; -foreach my $storePath (@storePaths) { - my $pathHash = substr(basename($storePath), 0, 32); - my $narInfoFile = "$destDir/$pathHash.narinfo"; - if (!$force && -e $narInfoFile) { - my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read ‘$narInfoFile’\n"; - my $narFile = "$destDir/$narInfo->{url}"; - if (-e $narFile) { - print STDERR "skipping existing $storePath\n"; - # Add the NAR info to $narFiles if we're writing a - # manifest. - $narFiles{$storePath} = [ - { url => ("$archivesURL/" . basename $narInfo->{url}) - , hash => $narInfo->{fileHash} - , size => $narInfo->{fileSize} - , compressionType => $narInfo->{compression} - , narHash => $narInfo->{narHash} - , narSize => $narInfo->{narSize} - , references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}}) - , deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef - } - ] if $writeManifest; - next; - } - } - push @storePaths2, $storePath; -} - - -# Create a list of Nix derivations that turn each path into a Nix -# archive. -open NIX, ">$nixExpr"; -print NIX "["; - -foreach my $storePath (@storePaths2) { - die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/); - - # Construct a Nix expression that creates a Nix archive. - my $nixexpr = - "(import <nix/nar.nix> " . - "{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) "; - - print NIX $nixexpr; -} - -print NIX "]"; -close NIX; - - -# Build the Nix expression. -print STDERR "building compressed archives...\n"; -my @narPaths; -my $pid = open(READ, "-|", "$Nix::Config::binDir/nix-build", $nixExpr, "-o", "$tmpDir/result", @buildArgs) - or die "cannot run nix-build"; -while (<READ>) { - chomp; - die unless /^\//; - push @narPaths, $_; -} -close READ or die "nix-build failed: $?"; - - -# Write the cache info file. -my $cacheInfoFile = "$destDir/nix-cache-info"; -if (! -e $cacheInfoFile) { - open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!"; - print FILE "StoreDir: $Nix::Config::storeDir\n"; - print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas" - close FILE; -} - - -# Copy the archives and the corresponding NAR info files. -print STDERR "copying archives...\n"; - -my $totalNarSize = 0; -my $totalCompressedSize = 0; - -for (my $n = 0; $n < scalar @storePaths2; $n++) { - my $storePath = $storePaths2[$n]; - my $narDir = $narPaths[$n]; - my $baseName = basename $storePath; - - # Get info about the store path. - my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1); - - # In some exceptional cases (such as VM tests that use the Nix - # store of the host), the database doesn't contain the hash. So - # compute it. - if ($narHash =~ /^sha256:0*$/) { - my $nar = "$tmpDir/nar"; - system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0 - or die "cannot dump $storePath\n"; - $narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`; - die "cannot hash ‘$nar’" if $? != 0; - chomp $narHash; - $narHash = "sha256:$narHash"; - $narSize = stat("$nar")->size; - unlink $nar or die; - } - - $totalNarSize += $narSize; - - # Get info about the compressed NAR. - open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash"; - my $compressedHash = <HASH>; - chomp $compressedHash; - $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash"; - close HASH; - - my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : ""); - - my $narFile = "$narDir/$narName"; - (-f $narFile) or die "NAR file for $storePath not found"; - - my $compressedSize = stat($narFile)->size; - $totalCompressedSize += $compressedSize; - - printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath, - $compressedSize / (1024 * 1024), $compressedSize / $narSize * 100; - - # Copy the compressed NAR. - my $dst = "$destDir/$narName"; - if (! -f $dst) { - my $tmp = "$destDir/.tmp.$$.$narName"; - if ($link) { - link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n"; - } else { - copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n"; - } - rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; - } - - # Write the info file. - my $info; - $info .= "StorePath: $storePath\n"; - $info .= "URL: $narName\n"; - $info .= "Compression: $compressionType\n"; - $info .= "FileHash: sha256:$compressedHash\n"; - $info .= "FileSize: $compressedSize\n"; - $info .= "NarHash: $narHash\n"; - $info .= "NarSize: $narSize\n"; - $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; - if (defined $deriver) { - $info .= "Deriver: " . basename $deriver . "\n"; - if (isValidPath($deriver)) { - my $drv = derivationFromPath($deriver); - $info .= "System: $drv->{platform}\n"; - } - } - - if (defined $secretKeyFile) { - my $secretKey = readFile $secretKeyFile; - my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs); - my $sig = signString($secretKey, $fingerprint); - $info .= "Sig: $sig\n"; - } - - my $pathHash = substr(basename($storePath), 0, 32); - - $dst = "$destDir/$pathHash.narinfo"; - if ($force || ! -f $dst) { - my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo"; - open INFO, ">$tmp" or die; - print INFO "$info" or die; - close INFO or die; - rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n"; - } - - $narFiles{$storePath} = [ - { url => "$archivesURL/$narName" - , hash => "sha256:$compressedHash" - , size => $compressedSize - , compressionType => $compressionType - , narHash => "$narHash" - , narSize => $narSize - , references => join(" ", @{$refs}) - , deriver => $deriver - } - ] if $writeManifest; -} - -printf STDERR "total compressed size %.2f MiB, %.1f%%\n", - $totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100; - - -# Optionally write a manifest. -writeManifest($manifestPath // "$destDir/MANIFEST", \%narFiles, \()) if $writeManifest; diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 96cab6ad4269..4ce428f643e5 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -6,7 +6,7 @@ clearCache # Create the binary cache. outPath=$(nix-build dependencies.nix --no-out-link) -nix-push --dest $cacheDir $outPath +nix copy --recursive --to file://$cacheDir $outPath basicTests() { @@ -58,7 +58,7 @@ unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/*.nar.xz | head -n1) +nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) mv $nar $nar.good mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar @@ -117,7 +117,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -nix-push --dest $cacheDir --key-file $TEST_ROOT/sk1 $outPath +nix copy --recursive --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/local.mk b/tests/local.mk index 3d25327bd590..1e5439f061e2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,7 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - build-hook.sh nix-push.sh gc.sh gc-concurrent.sh \ + build-hook.sh gc.sh gc-concurrent.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 2ec986dd415b..553ada51d9f7 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -15,7 +15,7 @@ nix-channel --remove xyzzy # Create a channel. rm -rf $TEST_ROOT/foo mkdir -p $TEST_ROOT/foo -nix-push --dest $TEST_ROOT/foo --manifest --bzip2 $(nix-store -r $(nix-instantiate dependencies.nix)) +nix copy --recursive --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix)) rm -rf $TEST_ROOT/nixexprs mkdir -p $TEST_ROOT/nixexprs cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ diff --git a/tests/nix-push.sh b/tests/nix-push.sh deleted file mode 100644 index 8ea59516c62c..000000000000 --- a/tests/nix-push.sh +++ /dev/null @@ -1,12 +0,0 @@ -source common.sh - -clearStore - -drvPath=$(nix-instantiate dependencies.nix) -outPath=$(nix-store -r $drvPath) - -echo "pushing $drvPath" - -mkdir -p $TEST_ROOT/cache - -nix-push --dest $TEST_ROOT/cache --manifest $drvPath --bzip2 diff --git a/tests/repair.sh b/tests/repair.sh index 92f2f8fe60a5..782838704da7 100644 --- a/tests/repair.sh +++ b/tests/repair.sh @@ -46,7 +46,7 @@ fi # --verify can fix it. clearCache -nix-push --dest $cacheDir $path +nix copy --recursive --to file://$cacheDir $path chmod u+w $path2 rm -rf $path2 |