diff options
98 files changed, 1793 insertions, 3273 deletions
diff --git a/.gitignore b/.gitignore index de8e9354fbbe..70ee11f9bfae 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-pull /scripts/nix-push /scripts/nix-switch /scripts/nix-collect-garbage @@ -43,11 +42,8 @@ Makefile.config /scripts/nix-channel /scripts/nix-build /scripts/nix-copy-closure -/scripts/nix-generate-patches /scripts/NixConfig.pm /scripts/NixManifest.pm -/scripts/GeneratePatches.pm -/scripts/download-using-manifests.pl /scripts/copy-from-other-stores.pl /scripts/download-from-binary-cache.pl /scripts/find-runtime-roots.pl @@ -55,10 +51,6 @@ Makefile.config /scripts/nix-reduce-build /scripts/nix-http-export.cgi -# /src/bsdiff-4.3/ -/src/bsdiff-4.3/bsdiff -/src/bsdiff-4.3/bspatch - # /src/libexpr/ /src/libexpr/lexer-tab.cc /src/libexpr/lexer-tab.hh diff --git a/Makefile b/Makefile index 39870af75104..943b639183aa 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,6 @@ makefiles = \ src/download-via-ssh/local.mk \ src/nix-log2xml/local.mk \ src/nix-prefetch-url/local.mk \ - src/bsdiff-4.3/local.mk \ perl/local.mk \ scripts/local.mk \ corepkgs/local.mk \ diff --git a/configure.ac b/configure.ac index 2a043ed13922..1215782a1bde 100644 --- a/configure.ac +++ b/configure.ac @@ -16,6 +16,8 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], machine_name="i686";; amd64) machine_name="x86_64";; + armv6|armv7) + machine_name="${host_cpu}l";; *) machine_name="$host_cpu";; esac diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index acddd63e12f7..598b15827883 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -306,21 +306,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>build-cache-failure</literal></term> - - <listitem><para>If set to <literal>true</literal>, Nix will - “cache” build failures, meaning that it will remember (in its - database) that a derivation previously failed. If you then try to - build the derivation again, Nix will immediately fail rather than - perform the build again. Failures in fixed-output derivations - (such as <function>fetchurl</function> calls) are never cached. - The “failed” status of a derivation can be cleared using - <command>nix-store --clear-failed-paths</command>. By default, - failure caching is disabled.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>build-keep-log</literal></term> <listitem><para>If set to <literal>true</literal> (the default), @@ -450,18 +435,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>force-manifest</literal></term> - - <listitem><para>If this option is set to <literal>false</literal> - (default) and a Nix channel provides both a manifest and a binary - cache, only the binary cache will be used. If set to - <literal>true</literal>, the manifest will be fetched as well. - This is useful if you want to use binary patches (which are - currently not supported by binary caches).</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>system</literal></term> <listitem><para>This option specifies the canonical Nix system diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index a6f4a27203ac..0a1f2a8b722d 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -73,11 +73,10 @@ condition="manual">See also <xref linkend="sec-channels" <listitem><para>Downloads the Nix expressions of all subscribed channels (or only those included in - <replaceable>names</replaceable> if specified), makes them the + <replaceable>names</replaceable> if specified) and makes them the default for <command>nix-env</command> operations (by symlinking - them from the directory <filename>~/.nix-defexpr</filename>), and - performs a <command>nix-pull</command> on the manifests of all - channels to make pre-built binaries available.</para></listitem> + them from the directory + <filename>~/.nix-defexpr</filename>).</para></listitem> </varlistentry> @@ -187,16 +186,6 @@ following files:</para> </varlistentry> - <varlistentry><term><filename>MANIFEST.bz2</filename></term> - - <listitem><para>(Deprecated in favour of binary caches.) A - manifest as created by <command>nix-push</command>. Only used if - <filename>binary-cache-url</filename> is not present or if the - <filename>nix.conf</filename> option - <option>force-manifest</option> is set.</para></listitem> - - </varlistentry> - </variablelist> </refsection> diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml index 5848b84a0173..97e261ae993d 100644 --- a/doc/manual/command-ref/nix-copy-closure.xml +++ b/doc/manual/command-ref/nix-copy-closure.xml @@ -22,7 +22,6 @@ <arg choice='plain'><option>--to</option></arg> <arg choice='plain'><option>--from</option></arg> </group> - <arg><option>--sign</option></arg> <arg><option>--gzip</option></arg> <!-- <arg><option>- -show-progress</option></arg> @@ -87,23 +86,6 @@ those paths. If this bothers you, use </varlistentry> - <varlistentry><term><option>--sign</option></term> - - <listitem><para>Let the sending machine cryptographically sign the - dump of each path with the key in - <filename><replaceable>sysconfdir</replaceable>/nix/signing-key.sec</filename>. - If the user on the target machine does not have direct access to - the Nix store (i.e., if the target machine has a multi-user Nix - installation), then the target machine will check the dump against - <filename><replaceable>sysconfdir</replaceable>/nix/signing-key.pub</filename> - before unpacking it in its Nix store. This allows secure sharing - of store paths between untrusted users on two machines, provided - that there is a trust relation between the Nix installations on - both machines (namely, they have matching public/secret - keys).</para></listitem> - - </varlistentry> - <varlistentry><term><option>--gzip</option></term> <listitem><para>Enable compression of the SSH diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index e9a5f0e097c4..2ed4a5d9f666 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -367,6 +367,10 @@ number of possible ways: linkend="rsec-nix-store-realise">realised</link> and installed.</para></listitem> + <listitem><para>By default all outputs are installed for each derivation. + That can be reduced by setting <literal>meta.outputsToInstall</literal>. + </para></listitem> <!-- TODO: link nixpkgs docs on the ability to override those. --> + </itemizedlist> </para> diff --git a/doc/manual/command-ref/nix-generate-patches.xml b/doc/manual/command-ref/nix-generate-patches.xml deleted file mode 100644 index 70bec432d28e..000000000000 --- a/doc/manual/command-ref/nix-generate-patches.xml +++ /dev/null @@ -1,44 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-generate-patches"> - -<refmeta> - <refentrytitle>nix-generate-patches</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-generate-patches</refname> - <refpurpose>generates binary patches between NAR files</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-generate-patches</command> - <arg choice='plain'><replaceable>NAR-DIR</replaceable></arg> - <arg choice='plain'><replaceable>PATCH-DIR</replaceable></arg> - <arg choice='plain'><replaceable>PATCH-URI</replaceable></arg> - <arg choice='plain'><replaceable>OLD-MANIFEST</replaceable></arg> - <arg choice='plain'><replaceable>NEW-MANIFEST</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<para>The command <command>nix-generate-patches</command> generates -binary patches between NAR files listed in OLD-MANIFEST and NEW-MANIFEST. -The patches are written to the directory PATCH-DIR, and the prefix -PATCH-URI is used to generate URIs for the patches. The patches are -added to NEW-MANIFEST. All NARs are required to exist in NAR-DIR. -Patches are generated between succeeding versions of packages with -the same name.</para> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-install-package.xml b/doc/manual/command-ref/nix-install-package.xml index f7802a95d55e..e17166caaaf3 100644 --- a/doc/manual/command-ref/nix-install-package.xml +++ b/doc/manual/command-ref/nix-install-package.xml @@ -146,9 +146,7 @@ The elements are as follows: <varlistentry><term><replaceable>manifestURL</replaceable></term> - <listitem><para>The manifest to be pulled by - <command>nix-pull</command>. The manifest must contain - <replaceable>outPath</replaceable>.</para></listitem> + <listitem><para>Obsolete.</para></listitem> </varlistentry> diff --git a/doc/manual/command-ref/nix-pull.xml b/doc/manual/command-ref/nix-pull.xml deleted file mode 100644 index eb471677b63f..000000000000 --- a/doc/manual/command-ref/nix-pull.xml +++ /dev/null @@ -1,54 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-pull"> - -<refmeta> - <refentrytitle>nix-pull</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-pull</refname> - <refpurpose>register availability of pre-built binaries (deprecated)</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-pull</command> - <arg choice='plain'><replaceable>url</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<note><para>This command and the use of manifests is deprecated. It is -better to use binary caches.</para></note> - -<para>The command <command>nix-pull</command> obtains a list of -pre-built store paths from the URL <replaceable>url</replaceable>, and -for each of these store paths, registers a substitute derivation that -downloads and unpacks it into the Nix store. This is used to speed up -installations: if you attempt to install something that has already -been built and stored into the network cache, Nix can transparently -re-use the pre-built store paths.</para> - -<para>The file at <replaceable>url</replaceable> must be compatible -with the files created by <replaceable>nix-push</replaceable>.</para> - -</refsection> - - -<refsection><title>Examples</title> - -<screen> -$ nix-pull https://nixos.org/releases/nixpkgs/nixpkgs-15.05pre54468.69858d7/MANIFEST</screen> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-push.xml b/doc/manual/command-ref/nix-push.xml index b8156b4554fd..0749824a0ad4 100644 --- a/doc/manual/command-ref/nix-push.xml +++ b/doc/manual/command-ref/nix-push.xml @@ -73,8 +73,7 @@ automatically.</para> <listitem><para>Optionally, a single <emphasis>manifest</emphasis> file is created that contains the same metadata as the <filename>.narinfo</filename> files. This is for compatibility with - Nix versions prior to 1.2 (see <command>nix-pull</command> for - details).</para></listitem> + Nix versions prior to 1.2.</para></listitem> <listitem><para>A file named <option>nix-cache-info</option> is placed in the destination directory. The existence of this file @@ -135,7 +134,7 @@ automatically.</para> <varlistentry><term><option>--manifest</option></term> <listitem><para>Force the generation of a manifest suitable for - use by <command>nix-pull</command>. The manifest is stored as + use by old versions of Nix. The manifest is stored as <filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem> </varlistentry> @@ -203,20 +202,6 @@ $ nix-push --dest /tmp/cache $(nix-instantiate -A thunderbird) </para> -<para>To generate a manifest suitable for <command>nix-pull</command>: - -<screen> -$ nix-push --dest /tmp/cache $(nix-build -A thunderbird) --manifest -</screen> - -On another machine you can then do: - -<screen> -$ nix-pull http://example.org/cache -</screen> - -to cause the binaries to be used by subsequent Nix operations.</para> - <para>To generate a signed binary cache, you must first generate a key pair, in this example called <literal>cache.example.org-1</literal>, storing the secret key in <filename>./sk</filename> and the public key diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 58a33117940f..340f61210b2d 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1350,82 +1350,6 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil <!--######################################################################--> -<refsection><title>Operation <option>--query-failed-paths</option></title> - -<refsection> - <title>Synopsis</title> - <cmdsynopsis> - <command>nix-store</command> - <arg choice='plain'><option>--query-failed-paths</option></arg> - </cmdsynopsis> -</refsection> - -<refsection><title>Description</title> - -<para>If build failure caching is enabled through the -<literal>build-cache-failure</literal> configuration option, the -operation <option>--query-failed-paths</option> will print out all -store paths that have failed to build.</para> - -</refsection> - -<refsection><title>Example</title> - -<screen> -$ nix-store --query-failed-paths -/nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59 -/nix/store/0011iy7sfwbc1qj5a1f6ifjnbcdail8a-haskell-gitit-ghc7.0.4-0.8.1 -/nix/store/001c0yn1hkh86gprvrb46cxnz3pki7q3-gamin-0.1.10 -<replaceable>…</replaceable> -</screen> - -</refsection> - -</refsection> - - -<!--######################################################################--> - -<refsection><title>Operation <option>--clear-failed-paths</option></title> - -<refsection> - <title>Synopsis</title> - <cmdsynopsis> - <command>nix-store</command> - <arg choice='plain'><option>--clear-failed-paths</option></arg> - <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> - </cmdsynopsis> -</refsection> - -<refsection><title>Description</title> - -<para>If build failure caching is enabled through the -<literal>build-cache-failure</literal> configuration option, the -operation <option>--clear-failed-paths</option> clears the “failed” -state of the given store paths, allowing them to be built again. This -is useful if the failure was actually transient (e.g. because the disk -was full).</para> - -<para>If a path denotes a derivation, its output paths are cleared. -You can provide the argument <literal>*</literal> to clear all store -paths.</para> - -</refsection> - -<refsection><title>Example</title> - -<screen> -$ nix-store --clear-failed-paths /nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59 -$ nix-store --clear-failed-paths * -</screen> - -</refsection> - -</refsection> - - -<!--######################################################################--> - <refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title> <refsection> diff --git a/doc/manual/command-ref/utilities.xml b/doc/manual/command-ref/utilities.xml index be2fe6e2d235..25e457e4e554 100644 --- a/doc/manual/command-ref/utilities.xml +++ b/doc/manual/command-ref/utilities.xml @@ -13,14 +13,10 @@ work with Nix.</para> <xi:include href="nix-collect-garbage.xml" /> <xi:include href="nix-copy-closure.xml" /> <xi:include href="nix-daemon.xml" /> -<!-- -<xi:include href="nix-generate-patches.xml" /> ---> <xi:include href="nix-hash.xml" /> <xi:include href="nix-install-package.xml" /> <xi:include href="nix-instantiate.xml" /> <xi:include href="nix-prefetch-url.xml" /> -<xi:include href="nix-pull.xml" /> <xi:include href="nix-push.xml" /> </chapter> diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 3d7e7fed9631..52f2884ab18f 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -39,7 +39,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ - nix-collect-garbage.1 nix-push.1 nix-pull.1 \ + nix-collect-garbage.1 nix-push.1 \ nix-prefetch-url.1 nix-channel.1 \ nix-install-package.1 nix-hash.1 nix-copy-closure.1 \ nix.conf.5 nix-daemon.8, \ @@ -76,17 +76,3 @@ all: $(d)/manual.html clean-files += $(d)/manual.html dist-files += $(d)/manual.html - - -# Generate the PDF manual. -$(d)/manual.pdf: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid - $(trace-gen) if test "$(dblatex)" != ""; then \ - cd doc/manual && $(XSLTPROC) --xinclude --stringparam profile.condition manual \ - $(docbookxsl)/profiling/profile.xsl manual.xml | \ - $(dblatex) -o $(notdir $@) $(dblatex_opts) -; \ - else \ - echo "Please install dblatex and rerun configure."; \ - exit 1; \ - fi - -clean-files += $(d)/manual.pdf diff --git a/doc/signing.txt b/doc/signing.txt deleted file mode 100644 index 7403cac470b2..000000000000 --- a/doc/signing.txt +++ /dev/null @@ -1,24 +0,0 @@ -Generate a private key: - -$ (umask 277 && openssl genrsa -out /etc/nix/signing-key.sec 2048) - -The private key should be kept secret (only readable to the Nix daemon -user). - - -Generate the corresponding public key: - -$ openssl rsa -in /etc/nix/signing-key.sec -pubout > /etc/nix/signing-key.pub - -The public key should be copied to all machines to which you want to -export store paths. - - -Signing: - -$ nix-hash --type sha256 --flat svn.nar | openssl rsautl -sign -inkey mykey.sec > svn.nar.sign - - -Verifying a signature: - -$ test "$(nix-hash --type sha256 --flat svn.nar)" = "$(openssl rsautl -verify -inkey mykey.pub -pubin -in svn.nar.sign)" diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index b0dc71fab377..f985c5b0188c 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -7,7 +7,6 @@ $version = "@PACKAGE_VERSION@"; $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; $libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@"; $stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix"; -$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests"; $logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@"; diff --git a/perl/lib/Nix/GeneratePatches.pm b/perl/lib/Nix/GeneratePatches.pm deleted file mode 100644 index 612c8a3a15ba..000000000000 --- a/perl/lib/Nix/GeneratePatches.pm +++ /dev/null @@ -1,340 +0,0 @@ -package Nix::GeneratePatches; - -use strict; -use File::Temp qw(tempdir); -use File::stat; -use Nix::Config; -use Nix::Manifest; - -our @ISA = qw(Exporter); -our @EXPORT = qw(generatePatches propagatePatches copyPatches); - - -# Some patch generations options. - -# Max size of NAR archives to generate patches for. -my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"}; -$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize; - -# If patch is bigger than this fraction of full archive, reject. -my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; -$maxPatchFraction = 0.60 if !defined $maxPatchFraction; - -my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; -$timeLimit = 180 if !defined $timeLimit; - -my $hashAlgo = "sha256"; - - -sub findOutputPaths { - my $narFiles = shift; - - my %outPaths; - - foreach my $p (keys %{$narFiles}) { - - # Ignore derivations. - next if ($p =~ /\.drv$/); - - # Ignore builders (too much ambiguity -- they're all called - # `builder.sh'). - next if ($p =~ /\.sh$/); - next if ($p =~ /\.patch$/); - - # Don't bother including tar files etc. - next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); - - $outPaths{$p} = 1; - } - - return %outPaths; -} - - -sub getNameVersion { - my $p = shift; - $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; - my $name = $1; - my $version = $2; - return undef unless defined $name && defined $version; - $name =~ s/^-//; - $version =~ s/^-//; - return ($name, $version); -} - - -# A quick hack to get a measure of the `distance' between two -# versions: it's just the position of the first character that differs -# (or 999 if they are the same). -sub versionDiff { - my $s = shift; - my $t = shift; - my $i; - return 999 if $s eq $t; - for ($i = 0; $i < length $s; $i++) { - return $i if $i >= length $t or - substr($s, $i, 1) ne substr($t, $i, 1); - } - return $i; -} - - -sub getNarBz2 { - my $narPath = shift; - my $narFiles = shift; - my $storePath = shift; - - my $narFileList = $$narFiles{$storePath}; - die "missing path $storePath" unless defined $narFileList; - - my $narFile = @{$narFileList}[0]; - die unless defined $narFile; - - $narFile->{url} =~ /\/([^\/]+)$/; - die unless defined $1; - return "$narPath/$1"; -} - - -sub containsPatch { - my $patches = shift; - my $storePath = shift; - my $basePath = shift; - my $patchList = $$patches{$storePath}; - return 0 if !defined $patchList; - my $found = 0; - foreach my $patch (@{$patchList}) { - # !!! baseHash might differ - return 1 if $patch->{basePath} eq $basePath; - } - return 0; -} - - -sub generatePatches { - my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_; - - my %srcOutPaths = findOutputPaths $srcNarFiles; - my %dstOutPaths = findOutputPaths $dstNarFiles; - - # For each output path in the destination, see if we need to / can - # create a patch. - - print STDERR "creating patches...\n"; - - foreach my $p (keys %dstOutPaths) { - - # If exactly the same path already exists in the source, skip it. - next if defined $srcOutPaths{$p}; - - print " $p\n"; - - # If not, then we should find the paths in the source that are - # `most' likely to be present on a system that wants to - # install this path. - - (my $name, my $version) = getNameVersion $p; - next unless defined $name && defined $version; - - my @closest = (); - my $closestVersion; - my $minDist = -1; # actually, larger means closer - - # Find all source paths with the same name. - - foreach my $q (keys %srcOutPaths) { - (my $name2, my $version2) = getNameVersion $q; - next unless defined $name2 && defined $version2; - - if ($name eq $name2) { - - my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system}; - my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system}; - if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { - print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; - next; - } - - # If the sizes differ too much, then skip. This - # disambiguates between, e.g., a real component and a - # wrapper component (cf. Firefox in Nixpkgs). - my $srcSize = @{$$srcNarFiles{$q}}[0]->{size}; - my $dstSize = @{$$dstNarFiles{$p}}[0]->{size}; - my $ratio = $srcSize / $dstSize; - $ratio = 1 / $ratio if $ratio < 1; - # print " SIZE $srcSize $dstSize $ratio $q\n"; - - if ($ratio >= 3) { - print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; - next; - } - - # If there are multiple matching names, include the - # ones with the closest version numbers. - my $dist = versionDiff $version, $version2; - if ($dist > $minDist) { - $minDist = $dist; - @closest = ($q); - $closestVersion = $version2; - } elsif ($dist == $minDist) { - push @closest, $q; - } - } - } - - if (scalar(@closest) == 0) { - print " NO BASE: $p\n"; - next; - } - - foreach my $closest (@closest) { - - # Generate a patch between $closest and $p. - print STDERR " $p <- $closest\n"; - - # If the patch already exists, skip it. - if (containsPatch($srcPatches, $p, $closest) || - containsPatch($dstPatches, $p, $closest)) - { - print " skipping, already exists\n"; - next; - } - - my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest; - my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p; - - if (! -f $srcNarBz2) { - warn "patch source archive $srcNarBz2 is missing\n"; - next; - } - - system("$Nix::Config::bzip2 -d < $srcNarBz2 > $tmpDir/A") == 0 - or die "cannot unpack $srcNarBz2"; - - if (stat("$tmpDir/A")->size >= $maxNarSize) { - print " skipping, source is too large\n"; - next; - } - - system("$Nix::Config::bzip2 -d < $dstNarBz2 > $tmpDir/B") == 0 - or die "cannot unpack $dstNarBz2"; - - if (stat("$tmpDir/B")->size >= $maxNarSize) { - print " skipping, destination is too large\n"; - next; - } - - my $time1 = time(); - my $res = system("ulimit -t $timeLimit; $Nix::Config::libexecDir/nix/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); - my $time2 = time(); - if ($res) { - warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; - next; - } - - my $baseHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; - chomp $baseHash; - - my $narHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die; - chomp $narHash; - - my $narDiffHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die; - chomp $narDiffHash; - - my $narDiffSize = stat("$tmpDir/DIFF")->size; - my $dstNarBz2Size = stat($dstNarBz2)->size; - - print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; - - if ($narDiffSize >= $dstNarBz2Size) { - print " rejecting; patch bigger than full archive\n"; - next; - } - - if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) { - print " rejecting; patch too large relative to full archive\n"; - next; - } - - my $finalName = "$narDiffHash.nar-bsdiff"; - - if (-e "$patchesPath/$finalName") { - print " not copying, already exists\n"; - } - - else { - system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0 - or die "cannot copy diff"; - rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName") - or die "cannot rename $patchesPath/$finalName.tmp"; - } - - # Add the patch to the manifest. - addPatch $dstPatches, $p, - { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash" - , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash" - , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff" - }; - } - } -} - - -# Propagate useful patches from $srcPatches to $dstPatches. A patch -# is useful if it produces either paths in the $dstNarFiles or paths -# that can be used as the base for other useful patches. -sub propagatePatches { - my ($srcPatches, $dstNarFiles, $dstPatches) = @_; - - print STDERR "propagating patches...\n"; - - my $changed; - do { - # !!! we repeat this to reach the transitive closure; inefficient - $changed = 0; - - print STDERR "loop\n"; - - my %dstBasePaths; - foreach my $q (keys %{$dstPatches}) { - foreach my $patch (@{$$dstPatches{$q}}) { - $dstBasePaths{$patch->{basePath}} = 1; - } - } - - foreach my $p (keys %{$srcPatches}) { - my $patchList = $$srcPatches{$p}; - - my $include = 0; - - # Is path $p included in the destination? If so, include - # patches that produce it. - $include = 1 if defined $$dstNarFiles{$p}; - - # Is path $p a path that serves as a base for paths in the - # destination? If so, include patches that produce it. - # !!! check baseHash - $include = 1 if defined $dstBasePaths{$p}; - - if ($include) { - foreach my $patch (@{$patchList}) { - $changed = 1 if addPatch $dstPatches, $p, $patch; - } - } - - } - - } while $changed; -} - - -# Add all new patches in $srcPatches to $dstPatches. -sub copyPatches { - my ($srcPatches, $dstPatches) = @_; - foreach my $p (keys %{$srcPatches}) { - addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}}; - } -} - - -return 1; diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index 428decf09b54..0da376761201 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -13,7 +13,7 @@ use Nix::Config; use Nix::Store; our @ISA = qw(Exporter); -our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo fingerprintPath); +our @EXPORT = qw(readManifest writeManifest addPatch parseNARInfo fingerprintPath); sub addNAR { @@ -228,172 +228,6 @@ sub writeManifest { } -sub updateManifestDB { - my $manifestDir = $Nix::Config::manifestDir; - - my @manifests = glob "$manifestDir/*.nixmanifest"; - return undef if scalar @manifests == 0; - - mkpath($manifestDir); - - unlink "$manifestDir/cache.sqlite"; # remove obsolete cache - my $dbPath = "$manifestDir/cache-v2.sqlite"; - - # Open/create the database. - our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "") - or die "cannot open database ‘$dbPath’"; - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - $dbh->do("pragma foreign_keys = on"); - $dbh->do("pragma synchronous = off"); # we can always reproduce the cache - $dbh->do("pragma journal_mode = truncate"); - - # Initialise the database schema, if necessary. - $dbh->do(<<EOF); - create table if not exists Manifests ( - id integer primary key autoincrement not null, - path text unique not null, - timestamp integer not null - ); -EOF - - $dbh->do(<<EOF); - create table if not exists NARs ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - url text not null, - compressionType text not null, - hash text, - size integer, - narHash text, - narSize integer, - refs text, - deriver text, - system text, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists NARs_storePath on NARs(storePath)"); - - $dbh->do(<<EOF); - create table if not exists Patches ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - basePath text not null, - baseHash text not null, - url text not null, - hash text, - size integer, - narHash text, - narSize integer, - patchType text not null, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists Patches_storePath on Patches(storePath)"); - - # Acquire an exclusive lock to ensure that only one process - # updates the DB at the same time. This isn't really necessary, - # but it prevents work duplication and lock contention in SQLite. - my $lockFile = "$manifestDir/cache.lock"; - open MAINLOCK, ">>$lockFile" or die "unable to acquire lock ‘$lockFile’: $!\n"; - flock(MAINLOCK, LOCK_EX) or die; - - our $insertNAR = $dbh->prepare( - "insert into NARs(manifest, storePath, url, compressionType, hash, size, narHash, " . - "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die; - - our $insertPatch = $dbh->prepare( - "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . - "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - - $dbh->begin_work; - - # Read each manifest in $manifestDir and add it to the database, - # unless we've already done so on a previous run. - my %seen; - - for my $manifestLink (@manifests) { - my $manifest = Cwd::abs_path($manifestLink); - next unless -f $manifest; - my $timestamp = lstat($manifest)->mtime; - $seen{$manifest} = 1; - - next if scalar @{$dbh->selectcol_arrayref( - "select 1 from Manifests where path = ? and timestamp = ?", - {}, $manifest, $timestamp)} == 1; - - print STDERR "caching $manifest...\n"; - - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - - $dbh->do("insert into Manifests(path, timestamp) values (?, ?)", - {}, $manifest, $timestamp); - - our $id = $dbh->last_insert_id("", "", "", ""); - - sub addNARToDB { - my ($storePath, $narFile) = @_; - $insertNAR->execute( - $id, $storePath, $narFile->{url}, $narFile->{compressionType}, $narFile->{hash}, - $narFile->{size}, $narFile->{narHash}, $narFile->{narSize}, $narFile->{references}, - $narFile->{deriver}, $narFile->{system}); - }; - - sub addPatchToDB { - my ($storePath, $patch) = @_; - $insertPatch->execute( - $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url}, - $patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize}, - $patch->{patchType}); - }; - - my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB); - - if ($version < 3) { - die "you have an old-style or corrupt manifest ‘$manifestLink’; please delete it\n"; - } - if ($version >= 10) { - die "manifest ‘$manifestLink’ is too new; please delete it or upgrade Nix\n"; - } - } - - # Removed cached information for removed manifests from the DB. - foreach my $manifest (@{$dbh->selectcol_arrayref("select path from Manifests")}) { - next if defined $seen{$manifest}; - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - } - - $dbh->commit; - - close MAINLOCK; - - return $dbh; -} - - -# Delete all old manifests downloaded from a given URL. -sub deleteOldManifests { - my ($url, $curUrlFile) = @_; - for my $urlFile (glob "$Nix::Config::manifestDir/*.url") { - next if defined $curUrlFile && $urlFile eq $curUrlFile; - open URL, "<$urlFile" or die; - my $url2 = <URL>; - chomp $url2; - close URL; - next unless $url eq $url2; - my $base = $urlFile; $base =~ s/.url$//; - unlink "${base}.url"; - unlink "${base}.nixmanifest"; - } -} - - # Return a fingerprint of a store path to be used in binary cache # signatures. It contains the store path, the base-32 SHA-256 hash of # the contents of the path, and the references. diff --git a/perl/local.mk b/perl/local.mk index ed49e3e6685e..5b43c4b717fd 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -1,7 +1,6 @@ nix_perl_sources := \ $(d)/lib/Nix/Store.pm \ $(d)/lib/Nix/Manifest.pm \ - $(d)/lib/Nix/GeneratePatches.pm \ $(d)/lib/Nix/SSH.pm \ $(d)/lib/Nix/CopyClosure.pm \ $(d)/lib/Nix/Config.pm.in \ diff --git a/release.nix b/release.nix index 79df103ccfc6..b81571e2d560 100644 --- a/release.nix +++ b/release.nix @@ -25,7 +25,7 @@ let buildInputs = [ curl bison flex perl libxml2 libxslt bzip2 xz - dblatex (dblatex.tex or tetex) nukeReferences pkgconfig sqlite libsodium + pkgconfig sqlite libsodium docbook5 docbook5_xsl ] ++ lib.optional (!lib.inNixShell) git; @@ -57,28 +57,14 @@ let preDist = '' make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk - - make doc/manual/manual.pdf - cp doc/manual/manual.pdf $out/manual.pdf - - # The PDF containes filenames of included graphics (see - # http://www.tug.org/pipermail/pdftex/2007-August/007290.html). - # This causes a retained dependency on dblatex, which Hydra - # doesn't like (the output of the tarball job is distributed - # to Windows and Macs, so there should be no Linux binaries - # in the closure). - nuke-refs $out/manual.pdf - echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products - echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products ''; }; build = pkgs.lib.genAttrs systems (system: - # FIXME: temporarily use a different branch for the Darwin build. - with import (if system == "x86_64-darwin" then <nixpkgs-darwin> else <nixpkgs>) { inherit system; }; + with import <nixpkgs> { inherit system; }; releaseTools.nixBuild { name = "nix"; @@ -113,7 +99,7 @@ let binaryTarball = pkgs.lib.genAttrs systems (system: # FIXME: temporarily use a different branch for the Darwin build. - with import (if system == "x86_64-darwin" then <nixpkgs-darwin> else <nixpkgs>) { inherit system; }; + with import <nixpkgs> { inherit system; }; let toplevel = builtins.getAttr system jobs.build; diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in deleted file mode 100755 index ffc49f8fffde..000000000000 --- a/scripts/download-using-manifests.pl.in +++ /dev/null @@ -1,376 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::Config; -use Nix::Manifest; -use Nix::Store; -use Nix::Utils; -use POSIX qw(strftime); - -STDOUT->autoflush(1); -binmode STDERR, ":encoding(utf8)"; - -my $logFile = "$Nix::Config::logDir/downloads"; - -# For queries, skip expensive calls to nix-hash etc. We're just -# estimating the expected download size. -my $fast = 1; - -my $curl = "$Nix::Config::curl --fail --location"; - - -# Open the manifest cache and update it if necessary. -my $dbh = updateManifestDB(); -exit 0 unless defined $dbh; # exit if there are no manifests -print "\n"; - - -# $hashCache->{$algo}->{$path} yields the $algo-hash of $path. -my $hashCache; - - -sub parseHash { - my $hash = shift; - if ($hash =~ /^(.+):(.+)$/) { - return ($1, $2); - } else { - return ("md5", $hash); - } -} - - -# Compute the most efficient sequence of downloads to produce the -# given path. -sub computeSmallestDownload { - my $targetPath = shift; - - # Build a graph of all store paths that might contribute to the - # construction of $targetPath, and the special node "start". The - # edges are either patch operations, or downloads of full NAR - # files. The latter edges only occur between "start" and a store - # path. - my %graph; - - $graph{"start"} = {d => 0, pred => undef, edges => []}; - - my @queue = (); - my $queueFront = 0; - my %done; - - sub addNode { - my $graph = shift; - my $u = shift; - $$graph{$u} = {d => 999999999999, pred => undef, edges => []} - unless defined $$graph{$u}; - } - - sub addEdge { - my $graph = shift; - my $u = shift; - my $v = shift; - my $w = shift; - my $type = shift; - my $info = shift; - addNode $graph, $u; - push @{$$graph{$u}->{edges}}, - {weight => $w, start => $u, end => $v, type => $type, info => $info}; - my $n = scalar @{$$graph{$u}->{edges}}; - } - - push @queue, $targetPath; - - while ($queueFront < scalar @queue) { - my $u = $queue[$queueFront++]; - next if defined $done{$u}; - $done{$u} = 1; - - addNode \%graph, $u; - - # If the path already exists, it has distance 0 from the - # "start" node. - if (isValidPath($u)) { - addEdge \%graph, "start", $u, 0, "present", undef; - } - - else { - - # Add patch edges. - my $patchList = $dbh->selectall_arrayref( - "select * from Patches where storePath = ?", - { Slice => {} }, $u); - - foreach my $patch (@{$patchList}) { - if (isValidPath($patch->{basePath})) { - my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; - - my $hash = $hashCache->{$baseHashAlgo}->{$patch->{basePath}}; - if (!defined $hash) { - $hash = $fast && $baseHashAlgo eq "sha256" - ? queryPathHash($patch->{basePath}) - : hashPath($baseHashAlgo, $baseHashAlgo ne "md5", $patch->{basePath}); - $hash =~ s/.*://; - $hashCache->{$baseHashAlgo}->{$patch->{basePath}} = $hash; - } - - next if $hash ne $baseHash; - } - push @queue, $patch->{basePath}; - addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch; - } - - # Add NAR file edges to the start node. - my $narFileList = $dbh->selectall_arrayref( - "select * from NARs where storePath = ?", - { Slice => {} }, $u); - - foreach my $narFile (@{$narFileList}) { - # !!! how to handle files whose size is not known in advance? - # For now, assume some arbitrary size (1 GB). - # This has the side-effect of preferring non-Hydra downloads. - addEdge \%graph, "start", $u, ($narFile->{size} || 1000000000), "narfile", $narFile; - } - } - } - - - # Run Dijkstra's shortest path algorithm to determine the shortest - # sequence of download and/or patch actions that will produce - # $targetPath. - - my @todo = keys %graph; - - while (scalar @todo > 0) { - - # Remove the closest element from the todo list. - # !!! inefficient, use a priority queue - @todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo; - my $u = pop @todo; - - my $u_ = $graph{$u}; - - foreach my $edge (@{$u_->{edges}}) { - my $v_ = $graph{$edge->{end}}; - if ($v_->{d} > $u_->{d} + $edge->{weight}) { - $v_->{d} = $u_->{d} + $edge->{weight}; - # Store the edge; to edge->start is actually the - # predecessor. - $v_->{pred} = $edge; - } - } - } - - - # Retrieve the shortest path from "start" to $targetPath. - my @path = (); - my $cur = $targetPath; - return () unless defined $graph{$targetPath}->{pred}; - while ($cur ne "start") { - push @path, $graph{$cur}->{pred}; - $cur = $graph{$cur}->{pred}->{start}; - } - - return @path; -} - - -# Parse the arguments. - -if ($ARGV[0] eq "--query") { - - while (<STDIN>) { - chomp; - my ($cmd, @args) = split " ", $_; - - if ($cmd eq "have") { - foreach my $storePath (@args) { - print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0; - } - print "\n"; - } - - elsif ($cmd eq "info") { - foreach my $storePath (@args) { - - my $infos = $dbh->selectall_arrayref( - "select * from NARs where storePath = ?", - { Slice => {} }, $storePath); - - next unless scalar @{$infos} > 0; - my $info = @{$infos}[0]; - - print "$storePath\n"; - print "$info->{deriver}\n"; - my @references = split " ", $info->{refs}; - print scalar @references, "\n"; - print "$_\n" foreach @references; - - my @path = computeSmallestDownload $storePath; - - my $downloadSize = 0; - while (scalar @path > 0) { - my $edge = pop @path; - my $u = $edge->{start}; - my $v = $edge->{end}; - if ($edge->{type} eq "patch") { - $downloadSize += $edge->{info}->{size} || 0; - } - elsif ($edge->{type} eq "narfile") { - $downloadSize += $edge->{info}->{size} || 0; - } - } - - print "$downloadSize\n"; - - my $narSize = $info->{narSize} || 0; - print "$narSize\n"; - } - - print "\n"; - } - - else { die "unknown command ‘$cmd’"; } - } - - exit 0; -} - -elsif ($ARGV[0] ne "--substitute") { - die; -} - - -die unless scalar @ARGV == 3; -my $targetPath = $ARGV[1]; -my $destPath = $ARGV[2]; -$fast = 0; - - -# Create a temporary directory. -my $tmpDir = mkTempDir("nix-download"); - -my $tmpNar = "$tmpDir/nar"; -my $tmpNar2 = "$tmpDir/nar2"; - - -open LOGFILE, ">>$logFile" or die "cannot open log file $logFile"; - -my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); -print LOGFILE "$$ get $targetPath $date\n"; - -print STDERR "\n*** Trying to download/patch ‘$targetPath’\n"; - - -# Compute the shortest path. -my @path = computeSmallestDownload $targetPath; -die "don't know how to produce $targetPath\n" if scalar @path == 0; - - -# We don't need the manifest anymore, so close it as an optimisation: -# if we still have SQLite locks blocking other processes (we -# shouldn't), this gets rid of them. -$dbh->disconnect; - - -# Traverse the shortest path, perform the actions described by the -# edges. -my $curStep = 1; -my $maxStep = scalar @path; - -my $finalNarHash; - -while (scalar @path > 0) { - my $edge = pop @path; - my $u = $edge->{start}; - my $v = $edge->{end}; - - print STDERR "\n*** Step $curStep/$maxStep: "; - - if ($edge->{type} eq "present") { - print STDERR "using already present path ‘$v’\n"; - print LOGFILE "$$ present $v\n"; - - if ($curStep < $maxStep) { - # Since this is not the last step, the path will be used - # as a base to one or more patches. So turn the base path - # into a NAR archive, to which we can apply the patch. - print STDERR " packing base path...\n"; - system("$Nix::Config::binDir/nix-store --dump $v > $tmpNar") == 0 - or die "cannot dump ‘$v’"; - } - } - - elsif ($edge->{type} eq "patch") { - my $patch = $edge->{info}; - print STDERR "applying patch ‘$patch->{url}’ to ‘$u’ to create ‘$v’\n"; - - print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n"; - - # Download the patch. - print STDERR " downloading patch...\n"; - my $patchPath = "$tmpDir/patch"; - checkURL $patch->{url}; - system("$curl '$patch->{url}' -o $patchPath") == 0 - or die "cannot download patch ‘$patch->{url}’\n"; - - # Apply the patch to the NAR archive produced in step 1 (for - # the already present path) or a later step (for patch sequences). - print STDERR " applying patch...\n"; - system("$Nix::Config::libexecDir/nix/bspatch $tmpNar $tmpNar2 $patchPath") == 0 - or die "cannot apply patch ‘$patchPath’ to $tmpNar\n"; - - if ($curStep < $maxStep) { - # The archive will be used as the base of the next patch. - rename "$tmpNar2", "$tmpNar" or die "cannot rename NAR archive: $!"; - } else { - # This was the last patch. Unpack the final NAR archive - # into the target path. - print STDERR " unpacking patched archive...\n"; - system("$Nix::Config::binDir/nix-store --restore $destPath < $tmpNar2") == 0 - or die "cannot unpack $tmpNar2 to ‘$v’\n"; - } - - $finalNarHash = $patch->{narHash}; - } - - elsif ($edge->{type} eq "narfile") { - my $narFile = $edge->{info}; - print STDERR "downloading ‘$narFile->{url}’ to ‘$v’\n"; - - my $size = $narFile->{size} || -1; - print LOGFILE "$$ narfile $narFile->{url} $size $v\n"; - - checkURL $narFile->{url}; - - my $decompressor = - $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" : - $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" : - $narFile->{compressionType} eq "none" ? "" : - die "unknown compression type ‘$narFile->{compressionType}’"; - - if ($curStep < $maxStep) { - # The archive will be used a base to a patch. - system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0 - or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n"; - } else { - # Unpack the archive to the target path. - system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0 - or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n"; - } - - $finalNarHash = $narFile->{narHash}; - } - - $curStep++; -} - - -# Tell Nix about the expected hash so it can verify it. -die "cannot check integrity of the downloaded path since its hash is not known\n" - unless defined $finalNarHash; -print "$finalNarHash\n"; - - -print STDERR "\n"; -print LOGFILE "$$ success\n"; -close LOGFILE; diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index bef5cd4f15fa..509acc41fcbd 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/bin/sh set -e @@ -49,7 +49,10 @@ for i in $(cd $self/store >/dev/null && echo *); do fi if ! [ -e "$dest/store/$i" ]; then cp -Rp "$self/store/$i" "$i_tmp" + chmod -R a-w "$i_tmp" + chmod +w "$i_tmp" mv "$i_tmp" "$dest/store/$i" + chmod -w "$dest/store/$i" fi done echo "" >&2 diff --git a/scripts/local.mk b/scripts/local.mk index cdac56bf13cb..f6542a4cba2f 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -2,17 +2,14 @@ nix_bin_scripts := \ $(d)/nix-build \ $(d)/nix-channel \ $(d)/nix-copy-closure \ - $(d)/nix-generate-patches \ $(d)/nix-install-package \ - $(d)/nix-pull \ $(d)/nix-push bin-scripts += $(nix_bin_scripts) nix_substituters := \ $(d)/copy-from-other-stores.pl \ - $(d)/download-from-binary-cache.pl \ - $(d)/download-using-manifests.pl + $(d)/download-from-binary-cache.pl nix_noinst_scripts := \ $(d)/build-remote.pl \ diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index 5191b5855ae0..65084ff1f34a 100755 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in @@ -12,8 +12,6 @@ binmode STDERR, ":encoding(utf8)"; Nix::Config::readConfig; -my $manifestDir = $Nix::Config::manifestDir; - # Turn on caching in nix-prefetch-url. my $channelCache = "$Nix::Config::stateDir/channel-cache"; @@ -75,7 +73,6 @@ sub removeChannel { my ($name) = @_; readChannels; my $url = $channels{$name}; - deleteOldManifests($url . "/MANIFEST", undef) if defined $url; delete $channels{$name}; writeChannels; @@ -84,8 +81,7 @@ sub removeChannel { } -# Fetch Nix expressions and pull manifests from the subscribed -# channels. +# Fetch Nix expressions and binary cache URLs from the subscribed channels. sub update { my @channelNames = @_; @@ -97,7 +93,6 @@ sub update { next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames}; my $url = $channels{$name}; - my $origUrl = "$url/MANIFEST"; # We want to download the url to a file to see if it's a tarball while also checking if we # got redirected in the process, so that we can grab the various parts of a nix channel @@ -132,22 +127,8 @@ sub update { if ($ret != 0) { # Check if the channel advertises a binary cache. my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`; - my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true"; - if ($? == 0 && $binaryCacheURL ne "") { - $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "; - deleteOldManifests($origUrl, undef); - } else { - $getManifest = 1; - } - - if ($getManifest) { - # No binary cache, so pull the channel manifest. - mkdir $manifestDir, 0755 unless -e $manifestDir; - die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir; - $ENV{'NIX_ORIG_URL'} = $origUrl; - system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0 - or die "cannot pull manifest from ‘$url’\n"; - } + $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; " + if $? == 0 && $binaryCacheURL ne ""; # Download the channel tarball. my $fullURL = "$url/nixexprs.tar.xz"; diff --git a/scripts/nix-generate-patches.in b/scripts/nix-generate-patches.in deleted file mode 100755 index 0a29c0548c1f..000000000000 --- a/scripts/nix-generate-patches.in +++ /dev/null @@ -1,51 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use strict; -use Nix::Manifest; -use Nix::GeneratePatches; -use Nix::Utils; - -if (scalar @ARGV != 5) { - print STDERR <<EOF; -Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST - -This command generates binary patches between NAR files listed in -OLD-MANIFEST and NEW-MANIFEST. The patches are written to the -directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs -for the patches. The patches are added to NEW-MANIFEST. All NARs are -required to exist in NAR-DIR. Patches are generated between -succeeding versions of packages with the same name. -EOF - exit 1; -} - -my $narPath = $ARGV[0]; -my $patchesPath = $ARGV[1]; -my $patchesURL = $ARGV[2]; -my $srcManifest = $ARGV[3]; -my $dstManifest = $ARGV[4]; - -my (%srcNarFiles, %srcLocalPaths, %srcPatches); -readManifest $srcManifest, \%srcNarFiles, \%srcPatches; - -my (%dstNarFiles, %dstLocalPaths, %dstPatches); -readManifest $dstManifest, \%dstNarFiles, \%dstPatches; - -my $tmpDir = mkTempDir("nix-generate-patches"); - -generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches, - $narPath, $patchesPath, $patchesURL, $tmpDir; - -propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches; - -# Optionally add all new patches to the manifest in $NIX_ALL_PATCHES. -my $allPatchesFile = $ENV{"NIX_ALL_PATCHES"}; -if (defined $allPatchesFile) { - my (%dummy, %allPatches); - readManifest("$patchesPath/all-patches", \%dummy, \%allPatches) - if -f $allPatchesFile; - copyPatches \%dstPatches, \%allPatches; - writeManifest($allPatchesFile, {}, \%allPatches, 0); -} - -writeManifest $dstManifest, \%dstNarFiles, \%dstPatches; diff --git a/scripts/nix-install-package.in b/scripts/nix-install-package.in index b442c708b1a2..ba349774af54 100755 --- a/scripts/nix-install-package.in +++ b/scripts/nix-install-package.in @@ -89,7 +89,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )"; # store path. We'll let nix-env do that. $contents =~ - / ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x + / ^ \s* (\S+) \s+ (\S+) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x or barf "invalid package contents"; my $version = $1; my $manifestURL = $2; @@ -111,25 +111,9 @@ if ($interactive) { } -if (defined $binaryCacheURL) { +die "$0: package does not supply a binary cache\n" unless defined $binaryCacheURL; - push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL; - -} else { - - # Store the manifest in the temporary directory so that we don't - # pollute /nix/var/nix/manifests. This also requires that we - # don't use the Nix daemon (because otherwise - # download-using-manifests won't see our NIX_MANIFESTS_DIRS - # environment variable). - $ENV{NIX_MANIFESTS_DIR} = $tmpDir; - $ENV{NIX_REMOTE} = ""; - - print "\nPulling manifests...\n"; - system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0 - or barf "nix-pull failed: $?"; - -} +push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL; print "\nInstalling package...\n"; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 6616b12b0cf4..5e01de95156c 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,24 +1,71 @@ -if [ -n "$HOME" ]; then - NIX_LINK="$HOME/.nix-profile" - - # Set the default profile. - if ! [ -L "$NIX_LINK" ]; then - echo "creating $NIX_LINK" >&2 - _NIX_DEF_LINK=@localstatedir@/nix/profiles/default - @coreutils@/ln -s "$_NIX_DEF_LINK" "$NIX_LINK" +if [ -n "$HOME" ] && [ -n "$USER" ]; then + __savedpath="$PATH" + export PATH=@coreutils@ + + # Set up the per-user profile. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix + + : ${NIX_LINK:=$HOME/.nix-profile} + + : ${NIX_USER_PROFILE_DIR:=@localstatedir@/nix/profiles/per-user/$USER} + + mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" + + if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then + echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 fi - export PATH=$NIX_LINK/bin:$NIX_LINK/sbin:$PATH + if [ -w "$HOME" ]; then + if ! [ -L "$NIX_LINK" ]; then + echo "Nix: creating $NIX_LINK" >&2 + if [ "$USER" != root ]; then + if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then + echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 + fi + else + # Root installs in the system-wide profile by default. + ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" + fi + fi - # Subscribe the user to the Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" + # Subscribe the user to the unstable Nixpkgs channel by default. + if [ ! -e "$HOME/.nix-channels" ]; then + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" + fi + + # Create the per-user garbage collector roots directory. + __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" + mkdir -m 0755 -p "$__user_gcroots" + if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then + echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 + fi + unset __user_gcroots + + # Set up a default Nix expression from which to install stuff. + __nix_defexpr="$HOME"/.nix-defexpr + [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" + mkdir -m 0755 -p "$__nix_defexpr" + if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then + ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root + fi + unset __nix_defexpr fi # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that # <nixpkgs> paths work when the user has fetched the Nixpkgs # channel. - export NIX_PATH=${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs + export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + export NIX_USER_PROFILE_DIR + export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + + for i in $NIX_PROFILES; do + if [ -d "$i/lib/aspell" ]; then + export ASPELL_CONF="dict-dir $i/lib/aspell" + fi + done # Set $SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -34,4 +81,7 @@ if [ -n "$HOME" ]; then elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # old cacert in Nix profile export SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi + + export PATH="$NIX_LINK/bin:$NIX_LINK/sbin:$__savedpath" + unset __savedpath fi diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in deleted file mode 100755 index 995b50935964..000000000000 --- a/scripts/nix-pull.in +++ /dev/null @@ -1,102 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::Config; -use Nix::Manifest; - -binmode STDERR, ":encoding(utf8)"; - -my $manifestDir = $Nix::Config::manifestDir; - - -# Prevent access problems in shared-stored installations. -umask 0022; - - -# Create the manifests directory if it doesn't exist. -if (! -e $manifestDir) { - mkdir $manifestDir, 0755 or die "cannot create directory ‘$manifestDir’"; -} - - -# Make sure that the manifests directory is scanned for GC roots. -my $gcRootsDir = "$Nix::Config::stateDir/gcroots"; -my $manifestDirLink = "$gcRootsDir/manifests"; -if (! -l $manifestDirLink) { - symlink($manifestDir, $manifestDirLink) or die "cannot create symlink ‘$manifestDirLink’"; -} - - -# Process the URLs specified on the command line. - -sub downloadFile { - my $url = shift; - $ENV{"PRINT_PATH"} = 1; - $ENV{"QUIET"} = 1; - my ($dummy, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`; - die "cannot fetch ‘$url’" if $? != 0; - die "nix-prefetch-url did not return a path" unless defined $path; - chomp $path; - return $path; -} - -sub processURL { - my $url = shift; - - $url =~ s/\/$//; - - my $manifest; - - my $origUrl = $ENV{'NIX_ORIG_URL'} || $url; - - # First see if a bzipped manifest is available. - if (system("$Nix::Config::curl --fail --silent --location --head '$url'.bz2 > /dev/null") == 0) { - print "fetching list of Nix archives at ‘$url.bz2’...\n"; - $manifest = downloadFile "$url.bz2"; - } - - # Otherwise, just get the uncompressed manifest. - else { - print "fetching list of Nix archives at ‘$url’...\n"; - $manifest = downloadFile $url; - } - - my $baseName = "unnamed"; - if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component - $baseName = $1; - } - - my $hash = `$Nix::Config::binDir/nix-hash --flat '$manifest'` - or die "cannot hash ‘$manifest’"; - chomp $hash; - - my $urlFile = "$manifestDir/$baseName-$hash.url"; - open URL, ">$urlFile" or die "cannot create ‘$urlFile’"; - print URL $origUrl; - close URL; - - my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest"; - - unlink $finalPath if -e $finalPath; - - symlink("$manifest", "$finalPath") - or die "cannot link ‘$finalPath’ to ‘$manifest’"; - - deleteOldManifests($origUrl, $urlFile); -} - -while (@ARGV) { - my $url = shift @ARGV; - if ($url eq "--help") { - exec "man nix-pull" or die; - } elsif ($url eq "--skip-wrong-store") { - # No-op, no longer supported. - } else { - processURL $url; - } -} - - -# Update the cache. -updateManifestDB(); diff --git a/src/bsdiff-4.3/bsdiff.1 b/src/bsdiff-4.3/bsdiff.1 deleted file mode 100644 index ead6c4deb57f..000000000000 --- a/src/bsdiff-4.3/bsdiff.1 +++ /dev/null @@ -1,63 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSDIFF 1 -.Os FreeBSD -.Sh NAME -.Nm bsdiff -.Nd generate a patch between two binary files -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -compares -.Ao Ar oldfile Ac -to -.Ao Ar newfile Ac -and writes to -.Ao Ar patchfile Ac -a binary patch suitable for use by bspatch(1). -When -.Ao Ar oldfile Ac -and -.Ao Ar newfile Ac -are two versions of an executable program, the -patches produced are on average a factor of five smaller -than those produced by any other binary patch tool known -to the author. -.Pp -.Nm -uses memory equal to 17 times the size of -.Ao Ar oldfile Ac , -and requires -an absolute minimum working set size of 8 times the size of oldfile. -.Sh SEE ALSO -.Xr bspatch 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c deleted file mode 100644 index 374ed038fa1f..000000000000 --- a/src/bsdiff-4.3/bsdiff.c +++ /dev/null @@ -1,405 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $"); -#endif - -#include <sys/types.h> - -#include <bzlib.h> -#include <err.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) - -static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) -{ - off_t i,j,k,x,tmp,jj,kk; - - if(len<16) { - for(k=start;k<start+len;k+=j) { - j=1;x=V[I[k]+h]; - for(i=1;k+i<start+len;i++) { - if(V[I[k+i]+h]<x) { - x=V[I[k+i]+h]; - j=0; - }; - if(V[I[k+i]+h]==x) { - tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; - j++; - }; - }; - for(i=0;i<j;i++) V[I[k+i]]=k+j-1; - if(j==1) I[k]=-1; - }; - return; - }; - - x=V[I[start+len/2]+h]; - jj=0;kk=0; - for(i=start;i<start+len;i++) { - if(V[I[i]+h]<x) jj++; - if(V[I[i]+h]==x) kk++; - }; - jj+=start;kk+=jj; - - i=start;j=0;k=0; - while(i<jj) { - if(V[I[i]+h]<x) { - i++; - } else if(V[I[i]+h]==x) { - tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; - j++; - } else { - tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - while(jj+j<kk) { - if(V[I[jj+j]+h]==x) { - j++; - } else { - tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - if(jj>start) split(I,V,start,jj-start,h); - - for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; - if(jj==kk-1) I[jj]=-1; - - if(start+len>kk) split(I,V,kk,start+len-kk,h); -} - -static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) -{ - off_t buckets[256]; - off_t i,h,len; - - for(i=0;i<256;i++) buckets[i]=0; - for(i=0;i<oldsize;i++) buckets[old[i]]++; - for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; - for(i=255;i>0;i--) buckets[i]=buckets[i-1]; - buckets[0]=0; - - for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; - I[0]=oldsize; - for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; - V[oldsize]=0; - for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; - I[0]=-1; - - for(h=1;I[0]!=-(oldsize+1);h+=h) { - len=0; - for(i=0;i<oldsize+1;) { - if(I[i]<0) { - len-=I[i]; - i-=I[i]; - } else { - if(len) I[i-len]=-len; - len=V[I[i]]+1-i; - split(I,V,i,len,h); - i+=len; - len=0; - }; - }; - if(len) I[i-len]=-len; - }; - - for(i=0;i<oldsize+1;i++) I[V[i]]=i; -} - -static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) -{ - off_t i; - - for(i=0;(i<oldsize)&&(i<newsize);i++) - if(old[i]!=new[i]) break; - - return i; -} - -static off_t search(off_t *I,u_char *old,off_t oldsize, - u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) -{ - off_t x,y; - - if(en-st<2) { - x=matchlen(old+I[st],oldsize-I[st],new,newsize); - y=matchlen(old+I[en],oldsize-I[en],new,newsize); - - if(x>y) { - *pos=I[st]; - return x; - } else { - *pos=I[en]; - return y; - } - }; - - x=st+(en-st)/2; - if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { - return search(I,old,oldsize,new,newsize,x,en,pos); - } else { - return search(I,old,oldsize,new,newsize,st,x,pos); - }; -} - -static void offtout(off_t x,u_char *buf) -{ - off_t y; - - if(x<0) y=-x; else y=x; - - buf[0]=y%256;y-=buf[0]; - y=y/256;buf[1]=y%256;y-=buf[1]; - y=y/256;buf[2]=y%256;y-=buf[2]; - y=y/256;buf[3]=y%256;y-=buf[3]; - y=y/256;buf[4]=y%256;y-=buf[4]; - y=y/256;buf[5]=y%256;y-=buf[5]; - y=y/256;buf[6]=y%256;y-=buf[6]; - y=y/256;buf[7]=y%256; - - if(x<0) buf[7]|=0x80; -} - -int main(int argc,char *argv[]) -{ - int fd; - u_char *old,*new; - off_t oldsize,newsize; - off_t *I,*V; - off_t scan,pos,len; - off_t lastscan,lastpos,lastoffset; - off_t oldscore,scsc; - off_t s,Sf,lenf,Sb,lenb; - off_t overlap,Ss,lens; - off_t i; - off_t dblen,eblen; - u_char *db,*eb; - u_char buf[8]; - u_char header[32]; - FILE * pf; - BZFILE * pfbz2; - int bz2err; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - - if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) || - ((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL); - - qsufsort(I,V,old,oldsize); - - free(V); - - /* Allocate newsize+1 bytes instead of newsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[2],O_RDONLY,0))<0) || - ((newsize=lseek(fd,0,SEEK_END))==-1) || - ((new=malloc(newsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,new,newsize)!=newsize) || - (close(fd)==-1)) err(1,"%s",argv[2]); - - if(((db=malloc(newsize+1))==NULL) || - ((eb=malloc(newsize+1))==NULL)) err(1,NULL); - dblen=0; - eblen=0; - - /* Create the patch file */ - if ((pf = fopen(argv[3], "w")) == NULL) - err(1, "%s", argv[3]); - - /* Header is - 0 8 "BSDIFF40" - 8 8 length of bzip2ed ctrl block - 16 8 length of bzip2ed diff block - 24 8 length of new file */ - /* File is - 0 32 Header - 32 ?? Bzip2ed ctrl block - ?? ?? Bzip2ed diff block - ?? ?? Bzip2ed extra block */ - memcpy(header,"BSDIFF40",8); - offtout(0, header + 8); - offtout(0, header + 16); - offtout(newsize, header + 24); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - - /* Compute the differences, writing ctrl as we go */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - scan=0;len=0; - lastscan=0;lastpos=0;lastoffset=0; - while(scan<newsize) { - oldscore=0; - - for(scsc=scan+=len;scan<newsize;scan++) { - len=search(I,old,oldsize,new+scan,newsize-scan, - 0,oldsize,&pos); - if (len > 64 * 1024) break; - - for(;scsc<scan+len;scsc++) - if((scsc+lastoffset<oldsize) && - (old[scsc+lastoffset] == new[scsc])) - oldscore++; - - if(((len==oldscore) && (len!=0)) || - (len>oldscore+8)) break; - - if((scan+lastoffset<oldsize) && - (old[scan+lastoffset] == new[scan])) - oldscore--; - }; - - if((len!=oldscore) || (scan==newsize)) { - s=0;Sf=0;lenf=0; - for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { - if(old[lastpos+i]==new[lastscan+i]) s++; - i++; - if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; - }; - - lenb=0; - if(scan<newsize) { - s=0;Sb=0; - for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { - if(old[pos-i]==new[scan-i]) s++; - if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; - }; - }; - - if(lastscan+lenf>scan-lenb) { - overlap=(lastscan+lenf)-(scan-lenb); - s=0;Ss=0;lens=0; - for(i=0;i<overlap;i++) { - if(new[lastscan+lenf-overlap+i]== - old[lastpos+lenf-overlap+i]) s++; - if(new[scan-lenb+i]== - old[pos-lenb+i]) s--; - if(s>Ss) { Ss=s; lens=i+1; }; - }; - - lenf+=lens-overlap; - lenb-=lens; - }; - - for(i=0;i<lenf;i++) - db[dblen+i]=new[lastscan+i]-old[lastpos+i]; - for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) - eb[eblen+i]=new[lastscan+lenf+i]; - - dblen+=lenf; - eblen+=(scan-lenb)-(lastscan+lenf); - - offtout(lenf,buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((scan-lenb)-(lastscan+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((pos-lenb)-(lastpos+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - lastscan=scan-lenb; - lastpos=pos-lenb; - lastoffset=pos-scan; - }; - }; - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed ctrl data */ - if ((len = ftello(pf)) == -1) - err(1, "ftello"); - offtout(len-32, header + 8); - - /* Write compressed diff data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, db, dblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed diff data */ - if ((newsize = ftello(pf)) == -1) - err(1, "ftello"); - offtout(newsize - len, header + 16); - - /* Write compressed extra data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Seek to the beginning, write the header, and close the file */ - if (fseeko(pf, 0, SEEK_SET)) - err(1, "fseeko"); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - if (fclose(pf)) - err(1, "fclose"); - - /* Free the memory we used */ - free(db); - free(eb); - free(I); - free(old); - free(new); - - return 0; -} diff --git a/src/bsdiff-4.3/bspatch.1 b/src/bsdiff-4.3/bspatch.1 deleted file mode 100644 index 82a2781aa7dc..000000000000 --- a/src/bsdiff-4.3/bspatch.1 +++ /dev/null @@ -1,59 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSPATCH 1 -.Os FreeBSD -.Sh NAME -.Nm bspatch -.Nd apply a patch built with bsdiff(1) -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -generates -.Ao Ar newfile Ac -from -.Ao Ar oldfile Ac -and -.Ao Ar patchfile Ac -where -.Ao Ar patchfile Ac -is a binary patch built by bsdiff(1). -.Pp -.Nm -uses memory equal to the size of -.Ao Ar oldfile Ac -plus the size of -.Ao Ar newfile Ac , -but can tolerate a very small working set without a dramatic loss -of performance. -.Sh SEE ALSO -.Xr bsdiff 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bspatch.c b/src/bsdiff-4.3/bspatch.c deleted file mode 100644 index f9d33ddd64a2..000000000000 --- a/src/bsdiff-4.3/bspatch.c +++ /dev/null @@ -1,224 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $"); -#endif - -#include <bzlib.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <err.h> -#include <errno.h> -#include <unistd.h> -#include <fcntl.h> -#include <sys/types.h> - -static off_t offtin(u_char *buf) -{ - off_t y; - - y=buf[7]&0x7F; - y=y*256;y+=buf[6]; - y=y*256;y+=buf[5]; - y=y*256;y+=buf[4]; - y=y*256;y+=buf[3]; - y=y*256;y+=buf[2]; - y=y*256;y+=buf[1]; - y=y*256;y+=buf[0]; - - if(buf[7]&0x80) y=-y; - - return y; -} - - -void writeFull(const char * name, int fd, - const unsigned char * buf, size_t count) -{ - while (count) { - ssize_t res = write(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - err(1,"writing to %s",name); - } - count -= res; - buf += res; - } -} - - -int main(int argc,char * argv[]) -{ - FILE * f, * cpf, * dpf, * epf; - BZFILE * cpfbz2, * dpfbz2, * epfbz2; - int cbz2err, dbz2err, ebz2err; - int fd; - ssize_t oldsize,newsize; - ssize_t bzctrllen,bzdatalen; - u_char header[32],buf[8]; - u_char *old, *new; - off_t oldpos,newpos; - off_t ctrl[3]; - off_t lenread; - off_t i; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Open patch file */ - if ((f = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - - /* - File format: - 0 8 "BSDIFF40" - 8 8 X - 16 8 Y - 24 8 sizeof(newfile) - 32 X bzip2(control block) - 32+X Y bzip2(diff block) - 32+X+Y ??? bzip2(extra block) - with control block a set of triples (x,y,z) meaning "add x bytes - from oldfile to x bytes from the diff block; copy y bytes from the - extra block; seek forwards in oldfile by z bytes". - */ - - /* Read header */ - if (fread(header, 1, 32, f) < 32) { - if (feof(f)) - errx(1, "Corrupt patch\n"); - err(1, "fread(%s)", argv[3]); - } - - /* Check for appropriate magic */ - if (memcmp(header, "BSDIFF40", 8) != 0) - errx(1, "Corrupt patch\n"); - - /* Read lengths from header */ - bzctrllen=offtin(header+8); - bzdatalen=offtin(header+16); - newsize=offtin(header+24); - if((bzctrllen<0) || (bzdatalen<0) || (newsize<0)) - errx(1,"Corrupt patch\n"); - - /* Close patch file and re-open it via libbzip2 at the right places */ - if (fclose(f)) - err(1, "fclose(%s)", argv[3]); - if ((cpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(cpf, 32, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)32); - if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err); - if ((dpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(dpf, 32 + bzctrllen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen)); - if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err); - if ((epf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen + bzdatalen)); - if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err); - - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - if((new=malloc(newsize+1))==NULL) err(1,NULL); - - oldpos=0;newpos=0; - while(newpos<newsize) { - /* Read control data */ - for(i=0;i<=2;i++) { - lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8); - if ((lenread < 8) || ((cbz2err != BZ_OK) && - (cbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - ctrl[i]=offtin(buf); - }; - - /* Sanity-check */ - if(newpos+ctrl[0]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read diff string */ - lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]); - if ((lenread < ctrl[0]) || - ((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Add old data to diff string */ - for(i=0;i<ctrl[0];i++) - if((oldpos+i>=0) && (oldpos+i<oldsize)) - new[newpos+i]+=old[oldpos+i]; - - /* Adjust pointers */ - newpos+=ctrl[0]; - oldpos+=ctrl[0]; - - /* Sanity-check */ - if(newpos+ctrl[1]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read extra string */ - lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]); - if ((lenread < ctrl[1]) || - ((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Adjust pointers */ - newpos+=ctrl[1]; - oldpos+=ctrl[2]; - }; - - /* Clean up the bzip2 reads */ - BZ2_bzReadClose(&cbz2err, cpfbz2); - BZ2_bzReadClose(&dbz2err, dpfbz2); - BZ2_bzReadClose(&ebz2err, epfbz2); - if (fclose(cpf) || fclose(dpf) || fclose(epf)) - err(1, "fclose(%s)", argv[3]); - - /* Write the new file */ - if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) - err(1,"%s",argv[2]); - writeFull(argv[2], fd, new, newsize); - if(close(fd)==-1) - err(1,"%s",argv[2]); - - free(new); - free(old); - - return 0; -} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h deleted file mode 100644 index a851ded6f907..000000000000 --- a/src/bsdiff-4.3/compat-include/err.h +++ /dev/null @@ -1,12 +0,0 @@ -/* Simulate BSD's <err.h> functionality. */ - -#ifndef COMPAT_ERR_H_INCLUDED -#define COMPAT_ERR_H_INCLUDED 1 - -#include <stdio.h> -#include <stdlib.h> - -#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) -#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) - -#endif diff --git a/src/bsdiff-4.3/local.mk b/src/bsdiff-4.3/local.mk deleted file mode 100644 index c957ceab0c0f..000000000000 --- a/src/bsdiff-4.3/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += bsdiff bspatch - -bsdiff_DIR := $(d) -bsdiff_SOURCES := $(d)/bsdiff.c -bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bsdiff_INSTALL_DIR = $(libexecdir)/nix - -bspatch_DIR := $(d) -bspatch_SOURCES := $(d)/bspatch.c -bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bspatch_INSTALL_DIR = $(libexecdir)/nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8ce2f3dfa6af..7ad9a4e46d83 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,6 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" +#include "download.hh" #include <algorithm> #include <cstring> @@ -238,12 +239,38 @@ void initGC() /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const string & in) +static Strings parseNixPath(const string & s) { - string marker = "\001//"; - auto res = tokenizeString<Strings>(replaceStrings(in, "://", marker), ":"); - for (auto & s : res) - s = replaceStrings(s, marker, "://"); + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + return res; } @@ -278,7 +305,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) /* Initialise the Nix expression search path. */ Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i, true); + for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); @@ -301,11 +328,15 @@ Path EvalState::checkSourcePath(const Path & path_) if (!restricted) return path_; /* Resolve symlinks. */ + debug(format("checking access to ‘%s’") % path_); Path path = canonPath(path_, true); - for (auto & i : searchPath) - if (path == i.second || isInDir(path, i.second)) + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + if (path == r.second || isInDir(path, r.second)) return path; + } /* To support import-from-derivation, allow access to anything in the store. FIXME: only allow access to paths that have been diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 40e05712bab1..80e369f2d68f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -26,9 +26,9 @@ typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, struct PrimOp { PrimOpFun fun; - unsigned int arity; + size_t arity; Symbol name; - PrimOp(PrimOpFun fun, unsigned int arity, Symbol name) + PrimOp(PrimOpFun fun, size_t arity, Symbol name) : fun(fun), arity(arity), name(name) { } }; @@ -56,7 +56,8 @@ typedef std::map<Path, Path> SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); -typedef list<std::pair<string, Path> > SearchPath; +typedef std::pair<std::string, std::string> SearchPathElem; +typedef std::list<SearchPathElem> SearchPath; /* Initialise the Boehm GC, if applicable. */ @@ -98,12 +99,14 @@ private: SearchPath searchPath; + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + public: EvalState(const Strings & _searchPath, ref<Store> store); ~EvalState(); - void addToSearchPath(const string & s, bool warn = false); + void addToSearchPath(const string & s); Path checkSourcePath(const Path & path); @@ -125,6 +128,9 @@ public: Path findFile(const string & path); Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); + /* Evaluate an expression to normal form, storing the result in value `v'. */ void eval(Expr * e, Value & v); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 996c2c5f4975..4889fe206a31 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -30,7 +30,7 @@ string DrvInfo::queryOutPath() } -DrvInfo::Outputs DrvInfo::queryOutputs() +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ @@ -55,7 +55,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs() } else outputs["out"] = queryOutPath(); } - return outputs; + if (!onlyOutputsToInstall || !attrs) + return outputs; + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value * outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + const auto errMsg = Error("this derivation has bad ‘meta.outputsToInstall’"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { + if ((*i)->type != tString) throw errMsg; + auto out = outputs.find((*i)->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); + } + return result; } @@ -192,8 +208,8 @@ typedef set<Bindings *> Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', - then put information about it in `drvs' (unless it's already in - `doneExprs'). The result boolean indicates whether it makes sense + then put information about it in `drvs' (unless it's already in `done'). + The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ static bool getDerivation(EvalState & state, Value & v, const string & attrPath, DrvInfos & drvs, Done & done, diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 365c66c8d710..37fcbe829d3c 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -42,7 +42,8 @@ public: string queryDrvPath(); string queryOutPath(); string queryOutputName(); - Outputs queryOutputs(); + /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); Value * queryMeta(const string & name); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 5de9ccc6d011..620050a13b05 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,7 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_CXXFLAGS := -Wno-deprecated-register diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 11dc7bb5ccdf..20ae1a696097 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -600,7 +600,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) } -void EvalState::addToSearchPath(const string & s, bool warn) +void EvalState::addToSearchPath(const string & s) { size_t pos = s.find('='); string prefix; @@ -612,16 +612,7 @@ void EvalState::addToSearchPath(const string & s, bool warn) path = string(s, pos + 1); } - if (isUri(path)) - path = makeDownloader()->downloadCached(store, path, true); - - path = absPath(path); - if (pathExists(path)) { - debug(format("adding path ‘%1%’ to the search path") % path); - /* Resolve symlinks in the path to support restricted mode. */ - searchPath.push_back(std::pair<string, Path>(prefix, canonPath(path, true))); - } else if (warn) - printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % path); + searchPath.emplace_back(prefix, path); } @@ -634,17 +625,19 @@ Path EvalState::findFile(const string & path) Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) { for (auto & i : searchPath) { - assert(!isUri(i.second)); - Path res; + std::string suffix; if (i.first.empty()) - res = i.second + "/" + path; + suffix = "/" + path; else { - if (path.compare(0, i.first.size(), i.first) != 0 || - (path.size() > i.first.size() && path[i.first.size()] != '/')) + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) continue; - res = i.second + - (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size())); + suffix = path.size() == s ? "" : "/" + string(path, s); } + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } format f = format( @@ -655,4 +648,35 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos } +std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem) +{ + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) return i->second; + + std::pair<bool, std::string> res; + + if (isUri(elem.second)) { + try { + res = { true, makeDownloader()->downloadCached(store, elem.second, true) }; + } catch (DownloadError & e) { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second); + res = { false, "" }; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) + res = { true, path }; + else { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second); + res = { false, "" }; + } + } + + debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second); + + searchPathResolved[elem.second] = res; + return res; +} + + } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aaef467c098b..51680ad62ee2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "util.hh" #include "value-to-json.hh" #include "value-to-xml.hh" +#include "primops.hh" #include <sys/types.h> #include <sys/stat.h> @@ -777,7 +778,6 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - PathSet context; for (unsigned int n = 0; n < args[0]->listSize(); ++n) { Value & v2(*args[0]->listElems()[n]); state.forceAttrs(v2, pos); @@ -790,21 +790,23 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError(format("attribute ‘path’ missing, at %1%") % pos); - string path = state.coerceToPath(pos, *i->value, context); - searchPath.push_back(std::pair<string, Path>(prefix, state.checkSourcePath(path))); - } + PathSet context; + string path = state.coerceToString(pos, *i->value, context, false, false); - string path = state.forceStringNoCtx(*args[1], pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") + % path % e.path % pos); + } - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + searchPath.emplace_back(prefix, path); } - mkPath(v, state.findFile(searchPath, path, pos).c_str()); + string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } /* Read a directory (without . or ..) */ @@ -1725,6 +1727,16 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args *************************************************************/ +RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; + + +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +{ + if (!primOps) primOps = new PrimOps; + primOps->emplace_back(name, arity, fun); +} + + void EvalState::createBaseEnv() { baseEnv.up = 0; @@ -1889,6 +1901,10 @@ void EvalState::createBaseEnv() } addConstant("__nixPath", v); + if (RegisterPrimOp::primOps) + for (auto & primOp : *RegisterPrimOp::primOps) + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh new file mode 100644 index 000000000000..39d23b04a5ce --- /dev/null +++ b/src/libexpr/primops.hh @@ -0,0 +1,15 @@ +#include "eval.hh" + +#include <tuple> +#include <vector> + +namespace nix { + +struct RegisterPrimOp +{ + typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; + static PrimOps * primOps; + RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); +}; + +} diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e883967b71a1..ed997052be20 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -24,15 +24,9 @@ namespace nix { -volatile sig_atomic_t blockInt = 0; - - static void sigintHandler(int signo) { - if (!blockInt) { - _isInterrupted = 1; - blockInt = 1; - } + _isInterrupted = 1; } @@ -117,9 +111,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - // FIXME: do we need this? It's not thread-safe. - std::ios::sync_with_stdio(false); - if (getEnv("IN_SYSTEMD") == "1") logType = ltSystemd; @@ -290,8 +281,7 @@ int handleExceptions(const string & programName, std::function<void()> fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - blockInt = 1; /* ignore further SIGINTs */ - _isInterrupted = 0; + interruptThrown = true; throw; } } catch (Exit & e) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e6b68d486b47..473a0b2614bb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -156,7 +156,7 @@ bool BinaryCacheStore::isValidPath(const Path & storePath) return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::dumpPath(const Path & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) { auto res = readNarInfo(storePath); @@ -189,7 +189,7 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink auto res = readNarInfo(storePath); - dumpPath(storePath, sink); + narFromPath(storePath, sink); // FIXME: check integrity of NAR. @@ -271,7 +271,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, StringSink sink; Hash h; if (recursive) { - nix::dumpPath(srcPath, sink, filter); + dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { auto s = readFile(srcPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index f14b4859b04e..9e7b0ad9a384 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -124,7 +124,7 @@ public: Path addTextToStore(const string & name, const string & s, const PathSet & references, bool repair = false) override; - void dumpPath(const Path & path, Sink & sink) override; + void narFromPath(const Path & path, Sink & sink) override; void exportPath(const Path & path, bool sign, Sink & sink) override; @@ -156,12 +156,6 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override { notImpl(); } - PathSet queryFailedPaths() override - { return {}; } - - void clearFailedPaths(const PathSet & paths) override - { } - void optimiseStore() override { } @@ -170,6 +164,9 @@ public: ref<FSAccessor> getFSAccessor() override; + void addSignatures(const Path & storePath, const StringSet & sigs) + { notImpl(); } + }; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ed4e0f659da3..ba3f3a371d8c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -22,6 +22,7 @@ #include <sys/stat.h> #include <sys/utsname.h> #include <sys/select.h> +#include <sys/resource.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> @@ -239,6 +240,9 @@ private: /* Last time the goals in `waitingForAWhile' where woken up. */ time_t lastWokenUp; + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + public: /* Set if at least one derivation had a BuildError (i.e. permanent @@ -304,6 +308,12 @@ public: void waitForInput(); unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); }; @@ -1038,11 +1048,6 @@ void DerivationGoal::haveDerivation() return; } - /* Check whether any output previously failed to build. If so, - don't bother. */ - for (auto & i : invalidOutputs) - if (pathFailed(i)) return; - /* Reject doing a hash build of anything other than a fixed-output derivation. */ if (buildMode == bmHash) { @@ -1159,7 +1164,7 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ PathSet broken; for (auto & i : outputClosure) { - if (worker.store.pathContentsGood(i)) continue; + if (worker.pathContentsGood(i)) continue; printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") @@ -1313,12 +1318,6 @@ void DerivationGoal::tryToBuild() deletePath(path); } - /* Check again whether any output previously failed to build, - because some other process may have tried and failed before we - acquired the lock. */ - for (auto & i : drv->outputs) - if (pathFailed(i.second.path)) return; - /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ @@ -1540,17 +1539,6 @@ void DerivationGoal::buildDone() statusOk(status) ? BuildResult::OutputRejected : fixedOutput || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; - - /* Register the outputs of this build as "failed" so we - won't try to build them again (negative caching). - However, don't do this for fixed-output derivations, - since they're likely to fail for transient reasons - (e.g., fetchurl not being able to access the network). - Hook errors (like communication problems with the - remote machine) shouldn't be cached either. */ - if (settings.cacheFailure && !fixedOutput && !diskFull) - for (auto & i : drv->outputs) - worker.store.registerFailedPath(i.second.path); } done(st, e.msg()); @@ -2386,6 +2374,12 @@ void DerivationGoal::runChild() if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); #endif + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + /* Fill in the environment. */ Strings envStrs; for (auto & i : env) @@ -2743,6 +2737,15 @@ void DerivationGoal::registerOutputs() throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") % drvPath % path); } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + continue; } @@ -2790,7 +2793,7 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.markContentsGood(path); } ValidPathInfo info; @@ -2799,6 +2802,9 @@ void DerivationGoal::registerOutputs() info.narSize = hash.second; info.references = references; info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + infos.push_back(info); } @@ -2965,30 +2971,13 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) if (!wantOutput(i.first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i.second.path) && - (!checkHash || worker.store.pathContentsGood(i.second.path)); + (!checkHash || worker.pathContentsGood(i.second.path)); if (good == returnValid) result.insert(i.second.path); } return result; } -bool DerivationGoal::pathFailed(const Path & path) -{ - if (!settings.cacheFailure) return false; - - if (!worker.store.hasPathFailed(path)) return false; - - printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); - - done(BuildResult::CachedFailure); - - return true; -} - - Path DerivationGoal::addHashRewrite(const Path & path) { string h1 = string(path, settings.nixStore.size() + 1, 32); @@ -3010,7 +2999,7 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg) amDone(result.success() ? ecSuccess : ecFailed); if (result.status == BuildResult::TimedOut) worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure || result.status == BuildResult::CachedFailure) + if (result.status == BuildResult::PermanentFailure) worker.permanentFailure = true; } @@ -3373,7 +3362,7 @@ void SubstitutionGoal::finished() outputLock->setDeletion(true); outputLock.reset(); - worker.store.markContentsGood(storePath); + worker.markContentsGood(storePath); printMsg(lvlChatty, format("substitution of path ‘%1%’ succeeded") % storePath); @@ -3773,6 +3762,32 @@ unsigned int Worker::exitStatus() } +bool Worker::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path ‘%1%’...") % path); + ValidPathInfo info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info.narHash.type, path); + Hash nullHash(htSHA256); + res = info.narHash == nullHash || info.narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); + return res; +} + + +void Worker::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + ////////////////////////////////////////////////////////////////////// diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index caba22c1e274..747483afb30b 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,5 +1,6 @@ #include "crypto.hh" #include "util.hh" +#include "globals.hh" #if HAVE_SODIUM #include <sodium.h> @@ -98,4 +99,28 @@ bool verifyDetached(const std::string & data, const std::string & sig, #endif } +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.get("binary-cache-public-keys", Strings())) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + } diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 38d5fe2a80c6..9110af3aa9e5 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -49,4 +49,6 @@ typedef std::map<std::string, PublicKey> PublicKeys; bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); +PublicKeys getDefaultPublicKeys(); + } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 4776d0091685..7277751b48e7 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -18,6 +18,14 @@ double getTime() return tv.tv_sec + (tv.tv_usec / 1000000.0); } +std::string resolveUri(const std::string & uri) +{ + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + else + return uri; +} + struct CurlDownloader : public Downloader { CURL * curl; @@ -179,11 +187,9 @@ struct CurlDownloader : public Downloader if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; long httpStatus = -1; - if (res == CURLE_HTTP_RETURNED_ERROR) - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); if (res != CURLE_OK) { - long httpStatus = 0; Error err = httpStatus == 404 ? NotFound : httpStatus == 403 ? Forbidden : Misc; @@ -199,7 +205,7 @@ struct CurlDownloader : public Downloader DownloadResult download(string url, const DownloadOptions & options) override { DownloadResult res; - if (fetch(url, options)) { + if (fetch(resolveUri(url), options)) { res.cached = false; res.data = data; } else @@ -209,15 +215,15 @@ struct CurlDownloader : public Downloader } }; - ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); } - -Path Downloader::downloadCached(ref<Store> store, const string & url, bool unpack) +Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack) { + auto url = resolveUri(url_); + Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; createDirs(cacheDir); @@ -302,10 +308,11 @@ Path Downloader::downloadCached(ref<Store> store, const string & url, bool unpac bool isUri(const string & s) { + if (s.compare(0, 8, "channel:") == 0) return true; size_t pos = s.find("://"); if (pos == string::npos) return false; string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file"; + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel"; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e082f67143a3..52afa1b14e03 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -147,35 +147,36 @@ Path Store::addPermRoot(const Path & _storePath, void LocalStore::addTempRoot(const Path & path) { + auto state(_state.lock()); + /* Create the temporary roots file for this process. */ - if (fdTempRoots == -1) { + if (state->fdTempRoots == -1) { while (1) { Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); createDirs(dir); - fnTempRoots = (format("%1%/%2%") - % dir % getpid()).str(); + state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str(); AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) + if (pathExists(state->fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ - unlink(fnTempRoots.c_str()); + unlink(state->fnTempRoots.c_str()); - fdTempRoots = openLockFile(fnTempRoots, true); + state->fdTempRoots = openLockFile(state->fnTempRoots, true); fdGCLock.close(); - debug(format("acquiring read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); /* Check whether the garbage collector didn't get in our way. */ struct stat st; - if (fstat(fdTempRoots, &st) == -1) - throw SysError(format("statting ‘%1%’") % fnTempRoots); + if (fstat(state->fdTempRoots, &st) == -1) + throw SysError(format("statting ‘%1%’") % state->fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -187,15 +188,15 @@ void LocalStore::addTempRoot(const Path & path) /* Upgrade the lock to a write lock. This will cause us to block if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltWrite, true); + debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltWrite, true); string s = path + '\0'; - writeFull(fdTempRoots, s); + writeFull(state->fdTempRoots, s); /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e704837e8798..b6078253319f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -52,7 +52,6 @@ Settings::Settings() keepLog = true; compressLog = true; maxLogSize = 0; - cacheFailure = false; pollInterval = 5; checkRootReachability = false; gcKeepOutputs = false; @@ -175,7 +174,6 @@ void Settings::update() _get(keepLog, "build-keep-log"); _get(compressLog, "build-compress-log"); _get(maxLogSize, "build-max-log-size"); - _get(cacheFailure, "build-cache-failure"); _get(pollInterval, "build-poll-interval"); _get(checkRootReachability, "gc-check-reachability"); _get(gcKeepOutputs, "gc-keep-outputs"); @@ -196,7 +194,6 @@ void Settings::update() if (getEnv("NIX_OTHER_STORES") != "") substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); #endif - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); if (useSshSubstituter && !sshSubstituterHosts.empty()) substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 60b11afe6088..572fa7188c14 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -168,9 +168,6 @@ struct Settings { before being killed (0 means no limit). */ unsigned long maxLogSize; - /* Whether to cache build failures. */ - bool cacheFailure; - /* How often (in seconds) to poll for locks. */ unsigned int pollInterval; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1d707c043a72..8a719db150aa 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -10,7 +10,7 @@ private: Path cacheUri; - ref<Downloader> downloader; + Pool<Downloader> downloaders; public: @@ -18,7 +18,9 @@ public: const Path & secretKeyFile, const Path & _cacheUri) : BinaryCacheStore(localStore, secretKeyFile) , cacheUri(_cacheUri) - , downloader(makeDownloader()) + , downloaders( + std::numeric_limits<size_t>::max(), + []() { return makeDownloader(); }) { if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -36,13 +38,16 @@ protected: bool fileExists(const std::string & path) override { try { + auto downloader(downloaders.get()); DownloadOptions options; options.showProgress = DownloadOptions::no; options.head = true; downloader->download(cacheUri + "/" + path, options); return true; } catch (DownloadError & e) { - if (e.error == Downloader::NotFound) + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) return false; throw; } @@ -55,6 +60,7 @@ protected: std::string getFile(const std::string & path) override { + auto downloader(downloaders.get()); DownloadOptions options; options.showProgress = DownloadOptions::no; return downloader->download(cacheUri + "/" + path, options).data; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 87c216ec9f95..303c3af27b8d 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -23,7 +23,7 @@ struct LocalStoreAccessor : public FSAccessor struct stat st; if (lstat(path.c_str(), &st)) { - if (errno == ENOENT) return {Type::tMissing, 0, false}; + if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; throw SysError(format("getting status of ‘%1%’") % path); } @@ -69,9 +69,11 @@ ref<FSAccessor> LocalFSStore::getFSAccessor() return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this())); } -void LocalFSStore::dumpPath(const Path & path, Sink & sink) +void LocalFSStore::narFromPath(const Path & path, Sink & sink) { - nix::dumpPath(path, sink); + if (!isValidPath(path)) + throw Error(format("path ‘%s’ is not valid") % path); + dumpPath(path, sink); } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 532d511ef8bf..d6e1e4d7e0d4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include <iostream> #include <algorithm> #include <cstring> +#include <atomic> #include <sys/types.h> #include <sys/stat.h> @@ -36,168 +37,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -[[noreturn]] static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -217,20 +56,21 @@ void checkStoreNotSymlink() LocalStore::LocalStore() - : reservedPath(settings.nixDBPath + "/reserved") - , didSetSubstituterEnv(false) + : linksDir(settings.nixStore + "/.links") + , reservedPath(settings.nixDBPath + "/reserved") + , schemaPath(settings.nixDBPath + "/schema") { - schemaPath = settings.nixDBPath + "/schema"; + auto state(_state.lock()); if (settings.readOnlyMode) { - openDB(false); + openDB(*state, false); return; } /* Create missing state directories if they don't already exist. */ createDirs(settings.nixStore); makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); + createDirs(linksDir); Path profilesDir = settings.nixStateDir + "/profiles"; createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); @@ -302,7 +142,7 @@ LocalStore::LocalStore() } catch (SysError & e) { if (e.errNo != EACCES) throw; settings.readOnlyMode = true; - openDB(false); + openDB(*state, false); return; } @@ -320,7 +160,7 @@ LocalStore::LocalStore() else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - openDB(true); + openDB(*state, true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } @@ -346,21 +186,41 @@ LocalStore::LocalStore() have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 7) { upgradeStore7(); openDB(true); } + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); } - else openDB(false); + else openDB(*state, false); } LocalStore::~LocalStore() { + auto state(_state.lock()); + try { - for (auto & i : runningSubstituters) { + for (auto & i : state->runningSubstituters) { if (i.second.disabled) continue; i.second.to.close(); i.second.from.close(); @@ -373,9 +233,9 @@ LocalStore::~LocalStore() } try { - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); + if (state->fdTempRoots != -1) { + state->fdTempRoots.close(); + unlink(state->fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -401,13 +261,14 @@ bool LocalStore::haveWriteAccess() } -void LocalStore::openDB(bool create) +void LocalStore::openDB(State & state, bool create) { if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ string dbPath = settings.nixDBPath + "/db.sqlite"; + auto & db(state.db); if (sqlite3_open_v2(dbPath.c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); @@ -460,40 +321,31 @@ void LocalStore::openDB(bool create) } /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, + state.stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);"); + state.stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;"); + state.stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, + state.stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;"); + state.stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, + state.stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, + state.stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, + state.stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, + state.stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, + state.stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, + state.stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + state.stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -688,23 +540,19 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.narHash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path ‘%1%’ in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + state.stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.narHash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -721,12 +569,11 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che if (checkOutputs) checkDerivationOutputs(info.path, drv); for (auto & i : drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i.first); - stmtAddDerivationOutput.bind(i.second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for ‘%1%’ in database") % info.path); + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } @@ -734,79 +581,6 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che } -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path ‘%1%’") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - for (auto & i : paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path ‘%1%’ in database") % i); - } - - txn.commit(); - } end_retry_sqlite; -} - - Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); @@ -828,153 +602,117 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) assertStorePath(path); - retry_sqlite { + return retrySQLite<ValidPathInfo>([&]() { + auto state(_state.lock()); /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - stmtQueryPathInfo.bind(path); + if (!useQueryPathInfo.next()) + throw Error(format("path ‘%1%’ is not valid") % path); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + info.id = useQueryPathInfo.getInt(0); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.narHash = parseHashField(path, useQueryPathInfo.getStr(1)); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.narHash = parseHashField(path, s); + info.registrationTime = useQueryPathInfo.getInt(2); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); if (s) info.deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info.narSize = useQueryPathInfo.getInt(4); - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); + info.ultimate = useQueryPathInfo.getInt(5) == 1; - stmtQueryReferences.bind(info.id); + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info.sigs = tokenizeString<StringSet>(s, " "); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info.id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferences.next()) + info.references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.narHash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.narHash)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } -bool LocalStore::isValidPath_(const Path & path) +bool LocalStore::isValidPath(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return state.stmtQueryPathInfo.use()(path).next(); } bool LocalStore::isValidPath(const Path & path) { - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath(*state, path); + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { - PathSet res; - for (auto & i : paths) - if (isValidPath_(i)) res.insert(i); - return res; - } end_retry_sqlite; + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; + }); } -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } @@ -988,67 +726,51 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of ‘%1%’") % path); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of ‘%1%’") % path); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of ‘%1%’") % path); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } @@ -1058,29 +780,28 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); + return retrySQLite<Path>([&]() { + auto state(_state.lock()); + + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + if (!useQueryPathFromHashPart.next()) return ""; - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; + }); } void LocalStore::setSubstituterEnv() { - if (didSetSubstituterEnv) return; + static std::atomic_flag done; + + if (done.test_and_set()) return; /* Pass configuration options (including those overridden with --option) to substituters. */ setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; } @@ -1202,10 +923,12 @@ template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & r PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { + auto state(_state.lock()); + PathSet res; for (auto & i : settings.substituters) { if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[i]); + RunningSubstituter & run(state->runningSubstituters[i]); startSubstituter(i, run); if (run.disabled) continue; string s = "have "; @@ -1222,6 +945,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) res.insert(path); } } + return res; } @@ -1229,7 +953,9 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) void LocalStore::querySubstitutablePathInfos(const Path & substituter, PathSet & paths, SubstitutablePathInfos & infos) { - RunningSubstituter & run(runningSubstituters[substituter]); + auto state(_state.lock()); + + RunningSubstituter & run(state->runningSubstituters[substituter]); startSubstituter(substituter, run); if (run.disabled) return; @@ -1286,28 +1012,31 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { - SQLiteTxn txn(db); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); PathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); - if (isValidPath_(i.path)) - updatePathInfo(i); + if (isValidPath(*state, i.path)) + updatePathInfo(*state, i); else - addValidPath(i, false); + addValidPath(*state, i, false); paths.insert(i.path); } for (auto & i : infos) { - unsigned long long referrer = queryValidPathId(i.path); + auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - addReference(referrer, queryValidPathId(j)); + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1328,24 +1057,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSortPaths(paths); txn.commit(); - } end_retry_sqlite; + }); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) +void LocalStore::invalidatePath(State & state, const Path & path) { debug(format("invalidating path ‘%1%’") % path); - drvHashes.erase(path); - - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path ‘%1%’ in database") % path); + state.stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ @@ -1397,6 +1119,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, info.path = dstPath; info.narHash = hash.first; info.narSize = hash.second; + info.ultimate = true; registerValidPath(info); } @@ -1411,14 +1134,13 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); - debug(format("adding ‘%1%’ to the store") % srcPath); /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; if (recursive) - nix::dumpPath(srcPath, sink, filter); + dumpPath(srcPath, sink, filter); else sink.s = make_ref<std::string>(readFile(srcPath)); @@ -1456,6 +1178,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s, info.narHash = hash; info.narSize = sink.s->size(); info.references = references; + info.ultimate = true; registerValidPath(info); } @@ -1712,20 +1435,22 @@ void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteTxn txn(db); + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); + if (isValidPath(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") % path % showPaths(referrers)); - invalidatePath(path); + invalidatePath(*state, path); } txn.commit(); - } end_retry_sqlite; + }); } @@ -1790,7 +1515,10 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) update = true; } - if (update) updatePathInfo(info); + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, info); + } } @@ -1820,7 +1548,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (!isStorePath(path)) { printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); return; } @@ -1838,7 +1567,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (canInvalidate) { printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); } else { printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); if (repair) @@ -1858,32 +1588,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.narHash.type, path); - Hash nullHash(htSHA256); - res = info.narHash == nullHash || info.narHash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) @@ -1938,8 +1642,41 @@ void LocalStore::upgradeStore7() void LocalStore::vacuumDB() { - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); + auto state(_state.lock()); + + if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "vacuuming SQLite database"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = queryPathInfo(storePath); + + info.sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.get("secret-key-files", Strings()); + + for (auto & secretKeyFile : secretKeyFiles) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c7ea9e5038b4..14ff92c35cc5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,15 +1,14 @@ #pragma once -#include <string> -#include <unordered_set> +#include "sqlite.hh" +#include "pathlocks.hh" #include "store-api.hh" +#include "sync.hh" #include "util.hh" -#include "pathlocks.hh" - -class sqlite3; -class sqlite3_stmt; +#include <string> +#include <unordered_set> namespace nix { @@ -18,8 +17,8 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. */ -const int nixSchemaVersion = 7; + Nix 1.0. Version 7 is Nix 1.3. Version 9 is 1.12. */ +const int nixSchemaVersion = 9; extern string drvsLogDir; @@ -52,43 +51,46 @@ struct RunningSubstituter }; -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - class LocalStore : public LocalFSStore { private: - typedef std::map<Path, RunningSubstituter> RunningSubstituters; - RunningSubstituters runningSubstituters; - Path linksDir; + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; - Path reservedPath; + struct State + { + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; + + /* The file to which we write our temporary roots. */ + Path fnTempRoots; + AutoCloseFD fdTempRoots; + + typedef std::map<Path, RunningSubstituter> RunningSubstituters; + RunningSubstituters runningSubstituters; + + }; + + Sync<State, std::recursive_mutex> _state; + + const Path linksDir; + const Path reservedPath; + const Path schemaPath; public: @@ -188,90 +190,31 @@ public: void registerValidPaths(const ValidPathInfos & infos); - /* Register that the build of a derivation with output `path' has - failed. */ - void registerFailedPath(const Path & path); - - /* Query whether `path' previously failed to build. */ - bool hasPathFailed(const Path & path); - - PathSet queryFailedPaths() override; - - void clearFailedPaths(const PathSet & paths) override; - void vacuumDB(); /* Repair the contents of the given path by redownloading it using a substituter (if available). */ void repairPath(const Path & path); - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const Path & path); - - void markContentsGood(const Path & path); - void setSubstituterEnv(); -private: - - Path schemaPath; - - /* Lock file used for upgrading. */ - AutoCloseFD globalLock; - - /* The SQLite database object. */ - SQLite db; - - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtRegisterFailedPath; - SQLiteStmt stmtHasPathFailed; - SQLiteStmt stmtQueryFailedPaths; - SQLiteStmt stmtClearFailedPath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - - /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; - - bool didSetSubstituterEnv; - - /* The file to which we write our temporary roots. */ - Path fnTempRoots; - AutoCloseFD fdTempRoots; - - int getSchema(); - -public: + void addSignatures(const Path & storePath, const StringSet & sigs) override; static bool haveWriteAccess(); private: - void openDB(bool create); - - void makeStoreWritable(); - - unsigned long long queryValidPathId(const Path & path); + int getSchema(); - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + void openDB(State & state, bool create); - void addReference(unsigned long long referrer, unsigned long long reference); + void makeStoreWritable(); - void appendReferrer(const Path & from, const Path & to, bool lock); + uint64_t queryValidPathId(State & state, const Path & path); - void rewriteReferrers(const Path & path, bool purge, PathSet referrers); + uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void invalidatePath(const Path & path); + void invalidatePath(State & state, const Path & path); /* Delete a path from the Nix store. */ void invalidatePathChecked(const Path & path); @@ -279,7 +222,7 @@ private: void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths, bool repair, bool & errors); - void updatePathInfo(const ValidPathInfo & info); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); void upgradeStore7(); @@ -327,8 +270,14 @@ private: void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(const Path & path); - void queryReferrers_(const Path & path, PathSet & referrers); + bool isValidPath(State & state, const Path & path); + void queryReferrers(State & state, const Path & path, PathSet & referrers); + + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + void signPathInfo(ValidPathInfo & info); + + friend class DerivationGoal; }; diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 9028370ac707..680facdcfeb8 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,4 +1,3 @@ -#include "crypto.hh" #include "globals.hh" #include "nar-info.hh" @@ -104,15 +103,6 @@ std::string NarInfo::to_string() const return res; } -std::string NarInfo::fingerprint() const -{ - return - "1;" + path + ";" - + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", references); -} - Strings NarInfo::shortRefs() const { Strings refs; @@ -121,18 +111,4 @@ Strings NarInfo::shortRefs() const return refs; } -void NarInfo::sign(const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint())); -} - -unsigned int NarInfo::checkSignatures(const PublicKeys & publicKeys) const -{ - unsigned int good = 0; - for (auto & sig : sigs) - if (verifyDetached(fingerprint(), sig, publicKeys)) - good++; - return good; -} - } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 2d04e452611e..3c783cf83fef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -20,20 +20,6 @@ struct NarInfo : ValidPathInfo std::string to_string() const; - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint() const; - - void sign(const SecretKey & secretKey); - - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys. */ - unsigned int checkSignatures(const PublicKeys & publicKeys) const; - private: Strings shortRefs() const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3c417b4a6d06..761e835481a8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -61,27 +61,15 @@ ref<RemoteStore::Connection> RemoteStore::openConnection() string socketPath = settings.nixDaemonSocketFile; - /* Urgh, sockaddr_un allows path names of only 108 characters. So - chdir to the socket directory so that we can pass a relative - path name. !!! this is probably a bad idea in multi-threaded - applications... */ - AutoCloseFD fdPrevDir = open(".", O_RDONLY); - if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - if (chdir(dirOf(socketPath).c_str()) == -1) throw SysError(format("couldn't change to directory of ‘%1%’") % socketPath); - Path socketPathRel = "./" + baseNameOf(socketPath); - struct sockaddr_un addr; addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path ‘%1%’ is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket path ‘%1%’ is too long") % socketPath); + strcpy(addr.sun_path, socketPath.c_str()); if (connect(conn->fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); - if (fchdir(fdPrevDir) == -1) - throw SysError("couldn't change back to previous directory"); - conn->from.fd = conn->fd; conn->to.fd = conn->fd; @@ -264,6 +252,10 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path) info.references = readStorePaths<PathSet>(conn->from); info.registrationTime = readInt(conn->from); info.narSize = readLongLong(conn->from); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + info.ultimate = readInt(conn->from) != 0; + info.sigs = readStrings<StringSet>(conn->from); + } return info; } @@ -366,7 +358,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, try { conn->to.written = 0; conn->to.warn = true; - nix::dumpPath(srcPath, conn->to, filter); + dumpPath(srcPath, conn->to, filter); conn->to.warn = false; conn->processStderr(); } catch (SysError & e) { @@ -528,37 +520,30 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) } -PathSet RemoteStore::queryFailedPaths() +void RemoteStore::optimiseStore() { auto conn(connections->get()); - conn->to << wopQueryFailedPaths; + conn->to << wopOptimiseStore; conn->processStderr(); - return readStorePaths<PathSet>(conn->from); + readInt(conn->from); } -void RemoteStore::clearFailedPaths(const PathSet & paths) +bool RemoteStore::verifyStore(bool checkContents, bool repair) { auto conn(connections->get()); - conn->to << wopClearFailedPaths << paths; + conn->to << wopVerifyStore << checkContents << repair; conn->processStderr(); - readInt(conn->from); + return readInt(conn->from) != 0; } -void RemoteStore::optimiseStore() -{ - auto conn(connections->get()); - conn->to << wopOptimiseStore; - conn->processStderr(); - readInt(conn->from); -} -bool RemoteStore::verifyStore(bool checkContents, bool repair) +void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) { auto conn(connections->get()); - conn->to << wopVerifyStore << checkContents << repair; + conn->to << wopAddSignatures << storePath << sigs; conn->processStderr(); - return readInt(conn->from) != 0; + readInt(conn->from); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 85c8292c7698..45bc41804ccf 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -85,14 +85,12 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - PathSet queryFailedPaths() override; - - void clearFailedPaths(const PathSet & paths) override; - void optimiseStore() override; bool verifyStore(bool checkContents, bool repair) override; + void addSignatures(const Path & storePath, const StringSet & sigs) override; + private: struct Connection diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index c1b4a689afcb..91878af1580d 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -4,7 +4,9 @@ create table if not exists ValidPaths ( hash text not null, registrationTime integer not null, deriver text, - narSize integer + narSize integer, + ultimate integer, -- null implies "false" + sigs text -- space-separated ); create table if not exists Refs ( @@ -37,8 +39,3 @@ create table if not exists DerivationOutputs ( ); create index if not exists IndexDerivationOutputs on DerivationOutputs(path); - -create table if not exists FailedPaths ( - path text primary key not null, - time integer not null -); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc new file mode 100644 index 000000000000..f93fa0857588 --- /dev/null +++ b/src/libstore/sqlite.cc @@ -0,0 +1,167 @@ +#include "sqlite.hh" +#include "util.hh" + +#include <sqlite3.h> + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() +{ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; +} + +int SQLiteStmt::Use::step() +{ + return sqlite3_step(stmt); +} + +void SQLiteStmt::Use::exec() +{ + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh new file mode 100644 index 000000000000..326e4a4855b7 --- /dev/null +++ b/src/libstore/sqlite.hh @@ -0,0 +1,102 @@ +#pragma once + +#include <functional> +#include <string> + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } + void create(sqlite3 * db, const std::string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + }; + + Use use() + { + return Use(*this); + } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template<typename T> +T retrySQLite(std::function<T()> fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 378233654225..cc91ed287768 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,6 @@ -#include "store-api.hh" +#include "crypto.hh" #include "globals.hh" +#include "store-api.hh" #include "util.hh" @@ -135,14 +136,14 @@ void checkStoreName(const string & name) if <type> = "source": the serialisation of the path from which this store path is copied, as returned by hashPath() - if <type> = "output:out": + if <type> = "output:<id>": for non-fixed derivation outputs: the derivation (see hashDerivationModulo() in primops.cc) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:<rec><algo>:<hash>:", where - <rec> = "r:" for recursive (path) hashes, or "" or flat + <rec> = "r:" for recursive (path) hashes, or "" for flat (file) hashes <algo> = "md5", "sha1" or "sha256" <hash> = base-16 representation of the path or flat hash of @@ -309,6 +310,41 @@ void Store::exportPaths(const Paths & paths, } +std::string ValidPathInfo::fingerprint() const +{ + if (narSize == 0 || narHash.type == htUnknown) + throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known") + % path); + return + "1;" + path + ";" + + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", references); +} + + +void ValidPathInfo::sign(const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint())); +} + + +unsigned int ValidPathInfo::checkSignatures(const PublicKeys & publicKeys) const +{ + unsigned int good = 0; + for (auto & sig : sigs) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 1a7440148322..ae5631ba0b7c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,6 +2,7 @@ #include "hash.hh" #include "serialise.hh" +#include "crypto.hh" #include <string> #include <limits> @@ -95,8 +96,8 @@ struct ValidPathInfo Hash narHash; PathSet references; time_t registrationTime = 0; - unsigned long long narSize = 0; // 0 = unknown - unsigned long long id; // internal use only + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only /* Whether the path is ultimately trusted, that is, it was built locally or is content-addressable (e.g. added via addToStore() @@ -112,6 +113,23 @@ struct ValidPathInfo && narHash == i.narHash && references == i.references; } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey & secretKey); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys. */ + unsigned int checkSignatures(const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; }; typedef list<ValidPathInfo> ValidPathInfos; @@ -130,7 +148,6 @@ struct BuildResult InputRejected, OutputRejected, TransientFailure, // possibly transient - CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -223,7 +240,7 @@ public: const PathSet & references, bool repair = false) = 0; /* Write a NAR dump of a store path. */ - virtual void dumpPath(const Path & path, Sink & sink) = 0; + virtual void narFromPath(const Path & path, Sink & sink) = 0; /* Export a store path, that is, create a NAR dump of the store path and append its references and its deriver. Optionally, a @@ -307,13 +324,6 @@ public: /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - /* Return the set of paths that have failed to build.*/ - virtual PathSet queryFailedPaths() = 0; - - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ - virtual void clearFailedPaths(const PathSet & paths) = 0; - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ @@ -331,6 +341,10 @@ public: /* Return an object to access files in the Nix store. */ virtual ref<FSAccessor> getFSAccessor() = 0; + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + /* Utility functions. */ /* Read a derivation, after ensuring its existence through @@ -365,7 +379,7 @@ public: class LocalFSStore : public Store { public: - void dumpPath(const Path & path, Sink & sink) override; + void narFromPath(const Path & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; }; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 7d9bcb58a249..c10598d5d301 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10f +#define PROTOCOL_VERSION 0x110 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -45,6 +45,7 @@ typedef enum { wopOptimiseStore = 34, wopVerifyStore = 35, wopBuildDerivation = 36, + wopAddSignatures = 37, } WorkerOp; diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 4dae3305433f..2e5d2672e5f0 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = -llzma $(OPENSSL_LIBS) +libutil_LDFLAGS = -llzma -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index c99c098ac9c6..ebe64ffbdab7 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -22,11 +22,11 @@ namespace nix { scope. */ -template<class T> +template<class T, class M = std::mutex> class Sync { private: - std::mutex mutex; + M mutex; T data; public: @@ -38,7 +38,7 @@ public: { private: Sync * s; - std::unique_lock<std::mutex> lk; + std::unique_lock<M> lk; friend Sync; Lock(Sync * s) : s(s), lk(s->mutex) { } public: diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc new file mode 100644 index 000000000000..743038b588a7 --- /dev/null +++ b/src/libutil/thread-pool.cc @@ -0,0 +1,82 @@ +#include "thread-pool.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _nrThreads) + : nrThreads(_nrThreads) +{ + if (!nrThreads) { + nrThreads = std::thread::hardware_concurrency(); + if (!nrThreads) nrThreads = 1; + } +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state_(state.lock()); + state_->left.push(t); + wakeup.notify_one(); +} + +void ThreadPool::process() +{ + printMsg(lvlDebug, format("starting pool of %d threads") % nrThreads); + + std::vector<std::thread> workers; + + for (size_t n = 0; n < nrThreads; n++) + workers.push_back(std::thread([&]() { + bool first = true; + + while (true) { + work_t work; + { + auto state_(state.lock()); + if (state_->exception) return; + if (!first) { + assert(state_->pending); + state_->pending--; + } + first = false; + while (state_->left.empty()) { + if (!state_->pending) { + wakeup.notify_all(); + return; + } + if (state_->exception) return; + state_.wait(wakeup); + } + work = state_->left.front(); + state_->left.pop(); + state_->pending++; + } + + try { + work(); + } catch (std::exception & e) { + auto state_(state.lock()); + if (state_->exception) { + if (!dynamic_cast<Interrupted*>(&e)) + printMsg(lvlError, format("error: %s") % e.what()); + } else { + state_->exception = std::current_exception(); + wakeup.notify_all(); + } + } + } + + })); + + for (auto & thr : workers) + thr.join(); + + { + auto state_(state.lock()); + if (state_->exception) + std::rethrow_exception(state_->exception); + } +} + +} + + diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh new file mode 100644 index 000000000000..77641d88ba4e --- /dev/null +++ b/src/libutil/thread-pool.hh @@ -0,0 +1,52 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include <queue> +#include <functional> +#include <thread> + +namespace nix { + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t nrThreads = 0); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t nrThreads; + + struct State + { + std::queue<work_t> left; + size_t pending = 0; + std::exception_ptr exception; + }; + + Sync<State> state; + + std::condition_variable wakeup; + +}; + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 25246a3e89a9..55d490992108 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -548,7 +548,7 @@ void writeToStderr(const string & s) } -void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; +std::function<void(const unsigned char * buf, size_t count)> _writeToStderr; void readFull(int fd, unsigned char * buf, size_t count) @@ -1062,13 +1062,15 @@ void restoreSIGPIPE() volatile sig_atomic_t _isInterrupted = 0; +thread_local bool interruptThrown = false; + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!std::uncaught_exception()) { - _isInterrupted = 0; + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; throw Interrupted("interrupted by the user"); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3606f6ec9eb2..20bd62a0e752 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -167,7 +167,7 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs); void writeToStderr(const string & s); -extern void (*_writeToStderr) (const unsigned char * buf, size_t count); +extern std::function<void(const unsigned char * buf, size_t count)> _writeToStderr; /* Wrappers arount read()/write() that read/write exactly the @@ -316,6 +316,8 @@ void restoreSIGPIPE(); extern volatile sig_atomic_t _isInterrupted; +extern thread_local bool interruptThrown; + void _interrupted(); void inline checkInterrupt() diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 5189c9b4c179..c3cdb8395093 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -493,23 +493,6 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe break; } - case wopQueryFailedPaths: { - startWork(); - PathSet paths = store->queryFailedPaths(); - stopWork(); - to << paths; - break; - } - - case wopClearFailedPaths: { - PathSet paths = readStrings<PathSet>(from); - startWork(); - store->clearFailedPaths(paths); - stopWork(); - to << 1; - break; - } - case wopQueryPathInfo: { Path path = readStorePath(from); startWork(); @@ -517,6 +500,10 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe stopWork(); to << info.deriver << printHash(info.narHash) << info.references << info.registrationTime << info.narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info.ultimate + << info.sigs; + } break; } @@ -539,6 +526,18 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe break; } + case wopAddSignatures: { + Path path = readStorePath(from); + StringSet sigs = readStrings<StringSet>(from); + startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + stopWork(); + to << 1; + break; + } + default: throw Error(format("invalid operation %1%") % op); } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 4e0e28c1158c..e378d8196258 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -63,8 +63,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, if (drvPath != "") mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); - // Copy each output. - DrvInfo::Outputs outputs = i.queryOutputs(); + // Copy each output meant for installation. + DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 563f0805e08b..179015b52bfe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -784,7 +784,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) printMsg(lvlTalkative, format("checking path ‘%1%’...") % path); ValidPathInfo info = store->queryPathInfo(path); HashSink sink(info.narHash.type); - store->dumpPath(path, sink); + store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info.narHash) { printMsg(lvlError, @@ -821,24 +821,6 @@ static void opOptimise(Strings opFlags, Strings opArgs) store->optimiseStore(); } -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); - PathSet failed = store->queryFailedPaths(); - for (auto & i : failed) - cout << format("%1%\n") % i; -} - - -static void opClearFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); -} - - /* Serve the nix store in a way usable by a restricted ssh user. */ static void opServe(Strings opFlags, Strings opArgs) { @@ -1102,10 +1084,6 @@ int main(int argc, char * * argv) op = opRepairPath; else if (*arg == "--optimise" || *arg == "--optimize") op = opOptimise; - else if (*arg == "--query-failed-paths") - op = opQueryFailedPaths; - else if (*arg == "--clear-failed-paths") - op = opClearFailedPaths; else if (*arg == "--serve") op = opServe; else if (*arg == "--generate-binary-cache-key") diff --git a/src/nix/command.cc b/src/nix/command.cc index 0c2103392f8e..a89246a937c1 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -69,4 +69,25 @@ void StoreCommand::run() run(openStoreAt(storeUri)); } +StorePathsCommand::StorePathsCommand() +{ + expectArgs("paths", &storePaths); + mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); +} + +void StorePathsCommand::run(ref<Store> store) +{ + for (auto & storePath : storePaths) + storePath = followLinksToStorePath(storePath); + + if (recursive) { + PathSet closure; + for (auto & storePath : storePaths) + store->computeFSClosure(storePath, closure, false, false); + storePaths = store->topoSortPaths(closure); + } + + run(store, storePaths); +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index 5bf391d9367f..8397244ca177 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -24,6 +24,23 @@ struct StoreCommand : virtual Command virtual void run(ref<Store>) = 0; }; +/* A command that operates on zero or more store paths. */ +struct StorePathsCommand : public StoreCommand +{ +private: + + Paths storePaths; + bool recursive = false; + +public: + + StorePathsCommand(); + + virtual void run(ref<Store> store, Paths storePaths) = 0; + + void run(ref<Store> store) override; +}; + typedef std::map<std::string, ref<Command>> Commands; /* An argument parser that supports multiple subcommands, diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc new file mode 100644 index 000000000000..ed7b578e2f49 --- /dev/null +++ b/src/nix/progress-bar.cc @@ -0,0 +1,72 @@ +#include "progress-bar.hh" + +#include <iostream> + +namespace nix { + +ProgressBar::ProgressBar() +{ + _writeToStderr = [&](const unsigned char * buf, size_t count) { + auto state_(state.lock()); + assert(!state_->done); + std::cerr << "\r\e[K" << std::string((const char *) buf, count); + render(*state_); + }; +} + +ProgressBar::~ProgressBar() +{ + done(); +} + +void ProgressBar::updateStatus(const std::string & s) +{ + auto state_(state.lock()); + assert(!state_->done); + state_->status = s; + render(*state_); +} + +void ProgressBar::done() +{ + auto state_(state.lock()); + assert(state_->activities.empty()); + state_->done = true; + std::cerr << "\r\e[K"; + std::cerr.flush(); + _writeToStderr = decltype(_writeToStderr)(); +} + +void ProgressBar::render(State & state_) +{ + std::cerr << '\r' << state_.status; + if (!state_.activities.empty()) { + if (!state_.status.empty()) std::cerr << ' '; + std::cerr << *state_.activities.rbegin(); + } + std::cerr << "\e[K"; + std::cerr.flush(); +} + + +ProgressBar::Activity ProgressBar::startActivity(const FormatOrString & fs) +{ + return Activity(*this, fs); +} + +ProgressBar::Activity::Activity(ProgressBar & pb, const FormatOrString & fs) + : pb(pb) +{ + auto state_(pb.state.lock()); + state_->activities.push_back(fs.s); + it = state_->activities.end(); --it; + pb.render(*state_); +} + +ProgressBar::Activity::~Activity() +{ + auto state_(pb.state.lock()); + state_->activities.erase(it); +} + +} diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh new file mode 100644 index 000000000000..2dda24346c90 --- /dev/null +++ b/src/nix/progress-bar.hh @@ -0,0 +1,49 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +namespace nix { + +class ProgressBar +{ +private: + struct State + { + std::string status; + bool done = false; + std::list<std::string> activities; + }; + + Sync<State> state; + +public: + + ProgressBar(); + + ~ProgressBar(); + + void updateStatus(const std::string & s); + + void done(); + + class Activity + { + friend class ProgressBar; + private: + ProgressBar & pb; + std::list<std::string>::iterator it; + Activity(ProgressBar & pb, const FormatOrString & fs); + public: + ~Activity(); + }; + + Activity startActivity(const FormatOrString & fs); + +private: + + void render(State & state_); + +}; + +} diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc new file mode 100644 index 000000000000..bcc46c3e7d4f --- /dev/null +++ b/src/nix/sigs.cc @@ -0,0 +1,181 @@ +#include "affinity.hh" // FIXME +#include "command.hh" +#include "progress-bar.hh" +#include "shared.hh" +#include "store-api.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdCopySigs : StorePathsCommand +{ + Strings substituterUris; + + CmdCopySigs() + { + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + } + + std::string name() override + { + return "copy-sigs"; + } + + std::string description() override + { + return "copy path signatures from substituters (like binary caches)"; + } + + void run(ref<Store> store, Paths storePaths) override + { + restoreAffinity(); // FIXME + + if (substituterUris.empty()) + throw UsageError("you must specify at least one substituter using ‘-s’"); + + // FIXME: factor out commonality with MixVerify. + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + ProgressBar progressBar; + + ThreadPool pool; + + std::atomic<size_t> done{0}; + std::atomic<size_t> added{0}; + + auto showProgress = [&]() { + return (format("[%d/%d done]") % done % storePaths.size()).str(); + }; + + progressBar.updateStatus(showProgress()); + + auto doPath = [&](const Path & storePath) { + auto activity(progressBar.startActivity(format("getting signatures for ‘%s’") % storePath)); + + checkInterrupt(); + + auto info = store->queryPathInfo(storePath); + + StringSet newSigs; + + for (auto & store2 : substituters) { + if (!store2->isValidPath(storePath)) continue; + auto info2 = store2->queryPathInfo(storePath); + + /* Don't import signatures that don't match this + binary. */ + if (info.narHash != info2.narHash || + info.narSize != info2.narSize || + info.references != info2.references) + continue; + + for (auto & sig : info2.sigs) + if (!info.sigs.count(sig)) + newSigs.insert(sig); + } + + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } + + done++; + progressBar.updateStatus(showProgress()); + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + progressBar.done(); + + printMsg(lvlInfo, format("imported %d signatures") % added); + } +}; + +static RegisterCommand r1(make_ref<CmdCopySigs>()); + +struct CmdQueryPathSigs : StorePathsCommand +{ + CmdQueryPathSigs() + { + } + + std::string name() override + { + return "query-path-sigs"; + } + + std::string description() override + { + return "print store path signatures"; + } + + void run(ref<Store> store, Paths storePaths) override + { + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + std::cout << storePath << " "; + if (info.ultimate) std::cout << "ultimate "; + for (auto & sig : info.sigs) + std::cout << sig << " "; + std::cout << "\n"; + } + } +}; + +static RegisterCommand r2(make_ref<CmdQueryPathSigs>()); + +struct CmdSignPaths : StorePathsCommand +{ + Path secretKeyFile; + + CmdSignPaths() + { + mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile); + } + + std::string name() override + { + return "sign-paths"; + } + + std::string description() override + { + return "sign the specified paths"; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (secretKeyFile.empty()) + throw UsageError("you must specify a secret key file using ‘-k’"); + + SecretKey secretKey(readFile(secretKeyFile)); + + size_t added{0}; + + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + + auto info2(info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); + + if (!info.sigs.count(*info2.sigs.begin())) { + store->addSignatures(storePath, info2.sigs); + added++; + } + } + + printMsg(lvlInfo, format("added %d signatures") % added); + } +}; + +static RegisterCommand r3(make_ref<CmdSignPaths>()); diff --git a/src/nix/verify.cc b/src/nix/verify.cc new file mode 100644 index 000000000000..9214d3b651d1 --- /dev/null +++ b/src/nix/verify.cc @@ -0,0 +1,211 @@ +#include "affinity.hh" // FIXME +#include "command.hh" +#include "progress-bar.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct MixVerify : virtual Args +{ + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded; + + MixVerify() + { + mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + } + + void verifyPaths(ref<Store> store, const Paths & storePaths) + { + restoreAffinity(); // FIXME + + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + auto publicKeys = getDefaultPublicKeys(); + + std::atomic<size_t> untrusted{0}; + std::atomic<size_t> corrupted{0}; + std::atomic<size_t> done{0}; + std::atomic<size_t> failed{0}; + + ProgressBar progressBar; + + auto showProgress = [&](bool final) { + std::string s; + if (final) + s = (format("checked %d paths") % storePaths.size()).str(); + else + s = (format("[%d/%d checked") % done % storePaths.size()).str(); + if (corrupted > 0) + s += (format(", %d corrupted") % corrupted).str(); + if (untrusted > 0) + s += (format(", %d untrusted") % untrusted).str(); + if (failed > 0) + s += (format(", %d failed") % failed).str(); + if (!final) s += "]"; + return s; + }; + + progressBar.updateStatus(showProgress(false)); + + ThreadPool pool; + + auto doPath = [&](const Path & storePath) { + try { + checkInterrupt(); + + auto activity(progressBar.startActivity(format("checking ‘%s’") % storePath)); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + + HashSink sink(info.narHash.type); + store->narFromPath(storePath, sink); + + auto hash = sink.finish(); + + if (hash.first != info.narHash) { + corrupted = 1; + printMsg(lvlError, + format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") + % storePath % printHash(info.narHash) % printHash(hash.first)); + } + + } + + if (!noTrust) { + + bool good = false; + + if (info.ultimate && !sigsNeeded) + good = true; + + else { + + StringSet sigsSeen; + size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t validSigs = 0; + + auto doSigs = [&](StringSet sigs) { + for (auto sig : sigs) { + if (sigsSeen.count(sig)) continue; + sigsSeen.insert(sig); + if (info.checkSignature(publicKeys, sig)) + validSigs++; + } + }; + + doSigs(info.sigs); + + for (auto & store2 : substituters) { + if (validSigs >= actualSigsNeeded) break; + try { + if (!store2->isValidPath(storePath)) continue; + doSigs(store2->queryPathInfo(storePath).sigs); + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + } + } + + if (validSigs >= actualSigsNeeded) + good = true; + } + + if (!good) { + untrusted++; + printMsg(lvlError, format("path ‘%s’ is untrusted") % storePath); + } + + } + + done++; + + progressBar.updateStatus(showProgress(false)); + + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + failed++; + } + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + progressBar.done(); + + printMsg(lvlInfo, showProgress(true)); + + throw Exit( + (corrupted ? 1 : 0) | + (untrusted ? 2 : 0) | + (failed ? 4 : 0)); + } +}; + +struct CmdVerifyPaths : StorePathsCommand, MixVerify +{ + CmdVerifyPaths() + { + } + + std::string name() override + { + return "verify-paths"; + } + + std::string description() override + { + return "verify the integrity of store paths"; + } + + void run(ref<Store> store, Paths storePaths) override + { + verifyPaths(store, storePaths); + } +}; + +static RegisterCommand r1(make_ref<CmdVerifyPaths>()); + +struct CmdVerifyStore : StoreCommand, MixVerify +{ + CmdVerifyStore() + { + } + + std::string name() override + { + return "verify-store"; + } + + std::string description() override + { + return "verify the integrity of all paths in the Nix store"; + } + + void run(ref<Store> store) override + { + // FIXME: use store->verifyStore()? + + PathSet validPaths = store->queryAllValidPaths(); + + verifyPaths(store, Paths(validPaths.begin(), validPaths.end())); + } +}; + +static RegisterCommand r2(make_ref<CmdVerifyStore>()); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index c72d2defa5d0..5f88c595fdfb 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,7 +1,6 @@ source common.sh clearStore -clearManifests clearCache # Create the binary cache. diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix deleted file mode 100644 index 8ed474d1f27f..000000000000 --- a/tests/binary-patching.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ version }: - -with import ./config.nix; - -mkDerivation { - name = "foo-${toString version}"; - builder = builtins.toFile "builder.sh" - '' - mkdir $out - (for ((n = 1; n < 100000; n++)); do echo $n; done) > $out/foo - ${if version != 1 then '' - (for ((n = 100000; n < 110000; n++)); do echo $n; done) >> $out/foo - '' else ""} - ${if version == 3 then '' - echo foobar >> $out/foo - '' else ""} - ''; -} diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh deleted file mode 100644 index 188be109a0b5..000000000000 --- a/tests/binary-patching.sh +++ /dev/null @@ -1,61 +0,0 @@ -source common.sh - -clearManifests - -mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches - -RESULT=$TEST_ROOT/result - -# Build version 1 and 2 of the "foo" package. -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \ - $(nix-build -o $RESULT binary-patching.nix --arg version 1) -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1 - -out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2) -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2 -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2 - -out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3) -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3 -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3 - -rm $RESULT - -# Generate binary patches. -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2 - -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3 - -grep -q "patch {" $TEST_ROOT/manifest3 - -# Get rid of versions 2 and 3. -nix-store --delete $out2 $out3 - -# Pull the manifest containing the patches. -clearManifests -nix-pull file://$TEST_ROOT/manifest3 - -# Make sure that the download size prediction uses the patches rather -# than the full download. -nix-build -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB" - -# Now rebuild it. This should use the two patches generated above. -rm -f $TEST_ROOT/var/log/nix/downloads -nix-build -o $RESULT binary-patching.nix --arg version 3 -rm $RESULT -[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ] - -# Add a patch from version 1 directly to version 3. -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3 - -# Rebuild version 3. This should use the direct patch rather than the -# sequence of two patches. -nix-store --delete $out2 $out3 -clearManifests -rm $TEST_ROOT/var/log/nix/downloads -nix-pull file://$TEST_ROOT/manifest3 -nix-build -o $RESULT binary-patching.nix --arg version 3 -[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ] diff --git a/tests/common.sh.in b/tests/common.sh.in index eb9798a27b45..9e8962f1a60f 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -2,7 +2,7 @@ set -e datadir="@datadir@" -export TEST_ROOT=$(pwd)/test-tmp +export TEST_ROOT=${TMPDIR:-/tmp}/nix-test export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. @@ -54,10 +54,6 @@ clearStore() { rm -f "$NIX_STATE_DIR"/gcroots/ref } -clearManifests() { - rm -f $NIX_STATE_DIR/manifests/* -} - clearCache() { rm -rf "$cacheDir" } diff --git a/tests/filter-source.nix b/tests/filter-source.nix index a620f0fda5c4..9071636394af 100644 --- a/tests/filter-source.nix +++ b/tests/filter-source.nix @@ -8,5 +8,5 @@ mkDerivation { type != "symlink" && baseNameOf path != "foo" && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); - in builtins.filterSource filter ./test-tmp/filterin; + in builtins.filterSource filter ((builtins.getEnv "TEST_ROOT") + "/filterin"); } diff --git a/tests/install-package.sh b/tests/install-package.sh index 653dfee4c8d1..1916f72713e2 100644 --- a/tests/install-package.sh +++ b/tests/install-package.sh @@ -1,15 +1,14 @@ source common.sh -# Note: this test expects to be run *after* nix-push.sh. - drvPath=$(nix-instantiate ./dependencies.nix) -outPath=$(nix-store -q $drvPath) +outPath=$(nix-store -r $drvPath) +nix-push --dest $cacheDir $outPath clearStore clearProfiles cat > $TEST_ROOT/foo.nixpkg <<EOF -NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath +NIXPKG1 - simple $system $drvPath $outPath file://$cacheDir EOF nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg diff --git a/tests/local.mk b/tests/local.mk index 03f53b44c275..471821b270e8 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -4,14 +4,14 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ build-hook.sh substitutes.sh substitutes2.sh \ - fallback.sh nix-push.sh gc.sh gc-concurrent.sh nix-pull.sh \ + fallback.sh nix-push.sh gc.sh gc-concurrent.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh negative-caching.sh \ - binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ + remote-store.sh export.sh export-graph.sh \ + timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ - check-reqs.sh pass-as-file.sh tarball.sh + check-reqs.sh pass-as-file.sh tarball.sh restricted.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/negative-caching.nix b/tests/negative-caching.nix deleted file mode 100644 index 10df67a748fc..000000000000 --- a/tests/negative-caching.nix +++ /dev/null @@ -1,21 +0,0 @@ -with import ./config.nix; - -rec { - - fail = mkDerivation { - name = "fail"; - builder = builtins.toFile "builder.sh" "echo FAIL; exit 1"; - }; - - succeed = mkDerivation { - name = "succeed"; - builder = builtins.toFile "builder.sh" "echo SUCCEED; mkdir $out"; - }; - - depOnFail = mkDerivation { - name = "dep-on-fail"; - builder = builtins.toFile "builder.sh" "echo URGH; mkdir $out"; - inputs = [fail succeed]; - }; - -} diff --git a/tests/negative-caching.sh b/tests/negative-caching.sh deleted file mode 100644 index 4217bc38e121..000000000000 --- a/tests/negative-caching.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore - -set +e - -opts="--option build-cache-failure true --print-build-trace" - -# This build should fail, and the failure should be cached. -log=$(nix-build $opts negative-caching.nix -A fail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "@ build-failed" || fail "no build-failed trace" - -# Do it again. The build shouldn't be tried again. -log=$(nix-build $opts negative-caching.nix -A fail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "FAIL" && fail "failed build not cached" -echo "$log" | grep -q "@ build-failed .* cached" || fail "trace doesn't say cached" - -# Check that --keep-going works properly with cached failures. -log=$(nix-build $opts --keep-going negative-caching.nix -A depOnFail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "FAIL" && fail "failed build not cached (2)" -echo "$log" | grep -q "@ build-failed .* cached" || fail "trace doesn't say cached (2)" -echo "$log" | grep -q "@ build-succeeded .*-succeed" || fail "didn't keep going" diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index b3442f6a8471..c538afd606be 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -1,7 +1,6 @@ source common.sh clearProfiles -clearManifests rm -f $TEST_ROOT/.nix-channels @@ -45,7 +44,6 @@ nix-env -i dependencies clearProfiles -clearManifests rm -f $TEST_ROOT/.nix-channels # Test updating from a tarball diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 1418c65897d3..0e42cc0a3d72 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -4,7 +4,7 @@ with import <nixpkgs/nixos/lib/testing.nix> { inherit system; }; -makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; pkgC = pkgs.hello; in { +makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { nodes = { client = diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 3586a7efc3c8..db1edd73eef0 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -1,10 +1,11 @@ source common.sh home=$TEST_ROOT/home +user=$(whoami) rm -rf $home mkdir -p $home -HOME=$home $SHELL -e -c ". ../scripts/nix-profile.sh" -HOME=$home $SHELL -e -c ". ../scripts/nix-profile.sh" # test idempotency +HOME=$home USER=$user $SHELL -e -c ". ../scripts/nix-profile.sh" +HOME=$home USER=$user $SHELL -e -c ". ../scripts/nix-profile.sh" # test idempotency [ -L $home/.nix-profile ] [ -e $home/.nix-channels ] diff --git a/tests/nix-pull.sh b/tests/nix-pull.sh deleted file mode 100644 index 87239948c481..000000000000 --- a/tests/nix-pull.sh +++ /dev/null @@ -1,33 +0,0 @@ -source common.sh - -pullCache () { - echo "pulling cache..." - nix-pull file://$TEST_ROOT/cache/MANIFEST -} - -clearStore -clearManifests -pullCache - -drvPath=$(nix-instantiate dependencies.nix) -outPath=$(nix-store -q $drvPath) - -echo "building $outPath using substitutes..." -nix-store -r $outPath - -cat $outPath/input-2/bar - -clearStore -clearManifests -pullCache - -echo "building $drvPath using substitutes..." -nix-store -r $drvPath - -cat $outPath/input-2/bar - -# Check that the derivers are set properly. -test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver $(readLink $outPath/input-2) | grep -q -- "-input-2.drv" - -clearManifests diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 8312424f0ac6..b3908717a40e 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -1,7 +1,6 @@ source common.sh clearStore -clearManifests startDaemon diff --git a/tests/restricted.sh b/tests/restricted.sh new file mode 100644 index 000000000000..19096a9f8dd2 --- /dev/null +++ b/tests/restricted.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore + +nix-instantiate --option restrict-eval true --eval -E '1 + 2' +(! nix-instantiate --option restrict-eval true ./simple.nix) +nix-instantiate --option restrict-eval true ./simple.nix -I src=. +nix-instantiate --option restrict-eval true ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh + +(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix') +nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix' -I src=.. + +(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost') +nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost' -I src=../src + +(! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') +nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. + diff --git a/tests/secure-drv-outputs.sh b/tests/secure-drv-outputs.sh index 4888123da910..50a9c4428d30 100644 --- a/tests/secure-drv-outputs.sh +++ b/tests/secure-drv-outputs.sh @@ -5,7 +5,6 @@ source common.sh clearStore -clearManifests startDaemon diff --git a/tests/tarball.sh b/tests/tarball.sh index cb5258a9e548..254c4b626b87 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -16,8 +16,14 @@ tarball=$TEST_ROOT/tarball.tar.xz nix-env -f file://$tarball -qa --out-path | grep -q dependencies -nix-build file://$tarball +nix-build -o $TMPDIR/result file://$tarball -nix-build '<foo>' -I foo=file://$tarball +nix-build -o $TMPDIR/result '<foo>' -I foo=file://$tarball -nix-build -E "import (fetchTarball file://$tarball)" +nix-build -o $TMPDIR/result -E "import (fetchTarball file://$tarball)" + +nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz +nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz +(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz) + +nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=. |