diff options
308 files changed, 12417 insertions, 10081 deletions
diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 000000000000..2d1117f4bda6 --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,16 @@ +((c++-mode . ( + (c-file-style . "k&r") + (c-basic-offset . 4) + (indent-tabs-mode . nil) + (tab-width . 4) + (show-trailing-whitespace . t) + (indicate-empty-lines . t) + (eval . (c-set-offset 'innamespace 0)) + (eval . (c-set-offset 'defun-open 0)) + (eval . (c-set-offset 'inline-open 0)) + (eval . (c-set-offset 'arglist-intro '+)) + (eval . (c-set-offset 'arglist-cont 0)) + (eval . (c-set-offset 'arglist-cont-nonempty '+)) + (eval . (c-set-offset 'substatement-open 0)) + (eval . (c-set-offset 'access-label '-)) + ))) diff --git a/.gitignore b/.gitignore index 94067256b8bc..a175e8dfe291 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-pull /scripts/nix-push /scripts/nix-switch /scripts/nix-collect-garbage @@ -43,11 +42,8 @@ Makefile.config /scripts/nix-channel /scripts/nix-build /scripts/nix-copy-closure -/scripts/nix-generate-patches /scripts/NixConfig.pm /scripts/NixManifest.pm -/scripts/GeneratePatches.pm -/scripts/download-using-manifests.pl /scripts/copy-from-other-stores.pl /scripts/download-from-binary-cache.pl /scripts/find-runtime-roots.pl @@ -55,10 +51,6 @@ Makefile.config /scripts/nix-reduce-build /scripts/nix-http-export.cgi -# /src/bsdiff-4.3/ -/src/bsdiff-4.3/bsdiff -/src/bsdiff-4.3/bspatch - # /src/libexpr/ /src/libexpr/lexer-tab.cc /src/libexpr/lexer-tab.hh @@ -73,15 +65,9 @@ Makefile.config # /src/nix-env/ /src/nix-env/nix-env -# /src/nix-hash/ -/src/nix-hash/nix-hash - # /src/nix-instantiate/ /src/nix-instantiate/nix-instantiate -# /src/nix-log2xml/ -/src/nix-log2xml/nix-log2xml - # /src/nix-store/ /src/nix-store/nix-store diff --git a/Makefile b/Makefile index fe2e88a995aa..8f3688475e62 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,14 @@ makefiles = \ src/libstore/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ - src/nix-hash/local.mk \ + src/nix/local.mk \ src/nix-store/local.mk \ src/nix-instantiate/local.mk \ src/nix-env/local.mk \ src/nix-daemon/local.mk \ src/nix-collect-garbage/local.mk \ src/download-via-ssh/local.mk \ - src/nix-log2xml/local.mk \ - src/bsdiff-4.3/local.mk \ + src/nix-prefetch-url/local.mk \ perl/local.mk \ scripts/local.mk \ corepkgs/local.mk \ @@ -24,7 +23,7 @@ makefiles = \ doc/manual/local.mk \ tests/local.mk -GLOBAL_CXXFLAGS += -std=c++0x -g -Wall +GLOBAL_CXXFLAGS += -std=c++11 -g -Wall -include Makefile.config diff --git a/Makefile.config.in b/Makefile.config.in index 29ccc1b146f1..3a7892e8b6bb 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -3,12 +3,14 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -HAVE_OPENSSL = @HAVE_OPENSSL@ +ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ +LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ +SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ bsddiff_compat_include = @bsddiff_compat_include@ diff --git a/README b/README deleted file mode 100644 index 35a1f6cc08c9..000000000000 --- a/README +++ /dev/null @@ -1,10 +0,0 @@ -Nix is a purely functional package manager. For installation and -usage instructions, please read the manual, which can be found in -`docs/manual/manual.html', and additionally at the Nix website at -<http://nixos.org/>. - - -Acknowledgments - -This product includes software developed by the OpenSSL Project for -use in the OpenSSL Toolkit (http://www.OpenSSL.org/). diff --git a/README.md b/README.md new file mode 100644 index 000000000000..e77ce302f193 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +Nix, the purely functional package manager +------------------------------------------ + +Nix is a new take on package management that is fairly unique. Because of it's +purity aspects, a lot of issues found in traditional package managers don't +appear with Nix. + +To find out more about the tool, usage and installation instructions, please +read the manual, which is available on the Nix website at +<http://nixos.org/nix/manual>. + +## License + +Nix is released under the LGPL v2.1 + +This product includes software developed by the OpenSSL Project for +use in the OpenSSL Toolkit (http://www.OpenSSL.org/). diff --git a/configure.ac b/configure.ac index 1d4b7d3b8995..ff8e39fa05a2 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(README) +AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) AC_PROG_SED @@ -16,12 +16,14 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], machine_name="i686";; amd64) machine_name="x86_64";; + armv6|armv7) + machine_name="${host_cpu}l";; *) machine_name="$host_cpu";; esac case "$host_os" in - linux-gnu*) + linux-gnu*|linux-musl*) # For backward compatibility, strip the `-gnu' part. system="$machine_name-linux";; *) @@ -48,6 +50,7 @@ test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var # Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE if test "$sys_name" = sunos; then # Solaris requires -lsocket -lnsl for network functions LIBS="-lsocket -lnsl $LIBS" @@ -58,6 +61,7 @@ CFLAGS= CXXFLAGS= AC_PROG_CC AC_PROG_CXX +AX_CXX_COMPILE_STDCXX_11 # Use 64-bit file system calls so that we can support files > 2 GiB. @@ -76,18 +80,7 @@ static char buf[1024];]], AC_LANG_POP(C++) -# Check for chroot support (requires chroot() and bind mounts). -AC_CHECK_FUNCS([chroot]) -AC_CHECK_FUNCS([unshare]) AC_CHECK_FUNCS([statvfs]) -AC_CHECK_HEADERS([sched.h]) -AC_CHECK_HEADERS([sys/param.h]) -AC_CHECK_HEADERS([sys/mount.h], [], [], -[#ifdef HAVE_SYS_PARAM_H -# include <sys/param.h> -# endif -]) -AC_CHECK_HEADERS([sys/syscall.h]) # Check for lutimes, optionally used for changing the mtime of @@ -95,10 +88,6 @@ AC_CHECK_HEADERS([sys/syscall.h]) AC_CHECK_FUNCS([lutimes]) -# Check for sched_setaffinity. -AC_CHECK_FUNCS([sched_setaffinity]) - - # Check whether the store optimiser can optimise symlinks. AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) ln -s bla tmp_link @@ -117,15 +106,6 @@ AC_CHECK_HEADERS([locale]) AC_LANG_POP(C++) -# Check for <err.h>. -AC_CHECK_HEADER([err.h], [], [bsddiff_compat_include="-Icompat-include"]) -AC_SUBST([bsddiff_compat_include]) - - -# Check for <linux/fs.h> (for immutable file support). -AC_CHECK_HEADERS([linux/fs.h]) - - AC_DEFUN([NEED_PROG], [ AC_PATH_PROG($1, $2) @@ -183,16 +163,8 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_SUBST(storedir) -# Look for OpenSSL, an optional dependency. -AC_PATH_PROG(openssl, openssl, openssl) # if not found, call openssl in $PATH -AC_SUBST(openssl) -AC_DEFINE_UNQUOTED(OPENSSL_PATH, ["$openssl"], [Path of the OpenSSL binary]) - -PKG_CHECK_MODULES([OPENSSL], [libcrypto], - [AC_DEFINE([HAVE_OPENSSL], [1], [Whether to use OpenSSL.]) - CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS" - have_openssl=1], [have_openssl=]) -AC_SUBST(HAVE_OPENSSL, [$have_openssl]) +# Look for OpenSSL, a required dependency. +PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libbz2, a required dependency. @@ -218,6 +190,19 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], AC_SUBST(HAVE_SODIUM, [$have_sodium]) +# Look for liblzma, a required dependency. +PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) + + +# Look for aws-cpp-sdk-s3. +AC_LANG_PUSH(C++) +AC_CHECK_HEADERS([aws/s3/S3Client.h], + [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-cpp-sdk-s3.]) + enable_s3=1], [enable_s3=]) +AC_SUBST(ENABLE_S3, [$enable_s3]) +AC_LANG_POP(C++) + + # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]), @@ -261,6 +246,23 @@ AC_MSG_RESULT(yes) AC_SUBST(perlFlags) +# Check for otool, an optional dependency on Darwin. +AC_PATH_PROG(otool, otool) +AC_MSG_CHECKING([that otool works]) +case $host_os in + darwin*) + if test -z "$otool" || ! $otool --version 2>/dev/null; then + AC_MSG_RESULT(no) + AC_MSG_ERROR([Can't get version from otool; do you need to install developer tools?]) + fi + AC_MSG_RESULT(yes) + ;; + *) + AC_MSG_RESULT(not needed) + ;; +esac + + # Whether to build the Perl bindings AC_MSG_CHECKING([whether to build the Perl bindings]) AC_ARG_ENABLE(perl-bindings, AC_HELP_STRING([--enable-perl-bindings], diff --git a/corepkgs/buildenv.nix b/corepkgs/buildenv.nix index 5bf7b4e56042..70981a752c3c 100644 --- a/corepkgs/buildenv.nix +++ b/corepkgs/buildenv.nix @@ -8,7 +8,7 @@ derivation { builder = perl; args = [ "-w" ./buildenv.pl ]; - manifest = manifest; + inherit manifest; # !!! grmbl, need structured data for passing this in a clean way. derivations = @@ -23,5 +23,23 @@ derivation { # network traffic, so don't do that. preferLocalBuild = true; + # Also don't bother substituting. + allowSubstitutes = false; + + __sandboxProfile = '' + (allow sysctl-read) + (allow file-read* + (literal "/usr/lib/libSystem.dylib") + (literal "/usr/lib/libSystem.B.dylib") + (literal "/usr/lib/libobjc.A.dylib") + (literal "/usr/lib/libobjc.dylib") + (literal "/usr/lib/libauto.dylib") + (literal "/usr/lib/libc++abi.dylib") + (literal "/usr/lib/libc++.1.dylib") + (literal "/usr/lib/libDiagnosticMessagesClient.dylib") + (subpath "/usr/lib/system") + (subpath "/dev")) + ''; + inherit chrootDeps; } diff --git a/corepkgs/buildenv.pl b/corepkgs/buildenv.pl index 264442104320..dacc53701a01 100644 --- a/corepkgs/buildenv.pl +++ b/corepkgs/buildenv.pl @@ -149,7 +149,7 @@ foreach my $pkg (@pkgs) { # Symlink to the packages that have been "propagated" by packages -# installed by the user (i.e., package X declares that it want Y +# installed by the user (i.e., package X declares that it wants Y # installed as well). We do these later because they have a lower # priority in case of collisions. my $priorityCounter = 1000; # don't care about collisions diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 8918f4ddde8d..90e8edbea833 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -12,7 +12,6 @@ in rec { tar = "@tar@"; tarFlags = "@tarFlags@"; tr = "@tr@"; - curl = "@curl@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index 1ce88593cff2..5e0ad9da3c68 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -1,33 +1,29 @@ with import <nix/config.nix>; -{system ? builtins.currentSystem, url, outputHash ? "", outputHashAlgo ? "", md5 ? "", sha1 ? "", sha256 ? "", executable ? false}: +{ system ? builtins.currentSystem +, url +, outputHash ? "" +, outputHashAlgo ? "" +, md5 ? "", sha1 ? "", sha256 ? "" +, executable ? false +, unpack ? false +, name ? baseNameOf (toString url) +}: assert (outputHash != "" && outputHashAlgo != "") || md5 != "" || sha1 != "" || sha256 != ""; -let - - builder = builtins.toFile "fetchurl.sh" - ('' - echo "downloading $url into $out" - ${curl} --fail --location --max-redirs 20 --insecure "$url" > "$out" - '' + (if executable then "${coreutils}/chmod +x $out" else "")); - -in - derivation { - name = baseNameOf (toString url); - builder = shell; - args = [ "-e" builder ]; + builder = "builtin:fetchurl"; # New-style output content requirements. outputHashAlgo = if outputHashAlgo != "" then outputHashAlgo else if sha256 != "" then "sha256" else if sha1 != "" then "sha1" else "md5"; outputHash = if outputHash != "" then outputHash else if sha256 != "" then sha256 else if sha1 != "" then sha1 else md5; - outputHashMode = if executable then "recursive" else "flat"; + outputHashMode = if unpack || executable then "recursive" else "flat"; - inherit system url; + inherit name system url executable unpack; # No need to double the amount of network traffic preferLocalBuild = true; @@ -39,6 +35,4 @@ derivation { # by definition pure. "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" ]; - - inherit chrootDeps; } diff --git a/corepkgs/imported-drv-to-derivation.nix b/corepkgs/imported-drv-to-derivation.nix index bdb60169860a..eab8b050e8ff 100644 --- a/corepkgs/imported-drv-to-derivation.nix +++ b/corepkgs/imported-drv-to-derivation.nix @@ -1,10 +1,10 @@ -attrs @ { drvPath, outputs, ... }: +attrs @ { drvPath, outputs, name, ... }: let commonAttrs = (builtins.listToAttrs outputsList) // { all = map (x: x.value) outputsList; - inherit drvPath; + inherit drvPath name; type = "derivation"; }; diff --git a/dev-shell b/dev-shell index eae9246f4710..5a090ded6080 100755 --- a/dev-shell +++ b/dev-shell @@ -11,7 +11,7 @@ exec $s release.nix -A tarball --command " export NIX_PATH='$NIX_PATH' export NIX_BUILD_SHELL=$(type -p bash) export c=\$configureFlags - exec $s release.nix -A build.$(if [ $(uname -s) = Darwin ]; then echo x86_64-darwin; else echo x86_64-linux; fi) --exclude tarball --command ' + exec $s release.nix -A build.$(if [ $(uname -s) = Darwin ]; then echo x86_64-darwin; elif [[ $(uname -m) =~ ^i[3456]86$ ]]; then echo i686-linux; else echo x86_64-linux; fi) --exclude tarball --command ' configureFlags+=\" \$c --prefix=$(pwd)/inst --sysconfdir=$(pwd)/inst/etc\" return '" \ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c947d19fa0e1..4c8f3d9d3809 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -224,16 +224,16 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>build-use-chroot</literal></term> + <varlistentry><term><literal>build-use-sandbox</literal></term> <listitem><para>If set to <literal>true</literal>, builds will be - performed in a <emphasis>chroot environment</emphasis>, i.e., + performed in a <emphasis>sandboxed environment</emphasis>, i.e., they’re isolated from the normal file system hierarchy and will only see their dependencies in the Nix store, the temporary build directory, private versions of <filename>/proc</filename>, <filename>/dev</filename>, <filename>/dev/shm</filename> and - <filename>/dev/pts</filename>, and the paths configured with the - <link linkend='conf-build-chroot-dirs'><literal>build-chroot-dirs</literal> + <filename>/dev/pts</filename> (on Linux), and the paths configured with the + <link linkend='conf-build-sandbox-paths'><literal>build-sandbox-paths</literal> option</link>. This is useful to prevent undeclared dependencies on files in directories such as <filename>/usr/bin</filename>. In addition, on Linux, builds run in private PID, mount, network, IPC @@ -241,8 +241,8 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> system (except that fixed-output derivations do not run in private network namespace to ensure they can access the network).</para> - <para>Currently, chroots only work on Linux and Mac OS X. The use - of a chroot requires that Nix is run as root (so you should use + <para>Currently, sandboxing only work on Linux and Mac OS X. The use + of a sandbox requires that Nix is run as root (so you should use the <link linkend='conf-build-users-group'>“build users” feature</link> to perform the actual builds under different users than root).</para> @@ -250,7 +250,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> <para>If this option is set to <literal>relaxed</literal>, then fixed-output derivations and derivations that have the <varname>__noChroot</varname> attribute set to - <literal>true</literal> do not run in chroots.</para> + <literal>true</literal> do not run in sandboxes.</para> <para>The default is <literal>false</literal>.</para> @@ -259,17 +259,16 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-build-chroot-dirs"><term><literal>build-chroot-dirs</literal></term> + <varlistentry xml:id="conf-build-sandbox-paths"> + <term><literal>build-sandbox-paths</literal></term> - <listitem><para>A list of paths bind-mounted into Nix chroot - environments. Contrary to what the name suggests, the specified - paths do not have to be directories; you can bind-mount other - types of files as well. You can use the syntax + <listitem><para>A list of paths bind-mounted into Nix sandbox + environments. You can use the syntax <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal> - to mount a path in a different location in the chroot; for + to mount a path in a different location in the sandbox; for instance, <literal>/bin=/nix-bin</literal> will mount the path <literal>/nix-bin</literal> as <literal>/bin</literal> inside the - chroot.</para> + sandbox.</para> <para>Depending on how Nix was built, the default value for this option may be empty or provide <filename>/bin/sh</filename> as a @@ -278,10 +277,11 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry xml:id="conf-build-extra-chroot-dirs"><term><literal>build-extra-chroot-dirs</literal></term> + <varlistentry xml:id="conf-build-extra-sandbox-paths"> + <term><literal>build-extra-sandbox-paths</literal></term> <listitem><para>A list of additional paths appended to - <option>build-chroot-dirs</option>. Useful if you want to extend + <option>build-sandbox-paths</option>. Useful if you want to extend its default value.</para></listitem> </varlistentry> @@ -306,21 +306,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>build-cache-failure</literal></term> - - <listitem><para>If set to <literal>true</literal>, Nix will - “cache” build failures, meaning that it will remember (in its - database) that a derivation previously failed. If you then try to - build the derivation again, Nix will immediately fail rather than - perform the build again. Failures in fixed-output derivations - (such as <function>fetchurl</function> calls) are never cached. - The “failed” status of a derivation can be cleared using - <command>nix-store --clear-failed-paths</command>. By default, - failure caching is disabled.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>build-keep-log</literal></term> <listitem><para>If set to <literal>true</literal> (the default), @@ -426,7 +411,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> <listitem><para>The maximum number of parallel HTTP connections used by the binary cache substituter to get NAR info files. This number should be high to minimise latency. It defaults to - 150.</para></listitem> + 25.</para></listitem> </varlistentry> @@ -440,18 +425,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>force-manifest</literal></term> - - <listitem><para>If this option is set to <literal>false</literal> - (default) and a Nix channel provides both a manifest and a binary - cache, only the binary cache will be used. If set to - <literal>true</literal>, the manifest will be fetched as well. - This is useful if you want to use binary patches (which are - currently not supported by binary caches).</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>system</literal></term> <listitem><para>This option specifies the canonical Nix system @@ -593,19 +566,21 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> between different versions of the same system to be hard-coded into nix. </para> - <para>The hook is passed the derivation path and, if chroots are enabled, - the chroot directory. It can then modify the chroot and send a series of + <para>The hook is passed the derivation path and, if sandboxes are enabled, + the sandbox directory. It can then modify the sandbox and send a series of commands to modify various settings to stdout. The currently recognized commands are:</para> <variablelist> - <varlistentry xml:id="extra-chroot-dirs"><term><literal>extra-chroot-dirs</literal></term> + <varlistentry xml:id="extra-sandbox-paths"> + <term><literal>extra-sandbox-paths</literal></term> <listitem> <para>Pass a list of files and directories to be included in the - chroot for this build. One entry per line, terminated by an empty - line. Entries have the same format as build-chroot-dirs.</para> + sandbox for this build. One entry per line, terminated by an empty + line. Entries have the same format as + <literal>build-sandbox-paths</literal>.</para> </listitem> @@ -616,6 +591,18 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> + <varlistentry xml:id="conf-build-repeat"><term><literal>build-repeat</literal></term> + + <listitem><para>How many times to repeat builds to check whether + they are deterministic. The default value is 0. If the value is + non-zero, every build is repeated the specified number of + times. If the contents of any of the runs differs from the + previous ones, the build is rejected and the resulting store paths + are not registered as “valid” in Nix’s database.</para></listitem> + + </varlistentry> + + </variablelist> </para> diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index cb1ecfee123e..27efef945f15 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -11,6 +11,12 @@ <variablelist xml:id="env-common"> +<varlistentry><term><envar>IN_NIX_SHELL</envar></term> + + <listitem><para>Indicator that tells if the current environment was set up by + <command>nix-shell</command>.</para></listitem> + +</varlistentry> <varlistentry xml:id="env-NIX_PATH"><term><envar>NIX_PATH</envar></term> diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index a6f4a27203ac..0a1f2a8b722d 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -73,11 +73,10 @@ condition="manual">See also <xref linkend="sec-channels" <listitem><para>Downloads the Nix expressions of all subscribed channels (or only those included in - <replaceable>names</replaceable> if specified), makes them the + <replaceable>names</replaceable> if specified) and makes them the default for <command>nix-env</command> operations (by symlinking - them from the directory <filename>~/.nix-defexpr</filename>), and - performs a <command>nix-pull</command> on the manifests of all - channels to make pre-built binaries available.</para></listitem> + them from the directory + <filename>~/.nix-defexpr</filename>).</para></listitem> </varlistentry> @@ -187,16 +186,6 @@ following files:</para> </varlistentry> - <varlistentry><term><filename>MANIFEST.bz2</filename></term> - - <listitem><para>(Deprecated in favour of binary caches.) A - manifest as created by <command>nix-push</command>. Only used if - <filename>binary-cache-url</filename> is not present or if the - <filename>nix.conf</filename> option - <option>force-manifest</option> is set.</para></listitem> - - </varlistentry> - </variablelist> </refsection> diff --git a/doc/manual/command-ref/nix-collect-garbage.xml b/doc/manual/command-ref/nix-collect-garbage.xml index c88851299152..35a78c5b2015 100644 --- a/doc/manual/command-ref/nix-collect-garbage.xml +++ b/doc/manual/command-ref/nix-collect-garbage.xml @@ -28,6 +28,7 @@ <arg choice='plain'><option>--print-dead</option></arg> <arg choice='plain'><option>--delete</option></arg> </group> + <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg> <arg><option>--dry-run</option></arg> </cmdsynopsis> </refsynopsisdiv> diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml index 6168f859dffc..97e261ae993d 100644 --- a/doc/manual/command-ref/nix-copy-closure.xml +++ b/doc/manual/command-ref/nix-copy-closure.xml @@ -22,7 +22,6 @@ <arg choice='plain'><option>--to</option></arg> <arg choice='plain'><option>--from</option></arg> </group> - <arg><option>--sign</option></arg> <arg><option>--gzip</option></arg> <!-- <arg><option>- -show-progress</option></arg> @@ -30,6 +29,7 @@ <arg><option>--include-outputs</option></arg> <arg><option>--use-substitutes</option></arg> <arg><option>-s</option></arg> + <arg><option>-v</option></arg> <arg choice='plain'> <replaceable>user@</replaceable><replaceable>machine</replaceable> </arg> @@ -42,7 +42,7 @@ <para><command>nix-copy-closure</command> gives you an easy and efficient way to exchange software between machines. Given one or -more Nix store paths <replaceable>paths</replaceable> on the local +more Nix store <replaceable>paths</replaceable> on the local machine, <command>nix-copy-closure</command> computes the closure of those paths (i.e. all their dependencies in the Nix store), and copies all paths in the closure to the remote machine via the @@ -86,23 +86,6 @@ those paths. If this bothers you, use </varlistentry> - <varlistentry><term><option>--sign</option></term> - - <listitem><para>Let the sending machine cryptographically sign the - dump of each path with the key in - <filename><replaceable>sysconfdir</replaceable>/nix/signing-key.sec</filename>. - If the user on the target machine does not have direct access to - the Nix store (i.e., if the target machine has a multi-user Nix - installation), then the target machine will check the dump against - <filename><replaceable>sysconfdir</replaceable>/nix/signing-key.pub</filename> - before unpacking it in its Nix store. This allows secure sharing - of store paths between untrusted users on two machines, provided - that there is a trust relation between the Nix installations on - both machines (namely, they have matching public/secret - keys).</para></listitem> - - </varlistentry> - <varlistentry><term><option>--gzip</option></term> <listitem><para>Enable compression of the SSH @@ -138,6 +121,12 @@ those paths. If this bothers you, use </varlistentry> + <varlistentry><term><option>-v</option></term> + + <listitem><para>Show verbose output.</para></listitem> + + </varlistentry> + </variablelist> </refsection> diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 5e40317f189d..2ed4a5d9f666 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -367,6 +367,10 @@ number of possible ways: linkend="rsec-nix-store-realise">realised</link> and installed.</para></listitem> + <listitem><para>By default all outputs are installed for each derivation. + That can be reduced by setting <literal>meta.outputsToInstall</literal>. + </para></listitem> <!-- TODO: link nixpkgs docs on the ability to override those. --> + </itemizedlist> </para> @@ -378,7 +382,7 @@ number of possible ways: <variablelist> - <varlistentry><term><option>--prebuild-only</option> / <option>-b</option></term> + <varlistentry><term><option>--prebuilt-only</option> / <option>-b</option></term> <listitem><para>Use only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can @@ -1012,7 +1016,7 @@ user environment elements, etc. --> </varlistentry> - <varlistentry><term><option>--prebuild-only</option> / <option>-b</option></term> + <varlistentry><term><option>--prebuilt-only</option> / <option>-b</option></term> <listitem><para>Show only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can diff --git a/doc/manual/command-ref/nix-generate-patches.xml b/doc/manual/command-ref/nix-generate-patches.xml deleted file mode 100644 index 70bec432d28e..000000000000 --- a/doc/manual/command-ref/nix-generate-patches.xml +++ /dev/null @@ -1,44 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-generate-patches"> - -<refmeta> - <refentrytitle>nix-generate-patches</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-generate-patches</refname> - <refpurpose>generates binary patches between NAR files</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-generate-patches</command> - <arg choice='plain'><replaceable>NAR-DIR</replaceable></arg> - <arg choice='plain'><replaceable>PATCH-DIR</replaceable></arg> - <arg choice='plain'><replaceable>PATCH-URI</replaceable></arg> - <arg choice='plain'><replaceable>OLD-MANIFEST</replaceable></arg> - <arg choice='plain'><replaceable>NEW-MANIFEST</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<para>The command <command>nix-generate-patches</command> generates -binary patches between NAR files listed in OLD-MANIFEST and NEW-MANIFEST. -The patches are written to the directory PATCH-DIR, and the prefix -PATCH-URI is used to generate URIs for the patches. The patches are -added to NEW-MANIFEST. All NARs are required to exist in NAR-DIR. -Patches are generated between succeeding versions of packages with -the same name.</para> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-install-package.xml b/doc/manual/command-ref/nix-install-package.xml index f7802a95d55e..e17166caaaf3 100644 --- a/doc/manual/command-ref/nix-install-package.xml +++ b/doc/manual/command-ref/nix-install-package.xml @@ -146,9 +146,7 @@ The elements are as follows: <varlistentry><term><replaceable>manifestURL</replaceable></term> - <listitem><para>The manifest to be pulled by - <command>nix-pull</command>. The manifest must contain - <replaceable>outPath</replaceable>.</para></listitem> + <listitem><para>Obsolete.</para></listitem> </varlistentry> diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml index 5d1ab6931cd3..016d8863a94c 100644 --- a/doc/manual/command-ref/nix-prefetch-url.xml +++ b/doc/manual/command-ref/nix-prefetch-url.xml @@ -3,7 +3,7 @@ xmlns:xi="http://www.w3.org/2001/XInclude" version="5.0" xml:id="sec-nix-prefetch-url"> - + <refmeta> <refentrytitle>nix-prefetch-url</refentrytitle> <manvolnum>1</manvolnum> @@ -20,6 +20,7 @@ <cmdsynopsis> <command>nix-prefetch-url</command> <arg><option>--type</option> <replaceable>hashAlgo</replaceable></arg> + <arg><option>--print-path</option></arg> <arg choice='plain'><replaceable>url</replaceable></arg> <arg><replaceable>hash</replaceable></arg> </cmdsynopsis> @@ -54,8 +55,8 @@ error if signaled if the actual hash of the file does not match the specified hash.</para> <para>This command prints the hash on standard output. Additionally, -if the environment variable <envar>PRINT_PATH</envar> is set, the path -of the downloaded file in the Nix store is also printed.</para> +if the option <option>--print-path</option> is used, the path of the +downloaded file in the Nix store is also printed.</para> </refsection> @@ -63,7 +64,7 @@ of the downloaded file in the Nix store is also printed.</para> <refsection><title>Options</title> <variablelist> - + <varlistentry><term><option>--type</option> <replaceable>hashAlgo</replaceable></term> <listitem><para>Use the specified cryptographic hash algorithm, @@ -73,6 +74,35 @@ of the downloaded file in the Nix store is also printed.</para> </varlistentry> + <varlistentry><term><option>--print-path</option></term> + + <listitem><para>Print the store path of the downloaded file on + standard output.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--unpack</option></term> + + <listitem><para>Unpack the archive (which must be a tarball or zip + file) and add the result to the Nix store. The resulting hash can + be used with functions such as Nixpkgs’s + <varname>fetchzip</varname> or + <varname>fetchFromGitHub</varname>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--name</option></term> + + <listitem><para>Override the name of the file in the Nix store. By + default, this is + <literal><replaceable>hash</replaceable>-<replaceable>basename</replaceable></literal>, + where <replaceable>basename</replaceable> is the last component of + <replaceable>url</replaceable>. Overriding the name is necessary + when <replaceable>basename</replaceable> contains characters that + are not allowed in Nix store paths.</para></listitem> + + </varlistentry> + </variablelist> </refsection> @@ -81,14 +111,19 @@ of the downloaded file in the Nix store is also printed.</para> <refsection><title>Examples</title> <screen> -$ nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2 -0bbd1df101bc0294d440471e50feca71 +$ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz +0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i -$ PRINT_PATH=1 nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2 -0bbd1df101bc0294d440471e50feca71 -/nix/store/wvyz8ifdn7wyz1p3pqyn0ra45ka2l492-make-3.80.tar.bz2</screen> +$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz +0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + +$ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz +079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7 +/nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz +</screen> </refsection> - + </refentry> diff --git a/doc/manual/command-ref/nix-pull.xml b/doc/manual/command-ref/nix-pull.xml deleted file mode 100644 index eb471677b63f..000000000000 --- a/doc/manual/command-ref/nix-pull.xml +++ /dev/null @@ -1,54 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-pull"> - -<refmeta> - <refentrytitle>nix-pull</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-pull</refname> - <refpurpose>register availability of pre-built binaries (deprecated)</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-pull</command> - <arg choice='plain'><replaceable>url</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<note><para>This command and the use of manifests is deprecated. It is -better to use binary caches.</para></note> - -<para>The command <command>nix-pull</command> obtains a list of -pre-built store paths from the URL <replaceable>url</replaceable>, and -for each of these store paths, registers a substitute derivation that -downloads and unpacks it into the Nix store. This is used to speed up -installations: if you attempt to install something that has already -been built and stored into the network cache, Nix can transparently -re-use the pre-built store paths.</para> - -<para>The file at <replaceable>url</replaceable> must be compatible -with the files created by <replaceable>nix-push</replaceable>.</para> - -</refsection> - - -<refsection><title>Examples</title> - -<screen> -$ nix-pull https://nixos.org/releases/nixpkgs/nixpkgs-15.05pre54468.69858d7/MANIFEST</screen> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-push.xml b/doc/manual/command-ref/nix-push.xml index b8156b4554fd..0749824a0ad4 100644 --- a/doc/manual/command-ref/nix-push.xml +++ b/doc/manual/command-ref/nix-push.xml @@ -73,8 +73,7 @@ automatically.</para> <listitem><para>Optionally, a single <emphasis>manifest</emphasis> file is created that contains the same metadata as the <filename>.narinfo</filename> files. This is for compatibility with - Nix versions prior to 1.2 (see <command>nix-pull</command> for - details).</para></listitem> + Nix versions prior to 1.2.</para></listitem> <listitem><para>A file named <option>nix-cache-info</option> is placed in the destination directory. The existence of this file @@ -135,7 +134,7 @@ automatically.</para> <varlistentry><term><option>--manifest</option></term> <listitem><para>Force the generation of a manifest suitable for - use by <command>nix-pull</command>. The manifest is stored as + use by old versions of Nix. The manifest is stored as <filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem> </varlistentry> @@ -203,20 +202,6 @@ $ nix-push --dest /tmp/cache $(nix-instantiate -A thunderbird) </para> -<para>To generate a manifest suitable for <command>nix-pull</command>: - -<screen> -$ nix-push --dest /tmp/cache $(nix-build -A thunderbird) --manifest -</screen> - -On another machine you can then do: - -<screen> -$ nix-pull http://example.org/cache -</screen> - -to cause the binaries to be used by subsequent Nix operations.</para> - <para>To generate a signed binary cache, you must first generate a key pair, in this example called <literal>cache.example.org-1</literal>, storing the secret key in <filename>./sk</filename> and the public key diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index 9e3e6d1882ba..c64c93ec3ac2 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -176,6 +176,22 @@ also <xref linkend="sec-common-options" />.</phrase></para> </refsection> +<refsection><title>Environment variables</title> + +<variablelist> + + <varlistentry><term><envar>NIX_BUILD_SHELL</envar></term> + + <listitem><para>Shell used to start the interactive environment. + Defaults to the <command>bash</command> found in <envar>PATH</envar>.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + <refsection><title>Examples</title> <para>To build the dependencies of the package Pan, and start an @@ -251,9 +267,9 @@ dependencies in Nixpkgs.</para> <para>The lines starting with <literal>#! nix-shell</literal> specify <command>nix-shell</command> options (see above). Note that you cannot -write <literal>#1 /usr/bin/env nix-shell -i ...</literal> because -<command>/usr/bin/env</command> does not support passing options to -the interpreter.</para> +write <literal>#! /usr/bin/env nix-shell -i ...</literal> because +many operating systems only allow one argument in +<literal>#!</literal> lines.</para> <para>For example, here is a Python script that depends on Python and the <literal>prettytable</literal> package: diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index e21d53d8b9f0..340f61210b2d 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -194,6 +194,25 @@ printed.)</para> </varlistentry> + <varlistentry><term><option>--check</option></term> + + <listitem><para>This option allows you to check whether a + derivation is deterministic. It rebuilds the specified derivation + and checks whether the result is bitwise-identical with the + existing outputs, printing an error if that’s not the case. The + outputs of the specified derivation must already exist. When used + with <option>-K</option>, if an output path is not identical to + the corresponding output from the previous build, the new output + path is left in + <filename>/nix/store/<replaceable>name</replaceable>-check.</filename></para> + + <para>See also the <option>build-repeat</option> configuration + option, which repeats a derivation a number of times and prevents + its outputs from being registered as “valid” in the Nix store + unless they are identical.</para></listitem> + + </varlistentry> + </variablelist> </refsection> @@ -212,6 +231,73 @@ $ nix-store -r $(nix-instantiate ./test.nix) This is essentially what <link linkend="sec-nix-build"><command>nix-build</command></link> does.</para> +<para>To test whether a previously-built derivation is deterministic: + +<screen> +$ nix-build -r '<nixpkgs>' -A hello --check -K +</screen> + +</para> + +</refsection> + + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id='rsec-nix-store-serve'><title>Operation <option>--serve</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--serve</option></arg> + <arg><option>--write</option></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--serve</option> provides access to +the Nix store over stdin and stdout, and is intended to be used +as a means of providing Nix store access to a restricted ssh user. +</para> + +<para>The following flags are available:</para> + +<variablelist> + + <varlistentry><term><option>--write</option></term> + + <listitem><para>Allow the connected client to request the realization + of derivations. In effect, this can be used to make the host act + as a build slave.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>To turn a host into a build server, the +<filename>authorized_keys</filename> file can be used to provide build +access to a given SSH public key: + +<screen> +$ cat <<EOF >>/root/.ssh/authorized_keys +command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... +EOF +</screen> + +</para> + </refsection> @@ -1264,82 +1350,6 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil <!--######################################################################--> -<refsection><title>Operation <option>--query-failed-paths</option></title> - -<refsection> - <title>Synopsis</title> - <cmdsynopsis> - <command>nix-store</command> - <arg choice='plain'><option>--query-failed-paths</option></arg> - </cmdsynopsis> -</refsection> - -<refsection><title>Description</title> - -<para>If build failure caching is enabled through the -<literal>build-cache-failure</literal> configuration option, the -operation <option>--query-failed-paths</option> will print out all -store paths that have failed to build.</para> - -</refsection> - -<refsection><title>Example</title> - -<screen> -$ nix-store --query-failed-paths -/nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59 -/nix/store/0011iy7sfwbc1qj5a1f6ifjnbcdail8a-haskell-gitit-ghc7.0.4-0.8.1 -/nix/store/001c0yn1hkh86gprvrb46cxnz3pki7q3-gamin-0.1.10 -<replaceable>…</replaceable> -</screen> - -</refsection> - -</refsection> - - -<!--######################################################################--> - -<refsection><title>Operation <option>--clear-failed-paths</option></title> - -<refsection> - <title>Synopsis</title> - <cmdsynopsis> - <command>nix-store</command> - <arg choice='plain'><option>--clear-failed-paths</option></arg> - <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> - </cmdsynopsis> -</refsection> - -<refsection><title>Description</title> - -<para>If build failure caching is enabled through the -<literal>build-cache-failure</literal> configuration option, the -operation <option>--clear-failed-paths</option> clears the “failed” -state of the given store paths, allowing them to be built again. This -is useful if the failure was actually transient (e.g. because the disk -was full).</para> - -<para>If a path denotes a derivation, its output paths are cleared. -You can provide the argument <literal>*</literal> to clear all store -paths.</para> - -</refsection> - -<refsection><title>Example</title> - -<screen> -$ nix-store --clear-failed-paths /nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59 -$ nix-store --clear-failed-paths * -</screen> - -</refsection> - -</refsection> - - -<!--######################################################################--> - <refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title> <refsection> diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index d65f4009ee6e..5b7936393951 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -31,7 +31,6 @@ <arg><option>-K</option></arg> <arg><option>--fallback</option></arg> <arg><option>--readonly-mode</option></arg> -<arg><option>--log-type</option> <replaceable>type</replaceable></arg> <arg><option>--show-trace</option></arg> <arg> <option>-I</option> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index c7e8ae1ed05f..bc26a90616e4 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -201,61 +201,6 @@ </varlistentry> -<varlistentry xml:id="opt-log-type"><term><option>--log-type</option> -<replaceable>type</replaceable></term> - - <listitem> - - <para>This option determines how the output written to standard - error is formatted. Nix’s diagnostic messages are typically - <emphasis>nested</emphasis>. For instance, when tracing Nix - expression evaluation (<command>nix-env -vvvvv</command>, messages - from subexpressions are nested inside their parent expressions. Nix - builder output is also often nested. For instance, the Nix Packages - generic builder nests the various build tasks (unpack, configure, - compile, etc.), and the GNU Make in <literal>stdenv-linux</literal> - has been patched to provide nesting for recursive Make - invocations.</para> - - <para><replaceable>type</replaceable> can be one of the - following: - - <variablelist> - - <varlistentry><term><literal>pretty</literal></term> - - <listitem><para>Pretty-print the output, indicating different - nesting levels using spaces. This is the - default.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>escapes</literal></term> - - <listitem><para>Indicate nesting using escape codes that can be - interpreted by the <command>nix-log2xml</command> tool in the - Nix source distribution. The resulting XML file can be fed into - the <command>log2html.xsl</command> stylesheet to create an HTML - file that can be browsed interactively, using JavaScript to - expand and collapse parts of the output.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>flat</literal></term> - - <listitem><para>Remove all nesting.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - </listitem> - -</varlistentry> - - <varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term> <listitem><para>This option is accepted by diff --git a/doc/manual/command-ref/utilities.xml b/doc/manual/command-ref/utilities.xml index be2fe6e2d235..25e457e4e554 100644 --- a/doc/manual/command-ref/utilities.xml +++ b/doc/manual/command-ref/utilities.xml @@ -13,14 +13,10 @@ work with Nix.</para> <xi:include href="nix-collect-garbage.xml" /> <xi:include href="nix-copy-closure.xml" /> <xi:include href="nix-daemon.xml" /> -<!-- -<xi:include href="nix-generate-patches.xml" /> ---> <xi:include href="nix-hash.xml" /> <xi:include href="nix-install-package.xml" /> <xi:include href="nix-instantiate.xml" /> <xi:include href="nix-prefetch-url.xml" /> -<xi:include href="nix-pull.xml" /> <xi:include href="nix-push.xml" /> </chapter> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 6bdfdd55c4b6..eae5f5a029bf 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -32,13 +32,35 @@ available as <function>builtins.derivation</function>.</para> <varlistentry><term><function>builtins.add</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> - <listitem><para>Return the sum of the integers + <listitem><para>Return the sum of the numbers <replaceable>e1</replaceable> and <replaceable>e2</replaceable>.</para></listitem> </varlistentry> + <varlistentry><term><function>builtins.all</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Return <literal>true</literal> if the function + <replaceable>pred</replaceable> returns <literal>true</literal> + for all elements of <replaceable>list</replaceable>, + and <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry><term><function>builtins.any</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Return <literal>true</literal> if the function + <replaceable>pred</replaceable> returns <literal>true</literal> + for at least one element of <replaceable>list</replaceable>, + and <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.attrNames</function> <replaceable>set</replaceable></term> @@ -182,7 +204,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> <varlistentry><term><function>builtins.div</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> - <listitem><para>Return the quotient of the integers + <listitem><para>Return the quotient of the numbers <replaceable>e1</replaceable> and <replaceable>e2</replaceable>.</para></listitem> @@ -313,6 +335,37 @@ stdenv.mkDerivation { </varlistentry> + <varlistentry><term><function>builtins.foldl’</function> + <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Reduce a list by applying a binary operator, from + left to right, e.g. <literal>foldl’ op nul [x0 x1 x2 ...] = op (op + (op nul x0) x1) x2) ...</literal>. The operator is applied + strictly, i.e., its arguments are evaluated first. For example, + <literal>foldl’ (x: y: x + y) 0 [1 2 3]</literal> evaluates to + 6.</para></listitem> + + </varlistentry> + + + <varlistentry><term><function>builtins.functionArgs</function> + <replaceable>f</replaceable></term> + + <listitem><para> + Return a set containing the names of the formal arguments expected + by the function <replaceable>f</replaceable>. + The value of each attribute is a Boolean denoting whether the corresponding + argument has a default value. For instance, + <literal>functionArgs ({ x, y ? 123}: ...) = { x = false; y = true; }</literal>. + </para> + + <para>"Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + <literal>functionArgs (x: ...) = { }</literal>. + </para></listitem> + </varlistentry> + + <varlistentry><term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term> <listitem><para>Convert a JSON string to a Nix @@ -329,6 +382,24 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' </varlistentry> + <varlistentry><term><function>builtins.genList</function> + <replaceable>generator</replaceable> <replaceable>length</replaceable></term> + + <listitem><para>Generate list of size + <replaceable>length</replaceable>, with each element + <replaceable>i></replaceable> equal to the value returned by + <replaceable>generator</replaceable> <literal>i</literal>. For + example, + +<programlisting> +builtins.genList (x: x * x) 5 +</programlisting> + + returns the list <literal>[ 0 1 4 9 16 ]</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.getAttr</function> <replaceable>s</replaceable> <replaceable>set</replaceable></term> @@ -549,12 +620,12 @@ x: x + 456</programlisting> <varlistentry><term><function>builtins.lessThan</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> - <listitem><para>Return <literal>true</literal> if the integer - <replaceable>e1</replaceable> is less than the integer + <listitem><para>Return <literal>true</literal> if the number + <replaceable>e1</replaceable> is less than the number <replaceable>e2</replaceable>, and <literal>false</literal> otherwise. Evaluation aborts if either <replaceable>e1</replaceable> or <replaceable>e2</replaceable> - does not evaluate to an integer.</para></listitem> + does not evaluate to a number.</para></listitem> </varlistentry> @@ -605,7 +676,7 @@ map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting> <varlistentry><term><function>builtins.mul</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> - <listitem><para>Return the product of the integers + <listitem><para>Return the product of the numbers <replaceable>e1</replaceable> and <replaceable>e2</replaceable>.</para></listitem> @@ -663,7 +734,7 @@ in config.someSetting</programlisting> ./A</literal> will return the set <programlisting> -{ A = "regular"; B = "directory"; }</programlisting> +{ B = "regular"; C = "directory"; }</programlisting> The possible values for the file type are <literal>"regular"</literal>, <literal>"directory"</literal>, @@ -698,6 +769,23 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> </varlistentry> + <varlistentry><term><function>builtins.replaceStrings</function> + <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term> + + <listitem><para>Given string <replaceable>s</replaceable>, replace + every occurrence of the strings in <replaceable>from</replaceable> + with the corresponding string in + <replaceable>to</replaceable>. For example, + +<programlisting> +builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" +</programlisting> + + evaluates to <literal>"fabir"</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.seq</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> @@ -709,6 +797,29 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> </varlistentry> + <varlistentry><term><function>builtins.sort</function> + <replaceable>comparator</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Return <replaceable>list</replaceable> in sorted + order. It repeatedly calls the function + <replaceable>comparator</replaceable> with two elements. The + comparator should return <literal>true</literal> if the first + element is less than the second, and <literal>false</literal> + otherwise. For example, + +<programlisting> +builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] +</programlisting> + + produces the list <literal>[ 42 77 147 249 483 526 + ]</literal>.</para> + + <para>This is a stable sort: it preserves the relative order of + elements deemed equal by the comparator.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.stringLength</function> <replaceable>e</replaceable></term> @@ -722,7 +833,7 @@ removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> <varlistentry><term><function>builtins.sub</function> <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> - <listitem><para>Return the difference between the integers + <listitem><para>Return the difference between the numbers <replaceable>e1</replaceable> and <replaceable>e2</replaceable>.</para></listitem> @@ -849,7 +960,7 @@ in foo</programlisting> <varlistentry><term><function>builtins.toJSON</function> <replaceable>e</replaceable></term> <listitem><para>Return a string containing a JSON representation - of <replaceable>e</replaceable>. Strings, integers, booleans, + of <replaceable>e</replaceable>. Strings, integers, floats, booleans, nulls and lists are mapped to their JSON equivalents. Sets (except derivations) are represented as objects. Derivations are translated to a JSON string containing the derivation’s output diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml index 90e2786faaab..f2a73dccfe18 100644 --- a/doc/manual/expressions/derivations.xml +++ b/doc/manual/expressions/derivations.xml @@ -43,7 +43,7 @@ of which specify the inputs of the build.</para> <itemizedlist> - <listitem><para>Strings and integers are just passed + <listitem><para>Strings and numbers are just passed verbatim.</para></listitem> <listitem><para>A <emphasis>path</emphasis> (e.g., diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml index 0bf6632d6e3a..f1174ecb5d8d 100644 --- a/doc/manual/expressions/language-values.xml +++ b/doc/manual/expressions/language-values.xml @@ -140,8 +140,13 @@ stdenv.mkDerivation { </listitem> - <listitem><para><emphasis>Integers</emphasis>, e.g., - <literal>123</literal>.</para></listitem> + <listitem><para>Numbers, which can be <emphasis>integers</emphasis> (like + <literal>123</literal>) or <emphasis>floating point</emphasis> (like + <literal>123.43</literal> or <literal>.27e13</literal>).</para> + + <para>Numbers are type-compatible: pure integer operations will always + return integers, whereas any operation involving at least one floating point + number will have a floating point number as a result.</para></listitem> <listitem><para><emphasis>Paths</emphasis>, e.g., <filename>/bin/sh</filename> or <filename>./builder.sh</filename>. diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index e0dd98b7e67e..bd3901a13351 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -7,15 +7,14 @@ <title>Building and Testing</title> <para>You can now try to build Hello. Of course, you could do -<literal>nix-env -f pkgs/top-level/all-packages.nix -i hello</literal>, -but you may not want to install a possibly broken package just yet. -The best way to test the package is by using the command <command -linkend="sec-nix-build">nix-build</command>, which builds a Nix -expression and creates a symlink named <filename>result</filename> in -the current directory: +<literal>nix-env -i hello</literal>, but you may not want to install a +possibly broken package just yet. The best way to test the package is by +using the command <command linkend="sec-nix-build">nix-build</command>, +which builds a Nix expression and creates a symlink named +<filename>result</filename> in the current directory: <screen> -$ nix-build pkgs/top-level/all-packages.nix -A hello +$ nix-build -A hello building path `/nix/store/632d2b22514d...-hello-2.1.1' hello-2.1.1/ hello-2.1.1/intl/ @@ -29,8 +28,7 @@ $ ./result/bin/hello Hello, world!</screen> The <link linkend='opt-attr'><option>-A</option></link> option selects -the <literal>hello</literal> attribute from -<filename>all-packages.nix</filename>. This is faster than using the +the <literal>hello</literal> attribute. This is faster than using the symbolic package name specified by the <literal>name</literal> attribute (which also happens to be <literal>hello</literal>) and is unambiguous (there can be multiple packages with the symbolic name @@ -69,7 +67,7 @@ block (or perform other derivations if available) until the build finishes: <screen> -$ nix-build pkgs/top-level/all-packages.nix -A hello +$ nix-build -A hello waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</screen> So it is always safe to run multiple instances of Nix in parallel diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index f9ee98c726d2..e9a09cba7030 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -28,6 +28,7 @@ $ chown alice /nix </para> +<!-- <para>You can also manually download and install a binary package. Binary packages of the latest stable release are available for Fedora, Debian, Ubuntu, Mac OS X and various other systems from the <link @@ -53,12 +54,12 @@ $ dpkg -i nix_1.8-1_amd64.deb</screen> </para> -<para>For other platforms, including Mac OS X and other Linux -distributions, you can download a binary tarball that contains Nix and -all its dependencies. (This is what the install script at -<uri>https://nixos.org/nix/install</uri> uses.) You should unpack it -somewhere (e.g. in <filename>/tmp</filename>), and then run the script -named <command>install</command> inside the binary tarball: +<para>You can also download a binary tarball that contains Nix and all +its dependencies. (This is what the install script at +<uri>https://nixos.org/nix/install</uri> does automatically.) You +should unpack it somewhere (e.g. in <filename>/tmp</filename>), and +then run the script named <command>install</command> inside the binary +tarball: <screen> alice$ cd /tmp @@ -78,5 +79,14 @@ other auxiliary data, if desired: $ rm -rf /nix</screen> </para> +--> + +<para>You can uninstall Nix simply by running: + +<screen> +$ rm -rf /nix +</screen> + +</para> </chapter> diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index 47adc9a4fd67..49036d940bd4 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -30,13 +30,13 @@ or higher. If your distribution does not provide it, please install it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem> - <listitem><para>The Perl DBI and DBD::SQLite libraries, which are + <listitem><para>The Perl DBI, DBD::SQLite, and WWW::Curl libraries, which are available from <link xlink:href="http://search.cpan.org/">CPAN</link> if your distribution does not provide them.</para></listitem> <listitem><para>The <link - xlink:href="http://www.hpl.hp.com/personal/Hans_Boehm/gc/">Boehm + xlink:href="http://www.hboehm.info/gc/">Boehm garbage collector</link> to reduce the evaluator’s memory consumption (optional). To enable it, install <literal>pkgconfig</literal> and the Boehm garbage collector, and @@ -70,4 +70,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml index 66679ac9a85a..0c58984ac489 100644 --- a/doc/manual/introduction/about-nix.xml +++ b/doc/manual/introduction/about-nix.xml @@ -62,9 +62,10 @@ directories such as so if a package builds correctly on your system, this is because you specified the dependency explicitly.</para> -<para>Runtime dependencies are found by scanning binaries for the hash -parts of Nix store paths (such as <literal>r8vvq9kq…</literal>). This -sounds risky, but it works extremely well.</para> +<para>Once a package is built, runtime dependencies are found by +scanning binaries for the hash parts of Nix store paths (such as +<literal>r8vvq9kq…</literal>). This sounds risky, but it works +extremely well.</para> </simplesect> diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 3d7e7fed9631..52f2884ab18f 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -39,7 +39,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ - nix-collect-garbage.1 nix-push.1 nix-pull.1 \ + nix-collect-garbage.1 nix-push.1 \ nix-prefetch-url.1 nix-channel.1 \ nix-install-package.1 nix-hash.1 nix-copy-closure.1 \ nix.conf.5 nix-daemon.8, \ @@ -76,17 +76,3 @@ all: $(d)/manual.html clean-files += $(d)/manual.html dist-files += $(d)/manual.html - - -# Generate the PDF manual. -$(d)/manual.pdf: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid - $(trace-gen) if test "$(dblatex)" != ""; then \ - cd doc/manual && $(XSLTPROC) --xinclude --stringparam profile.condition manual \ - $(docbookxsl)/profiling/profile.xsl manual.xml | \ - $(dblatex) -o $(notdir $@) $(dblatex_opts) -; \ - else \ - echo "Please install dblatex and rerun configure."; \ - exit 1; \ - fi - -clean-files += $(d)/manual.pdf diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index ba72901378e5..8c2deb394183 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,9 @@ </partintro> --> +<xi:include href="rl-1.12.xml" /> +<xi:include href="rl-1.11.xml" /> +<xi:include href="rl-1.10.xml" /> <xi:include href="rl-1.9.xml" /> <xi:include href="rl-1.8.xml" /> <xi:include href="rl-1.7.xml" /> diff --git a/doc/manual/release-notes/rl-0.10.1.xml b/doc/manual/release-notes/rl-0.10.1.xml index 05cd2f654353..95829323d4fb 100644 --- a/doc/manual/release-notes/rl-0.10.1.xml +++ b/doc/manual/release-notes/rl-0.10.1.xml @@ -4,10 +4,10 @@ version="5.0" xml:id="ch-relnotes-0.10.1"> -<title>Release 0.10.1 (October 11, 2006)</title> +<title>Release 0.10.1 (2006-10-11)</title> <para>This release fixes two somewhat obscure bugs that occur when evaluating Nix expressions that are stored inside the Nix store (<literal>NIX-67</literal>). These do not affect most users.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.10.xml b/doc/manual/release-notes/rl-0.10.xml index 7815ae75b1b8..9afec4de94de 100644 --- a/doc/manual/release-notes/rl-0.10.xml +++ b/doc/manual/release-notes/rl-0.10.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.10"> -<title>Release 0.10 (October 6, 2006)</title> +<title>Release 0.10 (2006-10-06)</title> <note><para>This version of Nix uses Berkeley DB 4.4 instead of 4.3. The database is upgraded automatically, but you should be careful not @@ -320,4 +320,4 @@ irreversible.</para></warning> </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.11.xml b/doc/manual/release-notes/rl-0.11.xml index 9c5d8b8beb4b..7ad0ab5b71ad 100644 --- a/doc/manual/release-notes/rl-0.11.xml +++ b/doc/manual/release-notes/rl-0.11.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-0.11"> -<title>Release 0.11 (December 31, 2007)</title> +<title>Release 0.11 (2007-12-31)</title> <para>Nix 0.11 has many improvements over the previous stable release. The most important improvement is secure multi-user support. It also @@ -258,4 +258,4 @@ on Nix. Here is an (incomplete) list:</para> </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.12.xml b/doc/manual/release-notes/rl-0.12.xml index 1f04609b1654..fdba8c4d577f 100644 --- a/doc/manual/release-notes/rl-0.12.xml +++ b/doc/manual/release-notes/rl-0.12.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-0.12"> -<title>Release 0.12 (November 20, 2008)</title> +<title>Release 0.12 (2008-11-20)</title> <itemizedlist> @@ -172,4 +172,4 @@ the following paths will be downloaded/copied (30.02 MiB): </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.13.xml b/doc/manual/release-notes/rl-0.13.xml index 9cf144e4ea23..cce2e4a26b05 100644 --- a/doc/manual/release-notes/rl-0.13.xml +++ b/doc/manual/release-notes/rl-0.13.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-0.13"> -<title>Release 0.13 (November 5, 2009)</title> +<title>Release 0.13 (2009-11-05)</title> <para>This is primarily a bug fix release. It has some new features:</para> @@ -103,4 +103,4 @@ features:</para> </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.14.xml b/doc/manual/release-notes/rl-0.14.xml index a671db3b8588..e5fe9da78e7e 100644 --- a/doc/manual/release-notes/rl-0.14.xml +++ b/doc/manual/release-notes/rl-0.14.xml @@ -2,7 +2,9 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" version="5.0" - xml:id="ssec-relnotes-0.14"><title>Release 0.14 (February 4, 2010)</title> + xml:id="ssec-relnotes-0.14"> + +<title>Release 0.14 (2010-02-04)</title> <para>This release has the following improvements:</para> @@ -41,4 +43,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.15.xml b/doc/manual/release-notes/rl-0.15.xml index b76f674ba123..9f58a8efc5d4 100644 --- a/doc/manual/release-notes/rl-0.15.xml +++ b/doc/manual/release-notes/rl-0.15.xml @@ -4,11 +4,11 @@ version="5.0" xml:id="ssec-relnotes-0.15"> -<title>Release 0.15 (March 17, 2010)</title> +<title>Release 0.15 (2010-03-17)</title> <para>This is a bug-fix release. Among other things, it fixes building on Mac OS X (Snow Leopard), and improves the contents of <filename>/etc/passwd</filename> and <filename>/etc/group</filename> in <literal>chroot</literal> builds.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.16.xml b/doc/manual/release-notes/rl-0.16.xml index 5254e5f9d658..af1edc0ebbe0 100644 --- a/doc/manual/release-notes/rl-0.16.xml +++ b/doc/manual/release-notes/rl-0.16.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-0.16"> -<title>Release 0.16 (August 17, 2010)</title> +<title>Release 0.16 (2010-08-17)</title> <para>This release has the following improvements:</para> @@ -52,4 +52,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.6.xml b/doc/manual/release-notes/rl-0.6.xml index 83d9dc897c42..6dc6521d3c2a 100644 --- a/doc/manual/release-notes/rl-0.6.xml +++ b/doc/manual/release-notes/rl-0.6.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.6"> -<title>Release 0.6 (November 14, 2004)</title> +<title>Release 0.6 (2004-11-14)</title> <itemizedlist> @@ -119,4 +119,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.7.xml b/doc/manual/release-notes/rl-0.7.xml index fc8997fc1e36..6f95db4367db 100644 --- a/doc/manual/release-notes/rl-0.7.xml +++ b/doc/manual/release-notes/rl-0.7.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.7"> -<title>Release 0.7 (January 12, 2005)</title> +<title>Release 0.7 (2005-01-12)</title> <itemizedlist> @@ -32,4 +32,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.8.1.xml b/doc/manual/release-notes/rl-0.8.1.xml index b4a855b553a5..f7ffca0f8d59 100644 --- a/doc/manual/release-notes/rl-0.8.1.xml +++ b/doc/manual/release-notes/rl-0.8.1.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.8.1"> -<title>Release 0.8.1 (April 13, 2005)</title> +<title>Release 0.8.1 (2005-04-13)</title> <para>This is a bug fix release.</para> @@ -18,4 +18,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml index 970abb6e85e0..784b26c6b7d3 100644 --- a/doc/manual/release-notes/rl-0.8.xml +++ b/doc/manual/release-notes/rl-0.8.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.8"> -<title>Release 0.8 (April 11, 2005)</title> +<title>Release 0.8 (2005-04-11)</title> <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a result, <command>nix-pull</command> manifests and channels built @@ -243,4 +243,4 @@ $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'</screen> </para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.9.1.xml b/doc/manual/release-notes/rl-0.9.1.xml index a7d093aa8e02..85d11f416877 100644 --- a/doc/manual/release-notes/rl-0.9.1.xml +++ b/doc/manual/release-notes/rl-0.9.1.xml @@ -4,10 +4,10 @@ version="5.0" xml:id="ch-relnotes-0.9.1"> -<title>Release 0.9.1 (September 20, 2005)</title> +<title>Release 0.9.1 (2005-09-20)</title> <para>This bug fix release addresses a problem with the ATerm library when the <option>--with-aterm</option> flag in <command>configure</command> was <emphasis>not</emphasis> used.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.9.2.xml b/doc/manual/release-notes/rl-0.9.2.xml index 33141c8e94ff..cb705e98ac25 100644 --- a/doc/manual/release-notes/rl-0.9.2.xml +++ b/doc/manual/release-notes/rl-0.9.2.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.9.2"> -<title>Release 0.9.2 (September 21, 2005)</title> +<title>Release 0.9.2 (2005-09-21)</title> <para>This bug fix release fixes two problems on Mac OS X: @@ -25,4 +25,4 @@ </para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-0.9.xml b/doc/manual/release-notes/rl-0.9.xml index 07dd87cd9f61..fd1e633f78ea 100644 --- a/doc/manual/release-notes/rl-0.9.xml +++ b/doc/manual/release-notes/rl-0.9.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-relnotes-0.9"> -<title>Release 0.9 (September 16, 2005)</title> +<title>Release 0.9 (2005-09-16)</title> <para>NOTE: this version of Nix uses Berkeley DB 4.3 instead of 4.2. The database is upgraded automatically, but you should be careful not @@ -95,4 +95,4 @@ svnService = derivation { </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.0.xml b/doc/manual/release-notes/rl-1.0.xml index d000014eb311..ff11168d0932 100644 --- a/doc/manual/release-notes/rl-1.0.xml +++ b/doc/manual/release-notes/rl-1.0.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.0"> -<title>Release 1.0 (May 11, 2012)</title> +<title>Release 1.0 (2012-05-11)</title> <para>There have been numerous improvements and bug fixes since the previous release. Here are the most significant:</para> @@ -116,4 +116,4 @@ previous release. Here are the most significant:</para> </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.1.xml b/doc/manual/release-notes/rl-1.1.xml index 7ee076f193e9..2f26e7a24273 100644 --- a/doc/manual/release-notes/rl-1.1.xml +++ b/doc/manual/release-notes/rl-1.1.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.1"> -<title>Release 1.1 (July 18, 2012)</title> +<title>Release 1.1 (2012-07-18)</title> <para>This release has the following improvements:</para> @@ -97,4 +97,4 @@ </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.10.xml b/doc/manual/release-notes/rl-1.10.xml new file mode 100644 index 000000000000..689a95466343 --- /dev/null +++ b/doc/manual/release-notes/rl-1.10.xml @@ -0,0 +1,64 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.10"> + +<title>Release 1.10 (2015-09-03)</title> + +<para>This is primarily a bug fix release. It also has a number of new +features:</para> + +<itemizedlist> + + <listitem> + <para>A number of builtin functions have been added to reduce + Nixpkgs/NixOS evaluation time and memory consumption: + <function>all</function>, + <function>any</function>, + <function>concatStringsSep</function>, + <function>foldl’</function>, + <function>genList</function>, + <function>replaceStrings</function>, + <function>sort</function>. + </para> + </listitem> + + <listitem> + <para>The garbage collector is more robust when the disk is full.</para> + </listitem> + + <listitem> + <para>Nix supports a new API for building derivations that doesn’t + require a <literal>.drv</literal> file to be present on disk; it + only requires an in-memory representation of the derivation. This + is used by the Hydra continuous build system to make remote builds + more efficient.</para> + </listitem> + + <listitem> + <para>The function <literal><nix/fetchurl.nix></literal> now + uses a <emphasis>builtin</emphasis> builder (i.e. it doesn’t + require starting an external process; the download is performed by + Nix itself). This ensures that derivation paths don’t change when + Nix is upgraded, and obviates the need for ugly hacks to support + chroot execution.</para> + </listitem> + + <listitem> + <para><option>--version -v</option> now prints some configuration + information, in particular what compile-time optional features are + enabled, and the paths of various directories.</para> + </listitem> + + <listitem> + <para>Build users have their supplementary groups set correctly.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Eelco Dolstra, Guillaume +Maudoux, Iwan Aucamp, Jaka Hudoklin, Kirill Elagin, Ludovic Courtès, +Manolis Ragkousis, Nicolas B. Pierron and Shea Levy.</para> + +</section> diff --git a/doc/manual/release-notes/rl-1.11.xml b/doc/manual/release-notes/rl-1.11.xml new file mode 100644 index 000000000000..efb03d61393f --- /dev/null +++ b/doc/manual/release-notes/rl-1.11.xml @@ -0,0 +1,148 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.11"> + +<title>Release 1.11 (2016-01-19)</title> + +<para>This is primarily a bug fix release. It also has a number of new +features:</para> + +<itemizedlist> + + <listitem> + <para><command>nix-prefetch-url</command> can now download URLs + specified in a Nix expression. For example, + +<screen> +$ nix-prefetch-url -A hello.src +</screen> + + will prefetch the file specified by the + <function>fetchurl</function> call in the attribute + <literal>hello.src</literal> from the Nix expression in the + current directory, and print the cryptographic hash of the + resulting file on stdout. This differs from <literal>nix-build -A + hello.src</literal> in that it doesn't verify the hash, and is + thus useful when you’re updating a Nix expression.</para> + + <para>You can also prefetch the result of functions that unpack a + tarball, such as <function>fetchFromGitHub</function>. For example: + +<screen> +$ nix-prefetch-url --unpack https://github.com/NixOS/patchelf/archive/0.8.tar.gz +</screen> + + or from a Nix expression: + +<screen> +$ nix-prefetch-url -A nix-repl.src +</screen> + + </para> + + </listitem> + + <listitem> + <para>The builtin function + <function><nix/fetchurl.nix></function> now supports + downloading and unpacking NARs. This removes the need to have + multiple downloads in the Nixpkgs stdenv bootstrap process (like a + separate busybox binary for Linux, or curl/mkdir/sh/bzip2 for + Darwin). Now all those files can be combined into a single NAR, + optionally compressed using <command>xz</command>.</para> + </listitem> + + <listitem> + <para>Nix now supports SHA-512 hashes for verifying fixed-output + derivations, and in <function>builtins.hashString</function>.</para> + </listitem> + + <listitem> + <para> + The new flag <option>--option build-repeat + <replaceable>N</replaceable></option> will cause every build to + be executed <replaceable>N</replaceable>+1 times. If the build + output differs between any round, the build is rejected, and the + output paths are not registered as valid. This is primarily + useful to verify build determinism. (We already had a + <option>--check</option> option to repeat a previously succeeded + build. However, with <option>--check</option>, non-deterministic + builds are registered in the DB. Preventing that is useful for + Hydra to ensure that non-deterministic builds don't end up + getting published to the binary cache.) + </para> + </listitem> + + <listitem> + <para> + The options <option>--check</option> and <option>--option + build-repeat <replaceable>N</replaceable></option>, if they + detect a difference between two runs of the same derivation and + <option>-K</option> is given, will make the output of the other + run available under + <filename><replaceable>store-path</replaceable>-check</filename>. This + makes it easier to investigate the non-determinism using tools + like <command>diffoscope</command>, e.g., + +<screen> +$ nix-build pkgs/stdenv/linux -A stage1.pkgs.zlib --check -K +error: derivation ‘/nix/store/l54i8wlw2265…-zlib-1.2.8.drv’ may not +be deterministic: output ‘/nix/store/11a27shh6n2i…-zlib-1.2.8’ +differs from ‘/nix/store/11a27shh6n2i…-zlib-1.2.8-check’ + +$ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zlib-1.2.8-check +… +├── lib/libz.a +│ ├── metadata +│ │ @@ -1,15 +1,15 @@ +│ │ -rw-r--r-- 30001/30000 3096 Jan 12 15:20 2016 adler32.o +… +│ │ +rw-r--r-- 30001/30000 3096 Jan 12 15:28 2016 adler32.o +… +</screen> + + </para></listitem> + + <listitem> + <para>Improved FreeBSD support.</para> + </listitem> + + <listitem> + <para><command>nix-env -qa --xml --meta</command> now prints + license information.</para> + </listitem> + + <listitem> + <para>The maximum number of parallel TCP connections that the + binary cache substituter will use has been decreased from 150 to + 25. This should prevent upsetting some broken NAT routers, and + also improves performance.</para> + </listitem> + + <listitem> + <para>The Nix language now supports floating point numbers. They are + based on regular C++ <literal>float</literal> and compatible with + existing integers and number-related operations. Export and import to and + from JSON and XML works, too. + </para> + </listitem> + <listitem> + <para>All "chroot"-containing strings got renamed to "sandbox". + In particular, some Nix options got renamed, but the old names + are still accepted as lower-priority aliases. + </para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Anders Claesson, Anthony +Cowley, Bjørn Forsman, Brian McKenna, Danny Wilson, davidak, Eelco Dolstra, +Fabian Schmitthenner, FrankHB, Ilya Novoselov, janus, Jim Garrison, John +Ericson, Jude Taylor, Ludovic Courtès, Manuel Jacob, Mathnerd314, +Pascal Wittmann, Peter Simons, Philip Potter, Preston Bennes, Rommel +M. Martinez, Sander van der Burg, Shea Levy, Tim Cuthbertson, Tuomas +Tynkkynen, Utku Demir and Vladimír Čunát.</para> + +</section> diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml new file mode 100644 index 000000000000..d6864b3f55d1 --- /dev/null +++ b/doc/manual/release-notes/rl-1.12.xml @@ -0,0 +1,24 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.12"> + +<title>Release 1.12 (TBA)</title> + +<para>This release has the following new features:</para> + +<itemizedlist> + + <listitem> + <para>It is no longer necessary to set the + <envar>NIX_REMOTE</envar> environment variable if you need to use + the Nix daemon. Nix will use the daemon automatically if you don’t + have write access to the Nix database.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from TBD.</para> + +</section> diff --git a/doc/manual/release-notes/rl-1.2.xml b/doc/manual/release-notes/rl-1.2.xml index 6c05444ff76e..dc272c420ddb 100644 --- a/doc/manual/release-notes/rl-1.2.xml +++ b/doc/manual/release-notes/rl-1.2.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.2"> -<title>Release 1.2 (December 6, 2012)</title> +<title>Release 1.2 (2012-12-06)</title> <para>This release has the following improvements and changes:</para> @@ -154,4 +154,4 @@ $ mount -o remount,ro,bind /nix/store <para>This release has contributions from Eelco Dolstra, Florian Friesdorf, Mats Erik Andersson and Shea Levy.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.3.xml b/doc/manual/release-notes/rl-1.3.xml index 04169ed01b82..e2009ee3ba4b 100644 --- a/doc/manual/release-notes/rl-1.3.xml +++ b/doc/manual/release-notes/rl-1.3.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.3"> -<title>Release 1.3 (January 4, 2013)</title> +<title>Release 1.3 (2013-01-04)</title> <para>This is primarily a bug fix release. When this version is first run on Linux, it removes any immutable bits from the Nix store and @@ -16,4 +16,4 @@ efficient.)</para> <para>This release has contributions from Eelco Dolstra and Stuart Pernsteiner.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.4.xml b/doc/manual/release-notes/rl-1.4.xml index d48e43f21c1b..aefb22f2b934 100644 --- a/doc/manual/release-notes/rl-1.4.xml +++ b/doc/manual/release-notes/rl-1.4.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.4"> -<title>Release 1.4 (February 26, 2013)</title> +<title>Release 1.4 (2013-02-26)</title> <para>This release fixes a security bug in multi-user operation. It was possible for derivations to cause the mode of files outside of the @@ -36,4 +36,4 @@ xlink:href="https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.5.1.xml b/doc/manual/release-notes/rl-1.5.1.xml index 046960313ffd..035c8dbcbb16 100644 --- a/doc/manual/release-notes/rl-1.5.1.xml +++ b/doc/manual/release-notes/rl-1.5.1.xml @@ -4,9 +4,9 @@ version="5.0" xml:id="ssec-relnotes-1.5.1"> -<title>Release 1.5.1 (February 28, 2013)</title> +<title>Release 1.5.1 (2013-02-28)</title> <para>The bug fix to the bug fix had a bug itself, of course. But this time it will work for sure!</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.5.2.xml b/doc/manual/release-notes/rl-1.5.2.xml index d2f53bbdc0bd..7e81dd243284 100644 --- a/doc/manual/release-notes/rl-1.5.2.xml +++ b/doc/manual/release-notes/rl-1.5.2.xml @@ -4,9 +4,9 @@ version="5.0" xml:id="ssec-relnotes-1.5.2"> -<title>Release 1.5.2 (May 13, 2013)</title> +<title>Release 1.5.2 (2013-05-13)</title> <para>This is primarily a bug fix release. It has contributions from Eelco Dolstra, Lluís Batlle i Rossell and Shea Levy.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.5.xml b/doc/manual/release-notes/rl-1.5.xml index 84e0e9024946..8e279d7693e0 100644 --- a/doc/manual/release-notes/rl-1.5.xml +++ b/doc/manual/release-notes/rl-1.5.xml @@ -4,9 +4,9 @@ version="5.0" xml:id="ssec-relnotes-1.5"> -<title>Release 1.5 (February 27, 2013)</title> +<title>Release 1.5 (2013-02-27)</title> <para>This is a brown paper bag release to fix a regression introduced by the hard link security fix in 1.4.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.6.1.xml b/doc/manual/release-notes/rl-1.6.1.xml index 1ca4dfebacdc..9ecc52734737 100644 --- a/doc/manual/release-notes/rl-1.6.1.xml +++ b/doc/manual/release-notes/rl-1.6.1.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.6.1"> -<title>Release 1.6.1 (October 28, 2013)</title> +<title>Release 1.6.1 (2013-10-28)</title> <para>This is primarily a bug fix release. Changes of interest are:</para> @@ -66,4 +66,4 @@ are:</para> </itemizedlist> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.6.xml b/doc/manual/release-notes/rl-1.6.xml index a3d61faf2b73..580563420949 100644 --- a/doc/manual/release-notes/rl-1.6.xml +++ b/doc/manual/release-notes/rl-1.6.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.6.0"> -<title>Release 1.6 (September 10, 2013)</title> +<title>Release 1.6 (2013-09-10)</title> <para>In addition to the usual bug fixes, this release has several new features:</para> @@ -124,4 +124,4 @@ in pkgs.bar Florian Friesdorf, Gergely Risko, Ivan Kozik, Ludovic Courtès and Shea Levy.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.7.xml b/doc/manual/release-notes/rl-1.7.xml index a9863b99cba0..44ecaa78da5f 100644 --- a/doc/manual/release-notes/rl-1.7.xml +++ b/doc/manual/release-notes/rl-1.7.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.7"> -<title>Release 1.7 (April 11, 2014)</title> +<title>Release 1.7 (2014-04-11)</title> <para>In addition to the usual bug fixes, this release has the following new features:</para> @@ -260,4 +260,4 @@ error: attribute `nixUnstabl' missing, at /etc/nixos/configurations/misc/eelco/m Eelco Dolstra, Ian-Woo Kim, Ludovic Courtès, Maxim Ivanov, Petr Rockai, Ricardo M. Correia and Shea Levy.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml index e551ee06055f..48caac2c6b60 100644 --- a/doc/manual/release-notes/rl-1.8.xml +++ b/doc/manual/release-notes/rl-1.8.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.8"> -<title>Release 1.8 (December 14, 2014)</title> +<title>Release 1.8 (2014-12-14)</title> <itemizedlist> @@ -60,7 +60,7 @@ $ nix-store -l $(which xterm) <listitem><para><command>nix-copy-closure</command> now uses <command>nix-store --serve</command> on the remote side to send or receive closures. This fixes a race condition between - <command>nix-copy-closureE</command> and the garbage + <command>nix-copy-closure</command> and the garbage collector.</para></listitem> <listitem><para>Derivations can specify the new special attribute diff --git a/doc/manual/release-notes/rl-1.9.xml b/doc/manual/release-notes/rl-1.9.xml index 9987226ff769..c8406bd2077c 100644 --- a/doc/manual/release-notes/rl-1.9.xml +++ b/doc/manual/release-notes/rl-1.9.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-1.9"> -<title>Release 1.9 (June 12, 2015)</title> +<title>Release 1.9 (2015-06-12)</title> <para>In addition to the usual bug fixes, this release has the following new features:</para> @@ -75,7 +75,7 @@ $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/ <listitem><para>In <command>nixos-rebuild</command> (on NixOS): <screen> -$ nixos-rebuild test -I nixpkgs=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz +$ nixos-rebuild test -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz </screen> </para></listitem> diff --git a/doc/signing.txt b/doc/signing.txt deleted file mode 100644 index 7403cac470b2..000000000000 --- a/doc/signing.txt +++ /dev/null @@ -1,24 +0,0 @@ -Generate a private key: - -$ (umask 277 && openssl genrsa -out /etc/nix/signing-key.sec 2048) - -The private key should be kept secret (only readable to the Nix daemon -user). - - -Generate the corresponding public key: - -$ openssl rsa -in /etc/nix/signing-key.sec -pubout > /etc/nix/signing-key.pub - -The public key should be copied to all machines to which you want to -export store paths. - - -Signing: - -$ nix-hash --type sha256 --flat svn.nar | openssl rsautl -sign -inkey mykey.sec > svn.nar.sign - - -Verifying a signature: - -$ test "$(nix-hash --type sha256 --flat svn.nar)" = "$(openssl rsautl -verify -inkey mykey.pub -pubin -in svn.nar.sign)" diff --git a/local.mk b/local.mk index 25957d92a659..2541f3f32290 100644 --- a/local.mk +++ b/local.mk @@ -1,12 +1,14 @@ ifeq ($(MAKECMDGOALS), dist) - dist-files += $(shell git ls-files) + # Make sure we are in repo root with `--git-dir` + dist-files += $(shell git --git-dir=.git ls-files || find * -type f) endif dist-files += configure config.h.in nix.spec clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr +GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr \ + -Wno-unneeded-internal-declaration $(foreach i, config.h $(call rwildcard, src/lib*, *.hh) src/nix-store/serve-protocol.hh, \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile new file mode 100644 index 000000000000..bfa0c34d0947 --- /dev/null +++ b/misc/docker/Dockerfile @@ -0,0 +1,20 @@ +FROM alpine + +RUN wget -O- http://nixos.org/releases/nix/nix-1.11.2/nix-1.11.2-x86_64-linux.tar.bz2 | bzcat - | tar xf - \ + && echo "nixbld:x:30000:nixbld1,nixbld2,nixbld3,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld30" >> /etc/group \ + && for i in $(seq 1 30); do echo "nixbld$i:x:$((30000 + $i)):30000:::" >> /etc/passwd; done \ + && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \ + && echo ". /root/.nix-profile/etc/profile.d/nix.sh" >> /etc/profile \ + && rm -r /nix-*-x86_64-linux + +ONBUILD ENV \ + ENV=/etc/profile \ + PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ + GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ + SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt + +ENV \ + ENV=/etc/profile \ + PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ + GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ + SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt diff --git a/misc/emacs/nix-mode.el b/misc/emacs/nix-mode.el index 790799d858cc..e129e9efe1d4 100644 --- a/misc/emacs/nix-mode.el +++ b/misc/emacs/nix-mode.el @@ -8,17 +8,31 @@ ;;; Code: +(defun nix-syntax-match-antiquote (limit) + (let ((pos (next-single-char-property-change (point) 'nix-syntax-antiquote + nil limit))) + (when (and pos (> pos (point))) + (goto-char pos) + (let ((char (char-after pos))) + (pcase char + (`?$ + (forward-char 2)) + (`?} + (forward-char 1))) + (set-match-data (list pos (point))) + t)))) + (defconst nix-font-lock-keywords - '("\\<if\\>" "\\<then\\>" "\\<else\\>" "\\<assert\\>" "\\<with\\>" - "\\<let\\>" "\\<in\\>" "\\<rec\\>" "\\<inherit\\>" "\\<or\\>" - ("\\<true\\>" . font-lock-builtin-face) - ("\\<false\\>" . font-lock-builtin-face) - ("\\<null\\>" . font-lock-builtin-face) - ("\\<import\\>" . font-lock-builtin-face) - ("\\<derivation\\>" . font-lock-builtin-face) - ("\\<baseNameOf\\>" . font-lock-builtin-face) - ("\\<toString\\>" . font-lock-builtin-face) - ("\\<isNull\\>" . font-lock-builtin-face) + '("\\_<if\\_>" "\\_<then\\_>" "\\_<else\\_>" "\\_<assert\\_>" "\\_<with\\_>" + "\\_<let\\_>" "\\_<in\\_>" "\\_<rec\\_>" "\\_<inherit\\_>" "\\_<or\\_>" + ("\\_<true\\_>" . font-lock-builtin-face) + ("\\_<false\\_>" . font-lock-builtin-face) + ("\\_<null\\_>" . font-lock-builtin-face) + ("\\_<import\\_>" . font-lock-builtin-face) + ("\\_<derivation\\_>" . font-lock-builtin-face) + ("\\_<baseNameOf\\_>" . font-lock-builtin-face) + ("\\_<toString\\_>" . font-lock-builtin-face) + ("\\_<isNull\\_>" . font-lock-builtin-face) ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+" . font-lock-constant-face) ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\-\.]*\\)[ \t]*=" @@ -26,7 +40,8 @@ ("<[a-zA-Z0-9._\\+-]+\\(/[a-zA-Z0-9._\\+-]+\\)*>" . font-lock-constant-face) ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+" - . font-lock-constant-face)) + . font-lock-constant-face) + (nix-syntax-match-antiquote 0 font-lock-preprocessor-face t)) "Font lock keywords for nix.") (defvar nix-mode-syntax-table @@ -38,6 +53,67 @@ table) "Syntax table for Nix mode.") +(defun nix-syntax-propertize-escaped-antiquote () + "Set syntax properies for escaped antiquote marks." + nil) + +(defun nix-syntax-propertize-multiline-string () + "Set syntax properies for multiline string delimiters." + (let* ((start (match-beginning 0)) + (end (match-end 0)) + (context (save-excursion (save-match-data (syntax-ppss start)))) + (string-type (nth 3 context))) + (pcase string-type + (`t + ;; inside a multiline string + ;; ending multi-line string delimiter + (put-text-property (1- end) end + 'syntax-table (string-to-syntax "|"))) + (`nil + ;; beginning multi-line string delimiter + (put-text-property start (1+ start) + 'syntax-table (string-to-syntax "|")))))) + +(defun nix-syntax-propertize-antiquote () + "Set syntax properties for antiquote marks." + (let* ((start (match-beginning 0))) + (put-text-property start (1+ start) + 'syntax-table (string-to-syntax "|")) + (put-text-property start (+ start 2) + 'nix-syntax-antiquote t))) + +(defun nix-syntax-propertize-close-brace () + "Set syntax properties for close braces. +If a close brace `}' ends an antiquote, the next character begins a string." + (let* ((start (match-beginning 0)) + (end (match-end 0)) + (context (save-excursion (save-match-data (syntax-ppss start)))) + (open (nth 1 context))) + (when open ;; a corresponding open-brace was found + (let* ((antiquote (get-text-property open 'nix-syntax-antiquote))) + (when antiquote + (put-text-property (+ start 1) (+ start 2) + 'syntax-table (string-to-syntax "|")) + (put-text-property start (1+ start) + 'nix-syntax-antiquote t)))))) + +(defun nix-syntax-propertize (start end) + "Special syntax properties for Nix." + ;; search for multi-line string delimiters + (goto-char start) + (remove-text-properties start end '(syntax-table nil nix-syntax-antiquote nil)) + (funcall + (syntax-propertize-rules + ("''\\${" + (0 (ignore (nix-syntax-propertize-escaped-antiquote)))) + ("''" + (0 (ignore (nix-syntax-propertize-multiline-string)))) + ("\\${" + (0 (ignore (nix-syntax-propertize-antiquote)))) + ("}" + (0 (ignore (nix-syntax-propertize-close-brace))))) + start end)) + (defun nix-indent-line () "Indent current line in a Nix expression." (interactive) @@ -69,7 +145,13 @@ The hook `nix-mode-hook' is run when Nix mode is started. (set-syntax-table nix-mode-syntax-table) ;; Font lock support. - (setq font-lock-defaults '(nix-font-lock-keywords nil nil nil nil)) + (setq-local font-lock-defaults '(nix-font-lock-keywords nil nil nil nil)) + + ;; Special syntax properties for Nix + (setq-local syntax-propertize-function 'nix-syntax-propertize) + + ;; Look at text properties when parsing + (setq-local parse-sexp-lookup-properties t) ;; Automatic indentation [C-j]. (set (make-local-variable 'indent-line-function) 'nix-indent-line) diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon new file mode 100755 index 000000000000..fea53716721d --- /dev/null +++ b/misc/systemv/nix-daemon @@ -0,0 +1,113 @@ +#!/bin/sh +# +# nix-daemon: Starts the nix package manager daemon +# +# chkconfig: 345 24 02 +# description: This is a daemon which enable the multi-user mode +# of the nix package manager. +# processname: nix-daemon +# pidfile: /var/run/nix/nix-daemon.pid + +### BEGIN INIT INFO +# Required-Start: +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 3 4 5 +# Default-Stop: 0 1 2 6 +# Short-Description: Starts the nix daemon +# Description: This is a daemon which enable the multi-user mode +# of the nix package manager. +### END INIT INFO + +NIX_DAEMON_BIN=/usr/bin/nix-daemon +#NIX_DAEMON_USER="root" +NIX_DAEMON_USER="nix-daemon" +NIX_DAEMON_OPTS="--daemon" + +umask 0022 + +if [ "$1" = 'status' ]; then + test -x $NIX_DAEMON_BIN || exit 4 +else + test -x $NIX_DAEMON_BIN || exit 5 +fi + +# Source function library. +. /etc/init.d/functions + +LOCKFILE=/var/lock/subsys/nix-daemon +RUNDIR=/var/run/nix +PIDFILE=${RUNDIR}/nix-daemon.pid +RETVAL=0 + +base=${0##*/} + +start() { + + mkdir -p ${RUNDIR} + chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} + + echo -n $"Starting nix daemon... " + + daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch ${LOCKFILE} + return $RETVAL +} + +stop() { + echo -n $"Shutting down nix daemon: " + killproc -p ${PIDFILE} $NIX_DAEMON_BIN + RETVAL=$? + [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} + echo + return $RETVAL +} + +reload() { + echo -n $"Reloading nix daemon... " + killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP + RETVAL=$? + echo + return $RETVAL +} + +restart() { + stop + start +} + +RETVAL=0 + +# caller switch +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status -p ${PIDFILE} $NIX_DAEMON_BIN + RETVAL=$? + ;; + restart) + restart + ;; + reload) + reload + ;; + condrestart) + if [ -f $LOCKFILE ]; then + restart + fi + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 2 + ;; +esac + +exit $RETVAL diff --git a/mk/lib.mk b/mk/lib.mk index 4ad5c636c8d4..bb82801d3b4e 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -61,7 +61,9 @@ ifeq ($(BUILD_SHARED_LIBS), 1) endif ifneq ($(OS), Darwin) ifneq ($(OS), SunOS) - GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries + ifneq ($(OS), FreeBSD) + GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries + endif endif endif SET_RPATH_TO_LIBS ?= 1 diff --git a/nix.spec.in b/nix.spec.in index 4dbfdfa09bc0..dff8efbdcbed 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -26,6 +26,7 @@ Requires: gzip Requires: xz BuildRequires: bzip2-devel BuildRequires: sqlite-devel +BuildRequires: libcurl-devel # Hack to make that shitty RPM scanning hack shut up. Provides: perl(Nix::SSH) @@ -162,7 +163,7 @@ systemctl start nix-daemon.socket %endif %files -%{_bindir}/nix-* +%{_bindir}/nix* %{_libdir}/*.so %{perl_vendorarch}/* %exclude %dir %{perl_vendorarch}/auto/ diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index b0dc71fab377..3575d99cb671 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -7,7 +7,6 @@ $version = "@PACKAGE_VERSION@"; $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; $libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@"; $stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix"; -$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests"; $logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@"; @@ -15,7 +14,6 @@ $storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@"; $bzip2 = "@bzip2@"; $xz = "@xz@"; $curl = "@curl@"; -$openssl = "@openssl@"; $useBindings = "@perlbindings@" eq "yes"; diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 800feb3b413a..1adce07a4ded 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -10,7 +10,7 @@ use IPC::Open2; sub copyToOpen { - my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_; + my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; $useSubstitutes = 0 if $dryRun || !defined $useSubstitutes; @@ -41,13 +41,13 @@ sub copyToOpen { # Send the "import paths" command. syswrite($to, pack("L<x4", 4)) or die; - exportPaths(fileno($to), $sign, @missing); + exportPaths(fileno($to), @missing); readInt($from) == 1 or die "remote machine ‘$sshHost’ failed to import closure\n"; } sub copyTo { - my ($sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_; + my ($sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; # Connect to the remote host. my ($from, $to); @@ -61,7 +61,7 @@ sub copyTo { return oldCopyTo(@_); } - copyToOpen($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes); + copyToOpen($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes); close $to; } @@ -70,7 +70,7 @@ sub copyTo { # For backwards compatibility with Nix <= 1.7. Will be removed # eventually. sub oldCopyTo { - my ($sshHost, $storePaths, $includeOutputs, $dryRun, $sign, $useSubstitutes) = @_; + my ($sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; # Get the closure of this path. my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, @@ -105,7 +105,7 @@ sub oldCopyTo { print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; unless ($dryRun) { open SSH, "| ssh $sshHost @globalSshOpts 'nix-store --import' > /dev/null" or die; - exportPaths(fileno(SSH), $sign, @missing); + exportPaths(fileno(SSH), @missing); close SSH or die "copying store paths to remote machine ‘$sshHost’ failed: $?"; } } diff --git a/perl/lib/Nix/GeneratePatches.pm b/perl/lib/Nix/GeneratePatches.pm deleted file mode 100644 index 612c8a3a15ba..000000000000 --- a/perl/lib/Nix/GeneratePatches.pm +++ /dev/null @@ -1,340 +0,0 @@ -package Nix::GeneratePatches; - -use strict; -use File::Temp qw(tempdir); -use File::stat; -use Nix::Config; -use Nix::Manifest; - -our @ISA = qw(Exporter); -our @EXPORT = qw(generatePatches propagatePatches copyPatches); - - -# Some patch generations options. - -# Max size of NAR archives to generate patches for. -my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"}; -$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize; - -# If patch is bigger than this fraction of full archive, reject. -my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; -$maxPatchFraction = 0.60 if !defined $maxPatchFraction; - -my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; -$timeLimit = 180 if !defined $timeLimit; - -my $hashAlgo = "sha256"; - - -sub findOutputPaths { - my $narFiles = shift; - - my %outPaths; - - foreach my $p (keys %{$narFiles}) { - - # Ignore derivations. - next if ($p =~ /\.drv$/); - - # Ignore builders (too much ambiguity -- they're all called - # `builder.sh'). - next if ($p =~ /\.sh$/); - next if ($p =~ /\.patch$/); - - # Don't bother including tar files etc. - next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); - - $outPaths{$p} = 1; - } - - return %outPaths; -} - - -sub getNameVersion { - my $p = shift; - $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; - my $name = $1; - my $version = $2; - return undef unless defined $name && defined $version; - $name =~ s/^-//; - $version =~ s/^-//; - return ($name, $version); -} - - -# A quick hack to get a measure of the `distance' between two -# versions: it's just the position of the first character that differs -# (or 999 if they are the same). -sub versionDiff { - my $s = shift; - my $t = shift; - my $i; - return 999 if $s eq $t; - for ($i = 0; $i < length $s; $i++) { - return $i if $i >= length $t or - substr($s, $i, 1) ne substr($t, $i, 1); - } - return $i; -} - - -sub getNarBz2 { - my $narPath = shift; - my $narFiles = shift; - my $storePath = shift; - - my $narFileList = $$narFiles{$storePath}; - die "missing path $storePath" unless defined $narFileList; - - my $narFile = @{$narFileList}[0]; - die unless defined $narFile; - - $narFile->{url} =~ /\/([^\/]+)$/; - die unless defined $1; - return "$narPath/$1"; -} - - -sub containsPatch { - my $patches = shift; - my $storePath = shift; - my $basePath = shift; - my $patchList = $$patches{$storePath}; - return 0 if !defined $patchList; - my $found = 0; - foreach my $patch (@{$patchList}) { - # !!! baseHash might differ - return 1 if $patch->{basePath} eq $basePath; - } - return 0; -} - - -sub generatePatches { - my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_; - - my %srcOutPaths = findOutputPaths $srcNarFiles; - my %dstOutPaths = findOutputPaths $dstNarFiles; - - # For each output path in the destination, see if we need to / can - # create a patch. - - print STDERR "creating patches...\n"; - - foreach my $p (keys %dstOutPaths) { - - # If exactly the same path already exists in the source, skip it. - next if defined $srcOutPaths{$p}; - - print " $p\n"; - - # If not, then we should find the paths in the source that are - # `most' likely to be present on a system that wants to - # install this path. - - (my $name, my $version) = getNameVersion $p; - next unless defined $name && defined $version; - - my @closest = (); - my $closestVersion; - my $minDist = -1; # actually, larger means closer - - # Find all source paths with the same name. - - foreach my $q (keys %srcOutPaths) { - (my $name2, my $version2) = getNameVersion $q; - next unless defined $name2 && defined $version2; - - if ($name eq $name2) { - - my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system}; - my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system}; - if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { - print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; - next; - } - - # If the sizes differ too much, then skip. This - # disambiguates between, e.g., a real component and a - # wrapper component (cf. Firefox in Nixpkgs). - my $srcSize = @{$$srcNarFiles{$q}}[0]->{size}; - my $dstSize = @{$$dstNarFiles{$p}}[0]->{size}; - my $ratio = $srcSize / $dstSize; - $ratio = 1 / $ratio if $ratio < 1; - # print " SIZE $srcSize $dstSize $ratio $q\n"; - - if ($ratio >= 3) { - print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; - next; - } - - # If there are multiple matching names, include the - # ones with the closest version numbers. - my $dist = versionDiff $version, $version2; - if ($dist > $minDist) { - $minDist = $dist; - @closest = ($q); - $closestVersion = $version2; - } elsif ($dist == $minDist) { - push @closest, $q; - } - } - } - - if (scalar(@closest) == 0) { - print " NO BASE: $p\n"; - next; - } - - foreach my $closest (@closest) { - - # Generate a patch between $closest and $p. - print STDERR " $p <- $closest\n"; - - # If the patch already exists, skip it. - if (containsPatch($srcPatches, $p, $closest) || - containsPatch($dstPatches, $p, $closest)) - { - print " skipping, already exists\n"; - next; - } - - my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest; - my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p; - - if (! -f $srcNarBz2) { - warn "patch source archive $srcNarBz2 is missing\n"; - next; - } - - system("$Nix::Config::bzip2 -d < $srcNarBz2 > $tmpDir/A") == 0 - or die "cannot unpack $srcNarBz2"; - - if (stat("$tmpDir/A")->size >= $maxNarSize) { - print " skipping, source is too large\n"; - next; - } - - system("$Nix::Config::bzip2 -d < $dstNarBz2 > $tmpDir/B") == 0 - or die "cannot unpack $dstNarBz2"; - - if (stat("$tmpDir/B")->size >= $maxNarSize) { - print " skipping, destination is too large\n"; - next; - } - - my $time1 = time(); - my $res = system("ulimit -t $timeLimit; $Nix::Config::libexecDir/nix/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); - my $time2 = time(); - if ($res) { - warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; - next; - } - - my $baseHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; - chomp $baseHash; - - my $narHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die; - chomp $narHash; - - my $narDiffHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die; - chomp $narDiffHash; - - my $narDiffSize = stat("$tmpDir/DIFF")->size; - my $dstNarBz2Size = stat($dstNarBz2)->size; - - print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; - - if ($narDiffSize >= $dstNarBz2Size) { - print " rejecting; patch bigger than full archive\n"; - next; - } - - if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) { - print " rejecting; patch too large relative to full archive\n"; - next; - } - - my $finalName = "$narDiffHash.nar-bsdiff"; - - if (-e "$patchesPath/$finalName") { - print " not copying, already exists\n"; - } - - else { - system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0 - or die "cannot copy diff"; - rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName") - or die "cannot rename $patchesPath/$finalName.tmp"; - } - - # Add the patch to the manifest. - addPatch $dstPatches, $p, - { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash" - , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash" - , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff" - }; - } - } -} - - -# Propagate useful patches from $srcPatches to $dstPatches. A patch -# is useful if it produces either paths in the $dstNarFiles or paths -# that can be used as the base for other useful patches. -sub propagatePatches { - my ($srcPatches, $dstNarFiles, $dstPatches) = @_; - - print STDERR "propagating patches...\n"; - - my $changed; - do { - # !!! we repeat this to reach the transitive closure; inefficient - $changed = 0; - - print STDERR "loop\n"; - - my %dstBasePaths; - foreach my $q (keys %{$dstPatches}) { - foreach my $patch (@{$$dstPatches{$q}}) { - $dstBasePaths{$patch->{basePath}} = 1; - } - } - - foreach my $p (keys %{$srcPatches}) { - my $patchList = $$srcPatches{$p}; - - my $include = 0; - - # Is path $p included in the destination? If so, include - # patches that produce it. - $include = 1 if defined $$dstNarFiles{$p}; - - # Is path $p a path that serves as a base for paths in the - # destination? If so, include patches that produce it. - # !!! check baseHash - $include = 1 if defined $dstBasePaths{$p}; - - if ($include) { - foreach my $patch (@{$patchList}) { - $changed = 1 if addPatch $dstPatches, $p, $patch; - } - } - - } - - } while $changed; -} - - -# Add all new patches in $srcPatches to $dstPatches. -sub copyPatches { - my ($srcPatches, $dstPatches) = @_; - foreach my $p (keys %{$srcPatches}) { - addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}}; - } -} - - -return 1; diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index 428decf09b54..0da376761201 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -13,7 +13,7 @@ use Nix::Config; use Nix::Store; our @ISA = qw(Exporter); -our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo fingerprintPath); +our @EXPORT = qw(readManifest writeManifest addPatch parseNARInfo fingerprintPath); sub addNAR { @@ -228,172 +228,6 @@ sub writeManifest { } -sub updateManifestDB { - my $manifestDir = $Nix::Config::manifestDir; - - my @manifests = glob "$manifestDir/*.nixmanifest"; - return undef if scalar @manifests == 0; - - mkpath($manifestDir); - - unlink "$manifestDir/cache.sqlite"; # remove obsolete cache - my $dbPath = "$manifestDir/cache-v2.sqlite"; - - # Open/create the database. - our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "") - or die "cannot open database ‘$dbPath’"; - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - $dbh->do("pragma foreign_keys = on"); - $dbh->do("pragma synchronous = off"); # we can always reproduce the cache - $dbh->do("pragma journal_mode = truncate"); - - # Initialise the database schema, if necessary. - $dbh->do(<<EOF); - create table if not exists Manifests ( - id integer primary key autoincrement not null, - path text unique not null, - timestamp integer not null - ); -EOF - - $dbh->do(<<EOF); - create table if not exists NARs ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - url text not null, - compressionType text not null, - hash text, - size integer, - narHash text, - narSize integer, - refs text, - deriver text, - system text, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists NARs_storePath on NARs(storePath)"); - - $dbh->do(<<EOF); - create table if not exists Patches ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - basePath text not null, - baseHash text not null, - url text not null, - hash text, - size integer, - narHash text, - narSize integer, - patchType text not null, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists Patches_storePath on Patches(storePath)"); - - # Acquire an exclusive lock to ensure that only one process - # updates the DB at the same time. This isn't really necessary, - # but it prevents work duplication and lock contention in SQLite. - my $lockFile = "$manifestDir/cache.lock"; - open MAINLOCK, ">>$lockFile" or die "unable to acquire lock ‘$lockFile’: $!\n"; - flock(MAINLOCK, LOCK_EX) or die; - - our $insertNAR = $dbh->prepare( - "insert into NARs(manifest, storePath, url, compressionType, hash, size, narHash, " . - "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die; - - our $insertPatch = $dbh->prepare( - "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . - "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - - $dbh->begin_work; - - # Read each manifest in $manifestDir and add it to the database, - # unless we've already done so on a previous run. - my %seen; - - for my $manifestLink (@manifests) { - my $manifest = Cwd::abs_path($manifestLink); - next unless -f $manifest; - my $timestamp = lstat($manifest)->mtime; - $seen{$manifest} = 1; - - next if scalar @{$dbh->selectcol_arrayref( - "select 1 from Manifests where path = ? and timestamp = ?", - {}, $manifest, $timestamp)} == 1; - - print STDERR "caching $manifest...\n"; - - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - - $dbh->do("insert into Manifests(path, timestamp) values (?, ?)", - {}, $manifest, $timestamp); - - our $id = $dbh->last_insert_id("", "", "", ""); - - sub addNARToDB { - my ($storePath, $narFile) = @_; - $insertNAR->execute( - $id, $storePath, $narFile->{url}, $narFile->{compressionType}, $narFile->{hash}, - $narFile->{size}, $narFile->{narHash}, $narFile->{narSize}, $narFile->{references}, - $narFile->{deriver}, $narFile->{system}); - }; - - sub addPatchToDB { - my ($storePath, $patch) = @_; - $insertPatch->execute( - $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url}, - $patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize}, - $patch->{patchType}); - }; - - my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB); - - if ($version < 3) { - die "you have an old-style or corrupt manifest ‘$manifestLink’; please delete it\n"; - } - if ($version >= 10) { - die "manifest ‘$manifestLink’ is too new; please delete it or upgrade Nix\n"; - } - } - - # Removed cached information for removed manifests from the DB. - foreach my $manifest (@{$dbh->selectcol_arrayref("select path from Manifests")}) { - next if defined $seen{$manifest}; - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - } - - $dbh->commit; - - close MAINLOCK; - - return $dbh; -} - - -# Delete all old manifests downloaded from a given URL. -sub deleteOldManifests { - my ($url, $curUrlFile) = @_; - for my $urlFile (glob "$Nix::Config::manifestDir/*.url") { - next if defined $curUrlFile && $urlFile eq $curUrlFile; - open URL, "<$urlFile" or die; - my $url2 = <URL>; - chomp $url2; - close URL; - next unless $url eq $url2; - my $base = $urlFile; $base =~ s/.url$//; - unlink "${base}.url"; - unlink "${base}.nixmanifest"; - } -} - - # Return a fingerprint of a store path to be used in binary cache # signatures. It contains the store path, the base-32 SHA-256 hash of # the contents of the path, and the references. diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index af3d2fa2e719..d226264d4df3 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -21,6 +21,7 @@ our @EXPORT = qw( signString checkSignature addToStore makeFixedOutputPath derivationFromPath + addTempRoot ); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index d3bfa19fd846..ee60ce13011d 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -6,10 +6,11 @@ #undef do_open #undef do_close -#include <store-api.hh> -#include <globals.hh> -#include <misc.hh> -#include <util.hh> +#include "derivations.hh" +#include "globals.hh" +#include "store-api.hh" +#include "util.hh" +#include "crypto.hh" #if HAVE_SODIUM #include <sodium.h> @@ -19,19 +20,22 @@ using namespace nix; -void doInit() +static ref<Store> store() { - if (!store) { + static std::shared_ptr<Store> _store; + if (!_store) { try { + logger = makeDefaultLogger(); settings.processEnvironment(); settings.loadConfFile(); settings.update(); settings.lockCPU = false; - store = openStore(); + _store = openStore(); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } } + return ref<Store>(_store); } @@ -45,7 +49,7 @@ PROTOTYPES: ENABLE void init() CODE: - doInit(); + store(); void setVerbosity(int level) @@ -56,10 +60,9 @@ void setVerbosity(int level) int isValidPath(char * path) CODE: try { - doInit(); - RETVAL = store->isValidPath(path); + RETVAL = store()->isValidPath(path); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } OUTPUT: RETVAL @@ -68,134 +71,124 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - doInit(); - PathSet paths; - store->queryReferences(path, paths); + PathSet paths = store()->queryPathInfo(path)->references; for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * queryPathHash(char * path) PPCODE: try { - doInit(); - Hash hash = store->queryPathHash(path); + auto hash = store()->queryPathInfo(path)->narHash; string s = "sha256:" + printHash32(hash); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * queryDeriver(char * path) PPCODE: try { - doInit(); - Path deriver = store->queryDeriver(path); + auto deriver = store()->queryPathInfo(path)->deriver; if (deriver == "") XSRETURN_UNDEF; XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * queryPathInfo(char * path, int base32) PPCODE: try { - doInit(); - ValidPathInfo info = store->queryPathInfo(path); - if (info.deriver == "") + auto info = store()->queryPathInfo(path); + if (info->deriver == "") XPUSHs(&PL_sv_undef); else - XPUSHs(sv_2mortal(newSVpv(info.deriver.c_str(), 0))); - string s = "sha256:" + (base32 ? printHash32(info.hash) : printHash(info.hash)); + XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0))); + string s = "sha256:" + (base32 ? printHash32(info->narHash) : printHash(info->narHash)); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - mXPUSHi(info.registrationTime); - mXPUSHi(info.narSize); + mXPUSHi(info->registrationTime); + mXPUSHi(info->narSize); AV * arr = newAV(); - for (PathSet::iterator i = info.references.begin(); i != info.references.end(); ++i) + for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i) av_push(arr, newSVpv(i->c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * queryPathFromHashPart(char * hashPart) PPCODE: try { - doInit(); - Path path = store->queryPathFromHashPart(hashPart); + Path path = store()->queryPathFromHashPart(hashPart); XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { - doInit(); PathSet paths; for (int n = 2; n < items; ++n) - computeFSClosure(*store, SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs); + store()->computeFSClosure(SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs); for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * topoSortPaths(...) PPCODE: try { - doInit(); PathSet paths; for (int n = 0; n < items; ++n) paths.insert(SvPV_nolen(ST(n))); - Paths sorted = topoSortPaths(*store, paths); + Paths sorted = store()->topoSortPaths(paths); for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i) XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * followLinksToStorePath(char * path) CODE: try { - doInit(); + store(); RETVAL = newSVpv(followLinksToStorePath(path).c_str(), 0); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } OUTPUT: RETVAL -void exportPaths(int fd, int sign, ...) +void exportPaths(int fd, ...) PPCODE: try { - doInit(); Paths paths; - for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); + for (int n = 1; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); FdSink sink(fd); - exportPaths(*store, paths, sign, sink); + store()->exportPaths(paths, sink); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } void importPaths(int fd) PPCODE: try { - doInit(); FdSource source(fd); - store->importPaths(false, source); + store()->importPaths(source, 0); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } @@ -206,7 +199,7 @@ SV * hashPath(char * algo, int base32, char * path) string s = base32 ? printHash32(h) : printHash(h); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } @@ -217,7 +210,7 @@ SV * hashFile(char * algo, int base32, char * path) string s = base32 ? printHash32(h) : printHash(h); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } @@ -228,7 +221,7 @@ SV * hashString(char * algo, int base32, char * s) string s = base32 ? printHash32(h) : printHash(h); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } @@ -239,28 +232,21 @@ SV * convertHash(char * algo, char * s, int toBase32) string s = toBase32 ? printHash32(h) : printHash(h); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } -SV * signString(SV * secretKey_, char * msg) +SV * signString(char * secretKey_, char * msg) PPCODE: try { #if HAVE_SODIUM - STRLEN secretKeyLen; - unsigned char * secretKey = (unsigned char *) SvPV(secretKey_, secretKeyLen); - if (secretKeyLen != crypto_sign_SECRETKEYBYTES) - throw Error("secret key is not valid"); - - unsigned char sig[crypto_sign_BYTES]; - unsigned long long sigLen; - crypto_sign_detached(sig, &sigLen, (unsigned char *) msg, strlen(msg), secretKey); - XPUSHs(sv_2mortal(newSVpv((char *) sig, sigLen))); + auto sig = SecretKey(secretKey_).signDetached(msg); + XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); #else throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); #endif } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } @@ -283,7 +269,7 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); #endif } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } OUTPUT: RETVAL @@ -292,24 +278,22 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - doInit(); - Path path = store->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo)); + Path path = store()->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo)); XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - doInit(); HashType ht = parseHashType(algo); Path path = makeFixedOutputPath(recursive, ht, parseHash16or32(ht, hash), name); XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } @@ -318,8 +302,7 @@ SV * derivationFromPath(char * drvPath) HV *hash; CODE: try { - doInit(); - Derivation drv = derivationFromPath(*store, drvPath); + Derivation drv = store()->derivationFromPath(drvPath); hash = newHV(); HV * outputs = newHV(); @@ -352,7 +335,16 @@ SV * derivationFromPath(char * drvPath) RETVAL = newRV_noinc((SV *)hash); } catch (Error & e) { - croak(e.what()); + croak("%s", e.what()); } OUTPUT: RETVAL + + +void addTempRoot(char * storePath) + PPCODE: + try { + store()->addTempRoot(storePath); + } catch (Error & e) { + croak("%s", e.what()); + } diff --git a/perl/local.mk b/perl/local.mk index 132676f53341..5b43c4b717fd 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -1,7 +1,6 @@ nix_perl_sources := \ $(d)/lib/Nix/Store.pm \ $(d)/lib/Nix/Manifest.pm \ - $(d)/lib/Nix/GeneratePatches.pm \ $(d)/lib/Nix/SSH.pm \ $(d)/lib/Nix/CopyClosure.pm \ $(d)/lib/Nix/Config.pm.in \ @@ -24,7 +23,9 @@ ifeq ($(perlbindings), yes) Store_CXXFLAGS = \ -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 -Wno-unused-variable -Wno-literal-suffix -Wno-reserved-user-defined-literal + -D_FILE_OFFSET_BITS=64 \ + -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ + -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion Store_LIBS = libstore libutil diff --git a/release.nix b/release.nix index cfec7b9e774c..4803788fe009 100644 --- a/release.nix +++ b/release.nix @@ -1,4 +1,5 @@ { nix ? { outPath = ./.; revCount = 1234; shortRev = "abcdef"; } +, nixpkgs ? { outPath = <nixpkgs>; revCount = 1234; shortRev = "abcdef"; } , officialRelease ? false }: @@ -23,9 +24,10 @@ let inherit officialRelease; buildInputs = - [ curl bison flex perl libxml2 libxslt bzip2 - tetex dblatex nukeReferences pkgconfig sqlite libsodium + [ curl bison flex perl libxml2 libxslt bzip2 xz + pkgconfig sqlite libsodium docbook5 docbook5_xsl + autoconf-archive ] ++ lib.optional (!lib.inNixShell) git; configureFlags = '' @@ -36,7 +38,9 @@ let postUnpack = '' # Clean up when building from a working tree. - (cd $sourceRoot && (git ls-files -o | xargs -r rm -v)) + if [[ -d $sourceRoot/.git ]]; then + git -C $sourceRoot clean -fd + fi ''; preConfigure = '' @@ -54,20 +58,7 @@ let preDist = '' make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk - - make doc/manual/manual.pdf - cp doc/manual/manual.pdf $out/manual.pdf - - # The PDF containes filenames of included graphics (see - # http://www.tug.org/pipermail/pdftex/2007-August/007290.html). - # This causes a retained dependency on dblatex, which Hydra - # doesn't like (the output of the tarball job is distributed - # to Windows and Macs, so there should be no Linux binaries - # in the closure). - nuke-refs $out/manual.pdf - echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products - echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products ''; }; @@ -81,8 +72,13 @@ let src = tarball; buildInputs = - [ curl perl bzip2 openssl pkgconfig sqlite boehmgc ] - ++ lib.optional stdenv.isLinux libsodium; + [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ] + ++ lib.optional stdenv.isLinux libsodium + ++ lib.optional stdenv.isLinux + (aws-sdk-cpp.override { + apis = ["s3"]; + customMemoryManagement = false; + }); configureFlags = '' --disable-init-state @@ -108,6 +104,7 @@ let binaryTarball = pkgs.lib.genAttrs systems (system: + # FIXME: temporarily use a different branch for the Darwin build. with import <nixpkgs> { inherit system; }; let @@ -150,7 +147,7 @@ let src = tarball; buildInputs = - [ curl perl bzip2 openssl pkgconfig sqlite + [ curl perl bzip2 openssl pkgconfig sqlite xz libsodium # These are for "make check" only: graphviz libxml2 libxslt ]; @@ -175,8 +172,6 @@ let }; - rpm_fedora18i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora18i386) []; - rpm_fedora18x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora18x86_64) []; rpm_fedora19i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora19i386) []; rpm_fedora19x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora19x86_64) []; rpm_fedora20i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora20i386) []; @@ -185,23 +180,21 @@ let rpm_fedora21x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora21x86_64) [ "libsodium-devel" ]; - deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386) []; - deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64) []; - deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ]; - deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ]; + deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium18" ]; + deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; - deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386) []; - deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64) []; - deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386) []; - deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64) []; - deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) []; - deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) []; - deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386) []; - deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64) []; - deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) []; - deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) []; - deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ]; - deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ]; + deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) [] []; + deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) [] []; + deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386) [] []; + deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64) [] []; + deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] []; + deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] []; + deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ] [ "libsodium13" ]; + deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; + deb_ubuntu1510i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1510i386) [ "libsodium-dev" ] [ "libsodium13"]; + deb_ubuntu1510x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1510x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; + deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ]; + deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; # System tests. @@ -230,6 +223,26 @@ let touch $out/nix-support/hydra-build-products ''); # */ + tests.evalNixpkgs = + import <nixpkgs/pkgs/top-level/make-tarball.nix> { + inherit nixpkgs; + inherit pkgs; + nix = build.x86_64-linux; + officialRelease = false; + }; + + tests.evalNixOS = + pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } + '' + export NIX_DB_DIR=$TMPDIR + export NIX_STATE_DIR=$TMPDIR + nix-store --init + + nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run + + touch $out + ''; + # Aggregate job containing the release-critical jobs. release = pkgs.releaseTools.aggregate { @@ -247,8 +260,8 @@ let binaryTarball.x86_64-darwin #binaryTarball.x86_64-freebsd binaryTarball.x86_64-linux - deb_debian7i386 - deb_debian7x86_64 + deb_debian8i386 + deb_debian8x86_64 deb_ubuntu1404i386 # LTS deb_ubuntu1404x86_64 # LTS deb_ubuntu1504i386 @@ -260,6 +273,8 @@ let tests.remoteBuilds tests.nix-copy-closure tests.binaryTarball + tests.evalNixpkgs + tests.evalNixOS ]; }; @@ -279,7 +294,7 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" "libcurl-devel" ] + [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" "libcurl-devel" "openssl-devel" "xz-devel" ] ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; @@ -291,7 +306,7 @@ let makeDeb_x86_64 = makeDeb "x86_64-linux"; makeDeb = - system: diskImageFun: extraPackages: + system: diskImageFun: extraPackages: extraDebPackages: with import <nixpkgs> { inherit system; }; @@ -300,14 +315,15 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" ] + [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ] ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; + postInstall = "make installcheck"; configureFlags = "--sysconfdir=/etc"; debRequires = - [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" ] - ++ lib.optionals (lib.elem "libsodium-dev" extraPackages) [ "libsodium13" ] ; + [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" "libssl1.0.0" "liblzma5" ] + ++ extraDebPackages; debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>"; doInstallCheck = true; }; diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index c9b4a502e23d..4bf42941116f 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -16,8 +16,6 @@ binmode STDERR, ":encoding(utf8)"; my $debug = defined $ENV{NIX_DEBUG_HOOK}; -setVerbosity(0); # make exportPath() less verbose - # General operation: # @@ -55,7 +53,7 @@ sub all { $_ || return 0 for @_; 1 } # Initialisation. my $loadIncreased = 0; -my ($localSystem, $maxSilentTime, $printBuildTrace, $buildTimeout) = @ARGV; +my ($localSystem, $maxSilentTime, $buildTimeout) = @ARGV; my $currentLoad = $ENV{"NIX_CURRENT_LOAD"} // "/run/nix/current-load"; my $conf = $ENV{"NIX_REMOTE_SYSTEMS"} // "@sysconfdir@/nix/machines"; @@ -225,13 +223,6 @@ my @inputs = split /\s/, readline(STDIN); my @outputs = split /\s/, readline(STDIN); -print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace; - - -my $maybeSign = ""; -$maybeSign = "--sign" if -e "$Nix::Config::confDir/signing-key.sec"; - - # Copy the derivation and its dependencies to the build machine. This # is guarded by an exclusive lock per machine to prevent multiple # build-remote instances from copying to a machine simultaneously. @@ -255,19 +246,17 @@ if ($@) { print STDERR "somebody is hogging $uploadLock, continuing...\n"; unlink $uploadLock; } -Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], 0, 0, $maybeSign ne ""); +Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], 0, 0); close UPLOADLOCK; # Perform the build. print STDERR "building ‘$drvPath’ on ‘$hostName’\n"; -print STDERR "@ build-remote-start $drvPath $hostName\n" if $printBuildTrace; writeInt(6, $to) or die; # == cmdBuildPaths writeStrings([$drvPath], $to); writeInt($maxSilentTime, $to); writeInt($buildTimeout, $to); my $res = readInt($from); -print STDERR "@ build-remote-done $drvPath $hostName\n" if $printBuildTrace; if ($res != 0) { my $msg = decode("utf-8", readString($from)); print STDERR "error: $msg on ‘$hostName’\n"; diff --git a/scripts/download-from-binary-cache.pl.in b/scripts/download-from-binary-cache.pl.in index bb63eafca522..a4f858610aca 100644 --- a/scripts/download-from-binary-cache.pl.in +++ b/scripts/download-from-binary-cache.pl.in @@ -21,7 +21,7 @@ Nix::Config::readConfig; my @caches; my $gotCaches = 0; -my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150); +my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 25); $maxParallelRequests = 1 if $maxParallelRequests < 1; my $ttlNegative = 24 * 3600; # when to purge negative lookups from the database @@ -80,7 +80,12 @@ sub addRequest { $curl->setopt(CURLOPT_WRITEDATA, $fh); $curl->setopt(CURLOPT_FOLLOWLOCATION, 1); $curl->setopt(CURLOPT_CAINFO, $caBundle) if defined $caBundle; - $curl->setopt(CURLOPT_SSL_VERIFYPEER, 0) unless isTrue($Nix::Config::config{"verify-https-binary-caches"} // "1"); + + unless (isTrue($Nix::Config::config{"verify-https-binary-caches"} // "1")) { + $curl->setopt(CURLOPT_SSL_VERIFYPEER, 0); + $curl->setopt(CURLOPT_SSL_VERIFYHOST, 0); + } + $curl->setopt(CURLOPT_USERAGENT, $userAgent); $curl->setopt(CURLOPT_NOBODY, 1) if $head; $curl->setopt(CURLOPT_FAILONERROR, 1); @@ -561,7 +566,7 @@ sub downloadBinary { die if $requireSignedBinaryCaches && !defined $info->{signedBy}; print STDERR "\n*** Downloading ‘$url’ ", ($requireSignedBinaryCaches ? "(signed by ‘$info->{signedBy}’) " : ""), "to ‘$storePath’...\n"; checkURL $url; - if (system("$Nix::Config::curl --fail --location --insecure --connect-timeout $curlConnectTimeout -A '$userAgent' '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { + if (system("$Nix::Config::curl --fail --location --connect-timeout $curlConnectTimeout -A '$userAgent' '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { warn "download of ‘$url’ failed" . ($! ? ": $!" : "") . "\n"; next; } diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in deleted file mode 100755 index 591cd6b43a3a..000000000000 --- a/scripts/download-using-manifests.pl.in +++ /dev/null @@ -1,377 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::Config; -use Nix::Manifest; -use Nix::Store; -use Nix::Utils; -use POSIX qw(strftime); - -STDOUT->autoflush(1); -binmode STDERR, ":encoding(utf8)"; - -my $logFile = "$Nix::Config::logDir/downloads"; - -# For queries, skip expensive calls to nix-hash etc. We're just -# estimating the expected download size. -my $fast = 1; - -# ‘--insecure’ is fine because Nix verifies the hash of the result. -my $curl = "$Nix::Config::curl --fail --location --insecure"; - - -# Open the manifest cache and update it if necessary. -my $dbh = updateManifestDB(); -exit 0 unless defined $dbh; # exit if there are no manifests -print "\n"; - - -# $hashCache->{$algo}->{$path} yields the $algo-hash of $path. -my $hashCache; - - -sub parseHash { - my $hash = shift; - if ($hash =~ /^(.+):(.+)$/) { - return ($1, $2); - } else { - return ("md5", $hash); - } -} - - -# Compute the most efficient sequence of downloads to produce the -# given path. -sub computeSmallestDownload { - my $targetPath = shift; - - # Build a graph of all store paths that might contribute to the - # construction of $targetPath, and the special node "start". The - # edges are either patch operations, or downloads of full NAR - # files. The latter edges only occur between "start" and a store - # path. - my %graph; - - $graph{"start"} = {d => 0, pred => undef, edges => []}; - - my @queue = (); - my $queueFront = 0; - my %done; - - sub addNode { - my $graph = shift; - my $u = shift; - $$graph{$u} = {d => 999999999999, pred => undef, edges => []} - unless defined $$graph{$u}; - } - - sub addEdge { - my $graph = shift; - my $u = shift; - my $v = shift; - my $w = shift; - my $type = shift; - my $info = shift; - addNode $graph, $u; - push @{$$graph{$u}->{edges}}, - {weight => $w, start => $u, end => $v, type => $type, info => $info}; - my $n = scalar @{$$graph{$u}->{edges}}; - } - - push @queue, $targetPath; - - while ($queueFront < scalar @queue) { - my $u = $queue[$queueFront++]; - next if defined $done{$u}; - $done{$u} = 1; - - addNode \%graph, $u; - - # If the path already exists, it has distance 0 from the - # "start" node. - if (isValidPath($u)) { - addEdge \%graph, "start", $u, 0, "present", undef; - } - - else { - - # Add patch edges. - my $patchList = $dbh->selectall_arrayref( - "select * from Patches where storePath = ?", - { Slice => {} }, $u); - - foreach my $patch (@{$patchList}) { - if (isValidPath($patch->{basePath})) { - my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; - - my $hash = $hashCache->{$baseHashAlgo}->{$patch->{basePath}}; - if (!defined $hash) { - $hash = $fast && $baseHashAlgo eq "sha256" - ? queryPathHash($patch->{basePath}) - : hashPath($baseHashAlgo, $baseHashAlgo ne "md5", $patch->{basePath}); - $hash =~ s/.*://; - $hashCache->{$baseHashAlgo}->{$patch->{basePath}} = $hash; - } - - next if $hash ne $baseHash; - } - push @queue, $patch->{basePath}; - addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch; - } - - # Add NAR file edges to the start node. - my $narFileList = $dbh->selectall_arrayref( - "select * from NARs where storePath = ?", - { Slice => {} }, $u); - - foreach my $narFile (@{$narFileList}) { - # !!! how to handle files whose size is not known in advance? - # For now, assume some arbitrary size (1 GB). - # This has the side-effect of preferring non-Hydra downloads. - addEdge \%graph, "start", $u, ($narFile->{size} || 1000000000), "narfile", $narFile; - } - } - } - - - # Run Dijkstra's shortest path algorithm to determine the shortest - # sequence of download and/or patch actions that will produce - # $targetPath. - - my @todo = keys %graph; - - while (scalar @todo > 0) { - - # Remove the closest element from the todo list. - # !!! inefficient, use a priority queue - @todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo; - my $u = pop @todo; - - my $u_ = $graph{$u}; - - foreach my $edge (@{$u_->{edges}}) { - my $v_ = $graph{$edge->{end}}; - if ($v_->{d} > $u_->{d} + $edge->{weight}) { - $v_->{d} = $u_->{d} + $edge->{weight}; - # Store the edge; to edge->start is actually the - # predecessor. - $v_->{pred} = $edge; - } - } - } - - - # Retrieve the shortest path from "start" to $targetPath. - my @path = (); - my $cur = $targetPath; - return () unless defined $graph{$targetPath}->{pred}; - while ($cur ne "start") { - push @path, $graph{$cur}->{pred}; - $cur = $graph{$cur}->{pred}->{start}; - } - - return @path; -} - - -# Parse the arguments. - -if ($ARGV[0] eq "--query") { - - while (<STDIN>) { - chomp; - my ($cmd, @args) = split " ", $_; - - if ($cmd eq "have") { - foreach my $storePath (@args) { - print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0; - } - print "\n"; - } - - elsif ($cmd eq "info") { - foreach my $storePath (@args) { - - my $infos = $dbh->selectall_arrayref( - "select * from NARs where storePath = ?", - { Slice => {} }, $storePath); - - next unless scalar @{$infos} > 0; - my $info = @{$infos}[0]; - - print "$storePath\n"; - print "$info->{deriver}\n"; - my @references = split " ", $info->{refs}; - print scalar @references, "\n"; - print "$_\n" foreach @references; - - my @path = computeSmallestDownload $storePath; - - my $downloadSize = 0; - while (scalar @path > 0) { - my $edge = pop @path; - my $u = $edge->{start}; - my $v = $edge->{end}; - if ($edge->{type} eq "patch") { - $downloadSize += $edge->{info}->{size} || 0; - } - elsif ($edge->{type} eq "narfile") { - $downloadSize += $edge->{info}->{size} || 0; - } - } - - print "$downloadSize\n"; - - my $narSize = $info->{narSize} || 0; - print "$narSize\n"; - } - - print "\n"; - } - - else { die "unknown command ‘$cmd’"; } - } - - exit 0; -} - -elsif ($ARGV[0] ne "--substitute") { - die; -} - - -die unless scalar @ARGV == 3; -my $targetPath = $ARGV[1]; -my $destPath = $ARGV[2]; -$fast = 0; - - -# Create a temporary directory. -my $tmpDir = mkTempDir("nix-download"); - -my $tmpNar = "$tmpDir/nar"; -my $tmpNar2 = "$tmpDir/nar2"; - - -open LOGFILE, ">>$logFile" or die "cannot open log file $logFile"; - -my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); -print LOGFILE "$$ get $targetPath $date\n"; - -print STDERR "\n*** Trying to download/patch ‘$targetPath’\n"; - - -# Compute the shortest path. -my @path = computeSmallestDownload $targetPath; -die "don't know how to produce $targetPath\n" if scalar @path == 0; - - -# We don't need the manifest anymore, so close it as an optimisation: -# if we still have SQLite locks blocking other processes (we -# shouldn't), this gets rid of them. -$dbh->disconnect; - - -# Traverse the shortest path, perform the actions described by the -# edges. -my $curStep = 1; -my $maxStep = scalar @path; - -my $finalNarHash; - -while (scalar @path > 0) { - my $edge = pop @path; - my $u = $edge->{start}; - my $v = $edge->{end}; - - print STDERR "\n*** Step $curStep/$maxStep: "; - - if ($edge->{type} eq "present") { - print STDERR "using already present path ‘$v’\n"; - print LOGFILE "$$ present $v\n"; - - if ($curStep < $maxStep) { - # Since this is not the last step, the path will be used - # as a base to one or more patches. So turn the base path - # into a NAR archive, to which we can apply the patch. - print STDERR " packing base path...\n"; - system("$Nix::Config::binDir/nix-store --dump $v > $tmpNar") == 0 - or die "cannot dump ‘$v’"; - } - } - - elsif ($edge->{type} eq "patch") { - my $patch = $edge->{info}; - print STDERR "applying patch ‘$patch->{url}’ to ‘$u’ to create ‘$v’\n"; - - print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n"; - - # Download the patch. - print STDERR " downloading patch...\n"; - my $patchPath = "$tmpDir/patch"; - checkURL $patch->{url}; - system("$curl '$patch->{url}' -o $patchPath") == 0 - or die "cannot download patch ‘$patch->{url}’\n"; - - # Apply the patch to the NAR archive produced in step 1 (for - # the already present path) or a later step (for patch sequences). - print STDERR " applying patch...\n"; - system("$Nix::Config::libexecDir/nix/bspatch $tmpNar $tmpNar2 $patchPath") == 0 - or die "cannot apply patch ‘$patchPath’ to $tmpNar\n"; - - if ($curStep < $maxStep) { - # The archive will be used as the base of the next patch. - rename "$tmpNar2", "$tmpNar" or die "cannot rename NAR archive: $!"; - } else { - # This was the last patch. Unpack the final NAR archive - # into the target path. - print STDERR " unpacking patched archive...\n"; - system("$Nix::Config::binDir/nix-store --restore $destPath < $tmpNar2") == 0 - or die "cannot unpack $tmpNar2 to ‘$v’\n"; - } - - $finalNarHash = $patch->{narHash}; - } - - elsif ($edge->{type} eq "narfile") { - my $narFile = $edge->{info}; - print STDERR "downloading ‘$narFile->{url}’ to ‘$v’\n"; - - my $size = $narFile->{size} || -1; - print LOGFILE "$$ narfile $narFile->{url} $size $v\n"; - - checkURL $narFile->{url}; - - my $decompressor = - $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" : - $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" : - $narFile->{compressionType} eq "none" ? "" : - die "unknown compression type ‘$narFile->{compressionType}’"; - - if ($curStep < $maxStep) { - # The archive will be used a base to a patch. - system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0 - or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n"; - } else { - # Unpack the archive to the target path. - system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0 - or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n"; - } - - $finalNarHash = $narFile->{narHash}; - } - - $curStep++; -} - - -# Tell Nix about the expected hash so it can verify it. -die "cannot check integrity of the downloaded path since its hash is not known\n" - unless defined $finalNarHash; -print "$finalNarHash\n"; - - -print STDERR "\n"; -print LOGFILE "$$ success\n"; -close LOGFILE; diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 3efe7b38435e..95f69cad84bc 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/bin/sh set -e @@ -25,7 +25,7 @@ echo "performing a single-user installation of Nix..." >&2 if ! [ -e $dest ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" - echo "directory $dest does not exist; creating it by running ‘$cmd’ using sudo" >&2 + echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then echo "$0: please manually run ‘$cmd’ as root to create $dest" >&2 exit 1 @@ -33,7 +33,7 @@ if ! [ -e $dest ]; then fi if ! [ -w $dest ]; then - echo "$0: directory $dest exists, but is not writable by you; please run ‘chown -R $USER $dest’ as root" >&2 + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run ‘chown -R $USER $dest’ as root." >&2 exit 1 fi @@ -49,7 +49,10 @@ for i in $(cd $self/store >/dev/null && echo *); do fi if ! [ -e "$dest/store/$i" ]; then cp -Rp "$self/store/$i" "$i_tmp" + chmod -R a-w "$i_tmp" + chmod +w "$i_tmp" mv "$i_tmp" "$dest/store/$i" + chmod -w "$dest/store/$i" fi done echo "" >&2 @@ -92,7 +95,7 @@ p=$NIX_LINK/etc/profile.d/nix.sh added= for i in .bash_profile .bash_login .profile; do fn="$HOME/$i" - if [ -e "$fn" ]; then + if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> $fn diff --git a/scripts/local.mk b/scripts/local.mk index 39e1df611c5c..13b13a86bc6c 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -2,26 +2,18 @@ nix_bin_scripts := \ $(d)/nix-build \ $(d)/nix-channel \ $(d)/nix-copy-closure \ - $(d)/nix-generate-patches \ $(d)/nix-install-package \ - $(d)/nix-prefetch-url \ - $(d)/nix-pull \ $(d)/nix-push bin-scripts += $(nix_bin_scripts) -nix_substituters := \ - $(d)/copy-from-other-stores.pl \ - $(d)/download-from-binary-cache.pl \ - $(d)/download-using-manifests.pl - nix_noinst_scripts := \ $(d)/build-remote.pl \ $(d)/find-runtime-roots.pl \ + $(d)/resolve-system-dependencies.pl \ $(d)/nix-http-export.cgi \ $(d)/nix-profile.sh \ - $(d)/nix-reduce-build \ - $(nix_substituters) + $(d)/nix-reduce-build noinst-scripts += $(nix_noinst_scripts) @@ -30,7 +22,7 @@ profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/find-runtime-roots.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) -$(foreach prog, $(nix_substituters), $(eval $(call install-program-in, $(prog), $(libexecdir)/nix/substituters))) +$(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 0a4431681cb1..2d45e37c52d6 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -6,6 +6,7 @@ use Nix::Config; use Nix::Store; use Nix::Utils; use File::Basename; +use Text::ParseWords; use Cwd; binmode STDERR, ":encoding(utf8)"; @@ -56,7 +57,7 @@ if ($runEnv && defined $ARGV[0] && $ARGV[0] !~ /nix-shell/) { while (<SCRIPT>) { chomp; if (/^\#\!\s*nix-shell (.*)$/) { - push @ARGV, split(/ /, $1); + push @ARGV, shellwords($1); } } } @@ -109,13 +110,6 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--log-type") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - push @instArgs, ($arg, $ARGV[$n]); - push @buildArgs, ($arg, $ARGV[$n]); - } - elsif ($arg eq "--option") { die "$0: ‘$arg’ requires two arguments\n" unless $n + 2 < scalar @ARGV; push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]); @@ -123,7 +117,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--log-type" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { + elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; push @buildArgs, ($arg, $ARGV[$n]); @@ -190,17 +184,31 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; my $interpreter = $ARGV[$n]; - # Überhack to support Perl. Perl examines the shebang and - # executes it unless it contains the string "perl" or "indir", - # or (undocumented) argv[0] does not contain "perl". Exploit - # the latter by doing "exec -a". - my $execArgs = $interpreter =~ /perl/ ? "-a PERL" : ""; + my $execArgs = ""; + sub shellEscape { my $s = $_; $s =~ s/'/'\\''/g; return "'" . $s . "'"; } - $envCommand = "exec $execArgs $interpreter $script ${\(join ' ', (map shellEscape, @savedArgs))}"; + + # Überhack to support Perl. Perl examines the shebang and + # executes it unless it contains the string "perl" or "indir", + # or (undocumented) argv[0] does not contain "perl". Exploit + # the latter by doing "exec -a". + if ($interpreter =~ /perl/) { + $execArgs = "-a PERL"; + } + + if ($interpreter =~ /ruby/) { + # Hack for Ruby. Ruby also examines the shebang. It tries to + # read the shebang to understand which packages to read from. Since + # this is handled via nix-shell -p, we wrap our ruby script execution + # in ruby -e 'load' which ignores the shebangs. + $envCommand = "exec $execArgs $interpreter -e 'load(\"$script\")' -- ${\(join ' ', (map shellEscape, @savedArgs))}"; + } else { + $envCommand = "exec $execArgs $interpreter $script ${\(join ' ', (map shellEscape, @savedArgs))}"; + } } elsif (substr($arg, 0, 1) eq "-") { @@ -269,7 +277,7 @@ foreach my $expr (@exprs) { my $tmp = $ENV{"TMPDIR"} // $ENV{"XDG_RUNTIME_DIR"} // "/tmp"; if ($pure) { foreach my $name (keys %ENV) { - next if grep { $_ eq $name } ("HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER"); + next if grep { $_ eq $name } ("HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL"); delete $ENV{$name}; } # NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index 05716645a780..65084ff1f34a 100755 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in @@ -12,8 +12,6 @@ binmode STDERR, ":encoding(utf8)"; Nix::Config::readConfig; -my $manifestDir = $Nix::Config::manifestDir; - # Turn on caching in nix-prefetch-url. my $channelCache = "$Nix::Config::stateDir/channel-cache"; @@ -75,7 +73,6 @@ sub removeChannel { my ($name) = @_; readChannels; my $url = $channels{$name}; - deleteOldManifests($url . "/MANIFEST", undef) if defined $url; delete $channels{$name}; writeChannels; @@ -84,8 +81,7 @@ sub removeChannel { } -# Fetch Nix expressions and pull manifests from the subscribed -# channels. +# Fetch Nix expressions and binary cache URLs from the subscribed channels. sub update { my @channelNames = @_; @@ -97,7 +93,6 @@ sub update { next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames}; my $url = $channels{$name}; - my $origUrl = "$url/MANIFEST"; # We want to download the url to a file to see if it's a tarball while also checking if we # got redirected in the process, so that we can grab the various parts of a nix channel @@ -105,8 +100,8 @@ sub update { my $tmpdir = tempdir( CLEANUP => 1 ); my $filename; ($url, $filename) = `cd $tmpdir && $Nix::Config::curl --silent --write-out '%{url_effective}\n%{filename_effective}' -L '$url' -O`; - die "$0: unable to check ‘$url’\n" if $? != 0; chomp $url; + die "$0: unable to check ‘$url’\n" if $? != 0; # If the URL contains a version number, append it to the name # attribute (so that "nix-env -q" on the channels profile @@ -132,22 +127,8 @@ sub update { if ($ret != 0) { # Check if the channel advertises a binary cache. my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`; - my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true"; - if ($? == 0 && $binaryCacheURL ne "") { - $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "; - deleteOldManifests($origUrl, undef); - } else { - $getManifest = 1; - } - - if ($getManifest) { - # No binary cache, so pull the channel manifest. - mkdir $manifestDir, 0755 unless -e $manifestDir; - die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir; - $ENV{'NIX_ORIG_URL'} = $origUrl; - system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0 - or die "cannot pull manifest from ‘$url’\n"; - } + $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; " + if $? == 0 && $binaryCacheURL ne ""; # Download the channel tarball. my $fullURL = "$url/nixexprs.tar.xz"; diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index d7a2c973948b..0078d7267353 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -1,5 +1,6 @@ #! @perl@ -w @perlFlags@ +use utf8; use strict; use Nix::SSH; use Nix::Config; @@ -11,7 +12,7 @@ binmode STDERR, ":encoding(utf8)"; if (scalar @ARGV < 1) { print STDERR <<EOF -Usage: nix-copy-closure [--from | --to] HOSTNAME [--sign] [--gzip] [--bzip2] [--xz] PATHS... +Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS... EOF ; exit 1; @@ -20,11 +21,11 @@ EOF # Get the target host. my $sshHost; -my $sign = 0; my $toMode = 1; my $includeOutputs = 0; my $dryRun = 0; my $useSubstitutes = 0; +my $verbosity = 1; # !!! Copied from nix-pack-closure, should put this in a module. @@ -36,9 +37,6 @@ while (@ARGV) { if ($arg eq "--help") { exec "man nix-copy-closure" or die; } - elsif ($arg eq "--sign") { - $sign = 1; - } elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") { warn "$0: ‘$arg’ is not implemented\n" if $arg ne "--gzip"; push @globalSshOpts, "-C"; @@ -61,6 +59,10 @@ while (@ARGV) { elsif ($arg eq "--use-substitutes" || $arg eq "-s") { $useSubstitutes = 1; } + elsif ($arg eq "-v") { + $verbosity++; + setVerbosity($verbosity); + } elsif (!defined $sshHost) { $sshHost = $arg; } @@ -75,7 +77,7 @@ die "$0: you did not specify a host name\n" unless defined $sshHost; if ($toMode) { # Copy TO the remote machine. Nix::CopyClosure::copyTo( $sshHost, [ @storePaths ], - $includeOutputs, $dryRun, $sign, $useSubstitutes); + $includeOutputs, $dryRun, $useSubstitutes); } else { # Copy FROM the remote machine. @@ -93,7 +95,7 @@ else { # Copy FROM the remote machine. if (scalar @missing > 0) { print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; writeInt(5, $to); # == cmdExportPaths - writeInt($sign ? 1 : 0, $to); + writeInt(0, $to); # obsolete writeStrings(\@missing, $to); importPaths(fileno($from)); } diff --git a/scripts/nix-generate-patches.in b/scripts/nix-generate-patches.in deleted file mode 100755 index 0a29c0548c1f..000000000000 --- a/scripts/nix-generate-patches.in +++ /dev/null @@ -1,51 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use strict; -use Nix::Manifest; -use Nix::GeneratePatches; -use Nix::Utils; - -if (scalar @ARGV != 5) { - print STDERR <<EOF; -Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST - -This command generates binary patches between NAR files listed in -OLD-MANIFEST and NEW-MANIFEST. The patches are written to the -directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs -for the patches. The patches are added to NEW-MANIFEST. All NARs are -required to exist in NAR-DIR. Patches are generated between -succeeding versions of packages with the same name. -EOF - exit 1; -} - -my $narPath = $ARGV[0]; -my $patchesPath = $ARGV[1]; -my $patchesURL = $ARGV[2]; -my $srcManifest = $ARGV[3]; -my $dstManifest = $ARGV[4]; - -my (%srcNarFiles, %srcLocalPaths, %srcPatches); -readManifest $srcManifest, \%srcNarFiles, \%srcPatches; - -my (%dstNarFiles, %dstLocalPaths, %dstPatches); -readManifest $dstManifest, \%dstNarFiles, \%dstPatches; - -my $tmpDir = mkTempDir("nix-generate-patches"); - -generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches, - $narPath, $patchesPath, $patchesURL, $tmpDir; - -propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches; - -# Optionally add all new patches to the manifest in $NIX_ALL_PATCHES. -my $allPatchesFile = $ENV{"NIX_ALL_PATCHES"}; -if (defined $allPatchesFile) { - my (%dummy, %allPatches); - readManifest("$patchesPath/all-patches", \%dummy, \%allPatches) - if -f $allPatchesFile; - copyPatches \%dstPatches, \%allPatches; - writeManifest($allPatchesFile, {}, \%allPatches, 0); -} - -writeManifest $dstManifest, \%dstNarFiles, \%dstPatches; diff --git a/scripts/nix-install-package.in b/scripts/nix-install-package.in index b442c708b1a2..ba349774af54 100755 --- a/scripts/nix-install-package.in +++ b/scripts/nix-install-package.in @@ -89,7 +89,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )"; # store path. We'll let nix-env do that. $contents =~ - / ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x + / ^ \s* (\S+) \s+ (\S+) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x or barf "invalid package contents"; my $version = $1; my $manifestURL = $2; @@ -111,25 +111,9 @@ if ($interactive) { } -if (defined $binaryCacheURL) { +die "$0: package does not supply a binary cache\n" unless defined $binaryCacheURL; - push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL; - -} else { - - # Store the manifest in the temporary directory so that we don't - # pollute /nix/var/nix/manifests. This also requires that we - # don't use the Nix daemon (because otherwise - # download-using-manifests won't see our NIX_MANIFESTS_DIRS - # environment variable). - $ENV{NIX_MANIFESTS_DIR} = $tmpDir; - $ENV{NIX_REMOTE} = ""; - - print "\nPulling manifests...\n"; - system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0 - or barf "nix-pull failed: $?"; - -} +push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL; print "\nInstalling package...\n"; diff --git a/scripts/nix-prefetch-url.in b/scripts/nix-prefetch-url.in deleted file mode 100755 index 6effbe208146..000000000000 --- a/scripts/nix-prefetch-url.in +++ /dev/null @@ -1,132 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use File::Basename; -use File::stat; -use Nix::Store; -use Nix::Config; -use Nix::Utils; - -binmode STDERR, ":encoding(utf8)"; - - -my $hashType = $ENV{'NIX_HASH_ALGO'} || "sha256"; # obsolete -my $cacheDir = $ENV{'NIX_DOWNLOAD_CACHE'}; - -my @args; -my $arg; -while ($arg = shift) { - if ($arg eq "--help") { - exec "man nix-prefetch-url" or die; - } elsif ($arg eq "--type") { - $hashType = shift; - die "$0: ‘$arg’ requires an argument\n" unless defined $hashType; - } elsif (substr($arg, 0, 1) eq "-") { - die "$0: unknown flag ‘$arg’\n"; - } else { - push @args, $arg; - } -} - -my $url = $args[0]; -my $expHash = $args[1]; - - -if (!defined $url || $url eq "") { - print STDERR <<EOF -Usage: nix-prefetch-url URL [EXPECTED-HASH] -EOF - ; - exit 1; -} - -my $tmpDir = mkTempDir("nix-prefetch-url"); - -# Hack to support the mirror:// scheme from Nixpkgs. -if ($url =~ /^mirror:\/\//) { - system("$Nix::Config::binDir/nix-build '<nixpkgs>' -A resolveMirrorURLs --argstr url '$url' -o $tmpDir/urls > /dev/null") == 0 - or die "$0: nix-build failed; maybe \$NIX_PATH is not set properly\n"; - my @expanded = split ' ', readFile("$tmpDir/urls"); - die "$0: cannot resolve ‘$url’" unless scalar @expanded > 0; - print STDERR "$url expands to $expanded[0]\n"; - $url = $expanded[0]; -} - -# Handle escaped characters in the URI. `+', `=' and `?' are the only -# characters that are valid in Nix store path names but have a special -# meaning in URIs. -my $name = basename $url; -die "cannot figure out file name for ‘$url’\n" if $name eq ""; -$name =~ s/%2b/+/g; -$name =~ s/%3d/=/g; -$name =~ s/%3f/?/g; - -my $finalPath; -my $hash; - -# If the hash was given, a file with that hash may already be in the -# store. -if (defined $expHash) { - $finalPath = makeFixedOutputPath(0, $hashType, $expHash, $name); - if (isValidPath($finalPath)) { $hash = $expHash; } else { $finalPath = undef; } -} - -# If we don't know the hash or a file with that hash doesn't exist, -# download the file and add it to the store. -if (!defined $finalPath) { - - my $tmpFile = "$tmpDir/$name"; - - # Optionally do timestamp-based caching of the download. - # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is - # the hash and the timestamp of the file at $url. The caching of - # the file *contents* is done in Nix store, where it can be - # garbage-collected independently. - my ($cachedTimestampFN, $cachedHashFN, @cacheFlags); - if (defined $cacheDir) { - my $urlHash = hashString("sha256", 1, $url); - writeFile "$cacheDir/$urlHash.url", $url; - $cachedHashFN = "$cacheDir/$urlHash.$hashType"; - $cachedTimestampFN = "$cacheDir/$urlHash.stamp"; - @cacheFlags = ("--time-cond", $cachedTimestampFN) if -f $cachedHashFN && -f $cachedTimestampFN; - } - - # Perform the download. - my @curlFlags = ("curl", $url, "-o", $tmpFile, "--fail", "--location", "--max-redirs", "20", "--disable-epsv", "--cookie-jar", "$tmpDir/cookies", "--remote-time", (split " ", ($ENV{NIX_CURL_FLAGS} || ""))); - (system $Nix::Config::curl @curlFlags, @cacheFlags) == 0 or die "$0: download of ‘$url’ failed\n"; - - if (defined $cacheDir && ! -e $tmpFile) { - # Curl didn't create $tmpFile, so apparently there's no newer - # file on the server. - $hash = readFile $cachedHashFN or die; - $finalPath = makeFixedOutputPath(0, $hashType, $hash, $name); - unless (isValidPath $finalPath) { - print STDERR "cached contents of ‘$url’ disappeared, redownloading...\n"; - $finalPath = undef; - (system $Nix::Config::curl @curlFlags) == 0 or die "$0: download of ‘$url’ failed\n"; - } - } - - if (!defined $finalPath) { - - # Compute the hash. - $hash = hashFile($hashType, $hashType ne "md5", $tmpFile); - - if (defined $cacheDir) { - writeFile $cachedHashFN, $hash; - my $st = stat($tmpFile) or die; - open STAMP, ">$cachedTimestampFN" or die; close STAMP; - utime($st->atime, $st->mtime, $cachedTimestampFN) or die; - } - - # Add the downloaded file to the Nix store. - $finalPath = addToStore($tmpFile, 0, $hashType); - } - - die "$0: hash mismatch for ‘$url’\n" if defined $expHash && $expHash ne $hash; -} - -print STDERR "path is ‘$finalPath’\n" unless $ENV{'QUIET'}; -print "$hash\n"; -print "$finalPath\n" if $ENV{'PRINT_PATH'}; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 672d1f035f91..5e01de95156c 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,28 +1,77 @@ -if [ -n "$HOME" ]; then - NIX_LINK="$HOME/.nix-profile" - - # Set the default profile. - if ! [ -L "$NIX_LINK" ]; then - echo "creating $NIX_LINK" >&2 - _NIX_DEF_LINK=@localstatedir@/nix/profiles/default - @coreutils@/ln -s "$_NIX_DEF_LINK" "$NIX_LINK" +if [ -n "$HOME" ] && [ -n "$USER" ]; then + __savedpath="$PATH" + export PATH=@coreutils@ + + # Set up the per-user profile. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix + + : ${NIX_LINK:=$HOME/.nix-profile} + + : ${NIX_USER_PROFILE_DIR:=@localstatedir@/nix/profiles/per-user/$USER} + + mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" + + if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then + echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 fi - export PATH=$NIX_LINK/bin:$NIX_LINK/sbin:$PATH + if [ -w "$HOME" ]; then + if ! [ -L "$NIX_LINK" ]; then + echo "Nix: creating $NIX_LINK" >&2 + if [ "$USER" != root ]; then + if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then + echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 + fi + else + # Root installs in the system-wide profile by default. + ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" + fi + fi - # Subscribe the user to the Nixpkgs channel by default. - if [ ! -e $HOME/.nix-channels ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels + # Subscribe the user to the unstable Nixpkgs channel by default. + if [ ! -e "$HOME/.nix-channels" ]; then + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" + fi + + # Create the per-user garbage collector roots directory. + __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" + mkdir -m 0755 -p "$__user_gcroots" + if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then + echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 + fi + unset __user_gcroots + + # Set up a default Nix expression from which to install stuff. + __nix_defexpr="$HOME"/.nix-defexpr + [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" + mkdir -m 0755 -p "$__nix_defexpr" + if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then + ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root + fi + unset __nix_defexpr fi # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that # <nixpkgs> paths work when the user has fetched the Nixpkgs # channel. - export NIX_PATH=${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs + export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + export NIX_USER_PROFILE_DIR + export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + + for i in $NIX_PROFILES; do + if [ -d "$i/lib/aspell" ]; then + export ASPELL_CONF="dict-dir $i/lib/aspell" + fi + done # Set $SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export SSL_CERT_FILE=/etc/ssl/ca-bundle.pem elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS export SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS @@ -32,4 +81,7 @@ if [ -n "$HOME" ]; then elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # old cacert in Nix profile export SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi + + export PATH="$NIX_LINK/bin:$NIX_LINK/sbin:$__savedpath" + unset __savedpath fi diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in deleted file mode 100755 index 995b50935964..000000000000 --- a/scripts/nix-pull.in +++ /dev/null @@ -1,102 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::Config; -use Nix::Manifest; - -binmode STDERR, ":encoding(utf8)"; - -my $manifestDir = $Nix::Config::manifestDir; - - -# Prevent access problems in shared-stored installations. -umask 0022; - - -# Create the manifests directory if it doesn't exist. -if (! -e $manifestDir) { - mkdir $manifestDir, 0755 or die "cannot create directory ‘$manifestDir’"; -} - - -# Make sure that the manifests directory is scanned for GC roots. -my $gcRootsDir = "$Nix::Config::stateDir/gcroots"; -my $manifestDirLink = "$gcRootsDir/manifests"; -if (! -l $manifestDirLink) { - symlink($manifestDir, $manifestDirLink) or die "cannot create symlink ‘$manifestDirLink’"; -} - - -# Process the URLs specified on the command line. - -sub downloadFile { - my $url = shift; - $ENV{"PRINT_PATH"} = 1; - $ENV{"QUIET"} = 1; - my ($dummy, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`; - die "cannot fetch ‘$url’" if $? != 0; - die "nix-prefetch-url did not return a path" unless defined $path; - chomp $path; - return $path; -} - -sub processURL { - my $url = shift; - - $url =~ s/\/$//; - - my $manifest; - - my $origUrl = $ENV{'NIX_ORIG_URL'} || $url; - - # First see if a bzipped manifest is available. - if (system("$Nix::Config::curl --fail --silent --location --head '$url'.bz2 > /dev/null") == 0) { - print "fetching list of Nix archives at ‘$url.bz2’...\n"; - $manifest = downloadFile "$url.bz2"; - } - - # Otherwise, just get the uncompressed manifest. - else { - print "fetching list of Nix archives at ‘$url’...\n"; - $manifest = downloadFile $url; - } - - my $baseName = "unnamed"; - if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component - $baseName = $1; - } - - my $hash = `$Nix::Config::binDir/nix-hash --flat '$manifest'` - or die "cannot hash ‘$manifest’"; - chomp $hash; - - my $urlFile = "$manifestDir/$baseName-$hash.url"; - open URL, ">$urlFile" or die "cannot create ‘$urlFile’"; - print URL $origUrl; - close URL; - - my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest"; - - unlink $finalPath if -e $finalPath; - - symlink("$manifest", "$finalPath") - or die "cannot link ‘$finalPath’ to ‘$manifest’"; - - deleteOldManifests($origUrl, $urlFile); -} - -while (@ARGV) { - my $url = shift @ARGV; - if ($url eq "--help") { - exec "man nix-pull" or die; - } elsif ($url eq "--skip-wrong-store") { - # No-op, no longer supported. - } else { - processURL $url; - } -} - - -# Update the cache. -updateManifestDB(); diff --git a/scripts/nix-push.in b/scripts/nix-push.in index 2d9d83f59b92..54456ac9512e 100755 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -258,13 +258,10 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) { } if (defined $secretKeyFile) { - my $s = readFile $secretKeyFile; - chomp $s; - my ($keyName, $secretKey) = split ":", $s; - die "invalid secret key file ‘$secretKeyFile’\n" unless defined $keyName && defined $secretKey; + my $secretKey = readFile $secretKeyFile; my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs); - my $sig = encode_base64(signString(decode_base64($secretKey), $fingerprint), ""); - $info .= "Sig: $keyName:$sig\n"; + my $sig = signString($secretKey, $fingerprint); + $info .= "Sig: $sig\n"; } my $pathHash = substr(basename($storePath), 0, 32); diff --git a/scripts/resolve-system-dependencies.pl.in b/scripts/resolve-system-dependencies.pl.in new file mode 100755 index 000000000000..a20f0dc020fe --- /dev/null +++ b/scripts/resolve-system-dependencies.pl.in @@ -0,0 +1,122 @@ +#! @perl@ -w @perlFlags@ + +use utf8; +use strict; +use warnings; +use Cwd qw(realpath); +use Errno; +use File::Basename qw(dirname); +use File::Path qw(make_path); +use File::Spec::Functions qw(catfile); +use List::Util qw(reduce); +use IPC::Open3; +use Nix::Config; +use Nix::Store qw(derivationFromPath); +use POSIX qw(uname); +use Storable qw(lock_retrieve lock_store); + +my ($sysname, undef, $version, undef, $machine) = uname; +$sysname =~ /Darwin/ or die "This tool is only meant to be used on Darwin systems."; + +my $cache = "$Nix::Config::stateDir/dependency-maps/$machine-$sysname-$version.map"; + +make_path dirname($cache); + +our $DEPS; +eval { + $DEPS = lock_retrieve($cache); +}; + +if($!{ENOENT}) { + lock_store {}, $cache; + $DEPS = {}; +} elsif($@) { + die "Unable to obtain a lock on dependency-map file $cache: $@"; +} + +sub mkset(@) { + my %set; + @set{@_} = (); + \%set +} + +sub union($$) { + my ($set1, $set2) = @_; + my %new = (%$set1, %$set2); + \%new +} + +sub cache_filepath($) { + my $fp = shift; + $fp =~ s/-/--/g; + $fp =~ s/\//-/g; + $fp =~ s/^-//g; + catfile $cache, $fp +} + +sub resolve_tree { + sub resolve_tree_inner { + my ($lib, $TREE) = @_; + return if (defined $TREE->{$lib}); + $TREE->{$lib} = mkset(@{cache_get($lib)}); + foreach my $dep (keys %{$TREE->{$lib}}) { + resolve_tree_inner($dep, $TREE); + } + values %$TREE + } + + reduce { union($a, $b) } {}, resolve_tree_inner(@_) +} + +sub cache_get { + my $key = shift; + if (defined $DEPS->{$key}) { + $DEPS->{$key} + } else { + cache_insert($key); + cache_get($key) + } +} + +sub cache_insert($) { + my $key = shift; + print STDERR "Finding dependencies for $key...\n"; + my @deps = find_deps($key); + $DEPS->{$key} = \@deps; +} + +sub find_deps($) { + my $lib = shift; + my($chld_in, $chld_out, $chld_err); + my $pid = open3($chld_in, $chld_out, $chld_err, "@otool@", "-L", "-arch", "x86_64", $lib); + waitpid($pid, 0); + my $line = readline $chld_out; + if($? == 0 and $line !~ /not an object file/) { + my @libs; + while(<$chld_out>) { + my $dep = (split /\s+/)[1]; + push @libs, $dep unless $dep eq $lib or $dep =~ /\@rpath/; + } + @libs + } elsif (-l $lib) { + (realpath($lib)) + } else { + () + } +} + +if (defined $ARGV[0]) { + my $deps = derivationFromPath($ARGV[0])->{"env"}->{"__impureHostDeps"}; + if (defined $deps) { + my @files = split(/\s+/, $deps); + my $depcache = {}; + my $depset = reduce { union($a, $b) } (map { resolve_tree($_, $depcache) } @files); + print "extra-chroot-dirs\n"; + print join("\n", keys %$depset); + print "\n"; + } + lock_store($DEPS, $cache); +} else { + print STDERR "Usage: $0 path/to/derivation.drv\n"; + exit 1 +} diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp index 79e452449ef8..a7641458c95e 100644 --- a/src/boost/format/exceptions.hpp +++ b/src/boost/format/exceptions.hpp @@ -33,7 +33,7 @@ namespace io { class format_error : public std::exception { public: - format_error() {} + format_error() { abort(); } virtual const char *what() const throw() { return "boost::format_error: " @@ -44,7 +44,7 @@ public: class bad_format_string : public format_error { public: - bad_format_string() {} + bad_format_string() { abort(); } virtual const char *what() const throw() { return "boost::bad_format_string: " @@ -55,7 +55,7 @@ public: class too_few_args : public format_error { public: - too_few_args() {} + too_few_args() { abort(); } virtual const char *what() const throw() { return "boost::too_few_args: " @@ -66,7 +66,7 @@ public: class too_many_args : public format_error { public: - too_many_args() {} + too_many_args() { abort(); } virtual const char *what() const throw() { return "boost::too_many_args: " @@ -78,7 +78,7 @@ public: class out_of_range : public format_error { public: - out_of_range() {} + out_of_range() { abort(); } virtual const char *what() const throw() { return "boost::out_of_range: " diff --git a/src/bsdiff-4.3/bsdiff.1 b/src/bsdiff-4.3/bsdiff.1 deleted file mode 100644 index ead6c4deb57f..000000000000 --- a/src/bsdiff-4.3/bsdiff.1 +++ /dev/null @@ -1,63 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSDIFF 1 -.Os FreeBSD -.Sh NAME -.Nm bsdiff -.Nd generate a patch between two binary files -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -compares -.Ao Ar oldfile Ac -to -.Ao Ar newfile Ac -and writes to -.Ao Ar patchfile Ac -a binary patch suitable for use by bspatch(1). -When -.Ao Ar oldfile Ac -and -.Ao Ar newfile Ac -are two versions of an executable program, the -patches produced are on average a factor of five smaller -than those produced by any other binary patch tool known -to the author. -.Pp -.Nm -uses memory equal to 17 times the size of -.Ao Ar oldfile Ac , -and requires -an absolute minimum working set size of 8 times the size of oldfile. -.Sh SEE ALSO -.Xr bspatch 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c deleted file mode 100644 index 374ed038fa1f..000000000000 --- a/src/bsdiff-4.3/bsdiff.c +++ /dev/null @@ -1,405 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $"); -#endif - -#include <sys/types.h> - -#include <bzlib.h> -#include <err.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) - -static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) -{ - off_t i,j,k,x,tmp,jj,kk; - - if(len<16) { - for(k=start;k<start+len;k+=j) { - j=1;x=V[I[k]+h]; - for(i=1;k+i<start+len;i++) { - if(V[I[k+i]+h]<x) { - x=V[I[k+i]+h]; - j=0; - }; - if(V[I[k+i]+h]==x) { - tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; - j++; - }; - }; - for(i=0;i<j;i++) V[I[k+i]]=k+j-1; - if(j==1) I[k]=-1; - }; - return; - }; - - x=V[I[start+len/2]+h]; - jj=0;kk=0; - for(i=start;i<start+len;i++) { - if(V[I[i]+h]<x) jj++; - if(V[I[i]+h]==x) kk++; - }; - jj+=start;kk+=jj; - - i=start;j=0;k=0; - while(i<jj) { - if(V[I[i]+h]<x) { - i++; - } else if(V[I[i]+h]==x) { - tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; - j++; - } else { - tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - while(jj+j<kk) { - if(V[I[jj+j]+h]==x) { - j++; - } else { - tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - if(jj>start) split(I,V,start,jj-start,h); - - for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; - if(jj==kk-1) I[jj]=-1; - - if(start+len>kk) split(I,V,kk,start+len-kk,h); -} - -static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) -{ - off_t buckets[256]; - off_t i,h,len; - - for(i=0;i<256;i++) buckets[i]=0; - for(i=0;i<oldsize;i++) buckets[old[i]]++; - for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; - for(i=255;i>0;i--) buckets[i]=buckets[i-1]; - buckets[0]=0; - - for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; - I[0]=oldsize; - for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; - V[oldsize]=0; - for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; - I[0]=-1; - - for(h=1;I[0]!=-(oldsize+1);h+=h) { - len=0; - for(i=0;i<oldsize+1;) { - if(I[i]<0) { - len-=I[i]; - i-=I[i]; - } else { - if(len) I[i-len]=-len; - len=V[I[i]]+1-i; - split(I,V,i,len,h); - i+=len; - len=0; - }; - }; - if(len) I[i-len]=-len; - }; - - for(i=0;i<oldsize+1;i++) I[V[i]]=i; -} - -static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) -{ - off_t i; - - for(i=0;(i<oldsize)&&(i<newsize);i++) - if(old[i]!=new[i]) break; - - return i; -} - -static off_t search(off_t *I,u_char *old,off_t oldsize, - u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) -{ - off_t x,y; - - if(en-st<2) { - x=matchlen(old+I[st],oldsize-I[st],new,newsize); - y=matchlen(old+I[en],oldsize-I[en],new,newsize); - - if(x>y) { - *pos=I[st]; - return x; - } else { - *pos=I[en]; - return y; - } - }; - - x=st+(en-st)/2; - if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { - return search(I,old,oldsize,new,newsize,x,en,pos); - } else { - return search(I,old,oldsize,new,newsize,st,x,pos); - }; -} - -static void offtout(off_t x,u_char *buf) -{ - off_t y; - - if(x<0) y=-x; else y=x; - - buf[0]=y%256;y-=buf[0]; - y=y/256;buf[1]=y%256;y-=buf[1]; - y=y/256;buf[2]=y%256;y-=buf[2]; - y=y/256;buf[3]=y%256;y-=buf[3]; - y=y/256;buf[4]=y%256;y-=buf[4]; - y=y/256;buf[5]=y%256;y-=buf[5]; - y=y/256;buf[6]=y%256;y-=buf[6]; - y=y/256;buf[7]=y%256; - - if(x<0) buf[7]|=0x80; -} - -int main(int argc,char *argv[]) -{ - int fd; - u_char *old,*new; - off_t oldsize,newsize; - off_t *I,*V; - off_t scan,pos,len; - off_t lastscan,lastpos,lastoffset; - off_t oldscore,scsc; - off_t s,Sf,lenf,Sb,lenb; - off_t overlap,Ss,lens; - off_t i; - off_t dblen,eblen; - u_char *db,*eb; - u_char buf[8]; - u_char header[32]; - FILE * pf; - BZFILE * pfbz2; - int bz2err; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - - if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) || - ((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL); - - qsufsort(I,V,old,oldsize); - - free(V); - - /* Allocate newsize+1 bytes instead of newsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[2],O_RDONLY,0))<0) || - ((newsize=lseek(fd,0,SEEK_END))==-1) || - ((new=malloc(newsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,new,newsize)!=newsize) || - (close(fd)==-1)) err(1,"%s",argv[2]); - - if(((db=malloc(newsize+1))==NULL) || - ((eb=malloc(newsize+1))==NULL)) err(1,NULL); - dblen=0; - eblen=0; - - /* Create the patch file */ - if ((pf = fopen(argv[3], "w")) == NULL) - err(1, "%s", argv[3]); - - /* Header is - 0 8 "BSDIFF40" - 8 8 length of bzip2ed ctrl block - 16 8 length of bzip2ed diff block - 24 8 length of new file */ - /* File is - 0 32 Header - 32 ?? Bzip2ed ctrl block - ?? ?? Bzip2ed diff block - ?? ?? Bzip2ed extra block */ - memcpy(header,"BSDIFF40",8); - offtout(0, header + 8); - offtout(0, header + 16); - offtout(newsize, header + 24); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - - /* Compute the differences, writing ctrl as we go */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - scan=0;len=0; - lastscan=0;lastpos=0;lastoffset=0; - while(scan<newsize) { - oldscore=0; - - for(scsc=scan+=len;scan<newsize;scan++) { - len=search(I,old,oldsize,new+scan,newsize-scan, - 0,oldsize,&pos); - if (len > 64 * 1024) break; - - for(;scsc<scan+len;scsc++) - if((scsc+lastoffset<oldsize) && - (old[scsc+lastoffset] == new[scsc])) - oldscore++; - - if(((len==oldscore) && (len!=0)) || - (len>oldscore+8)) break; - - if((scan+lastoffset<oldsize) && - (old[scan+lastoffset] == new[scan])) - oldscore--; - }; - - if((len!=oldscore) || (scan==newsize)) { - s=0;Sf=0;lenf=0; - for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { - if(old[lastpos+i]==new[lastscan+i]) s++; - i++; - if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; - }; - - lenb=0; - if(scan<newsize) { - s=0;Sb=0; - for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { - if(old[pos-i]==new[scan-i]) s++; - if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; - }; - }; - - if(lastscan+lenf>scan-lenb) { - overlap=(lastscan+lenf)-(scan-lenb); - s=0;Ss=0;lens=0; - for(i=0;i<overlap;i++) { - if(new[lastscan+lenf-overlap+i]== - old[lastpos+lenf-overlap+i]) s++; - if(new[scan-lenb+i]== - old[pos-lenb+i]) s--; - if(s>Ss) { Ss=s; lens=i+1; }; - }; - - lenf+=lens-overlap; - lenb-=lens; - }; - - for(i=0;i<lenf;i++) - db[dblen+i]=new[lastscan+i]-old[lastpos+i]; - for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) - eb[eblen+i]=new[lastscan+lenf+i]; - - dblen+=lenf; - eblen+=(scan-lenb)-(lastscan+lenf); - - offtout(lenf,buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((scan-lenb)-(lastscan+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((pos-lenb)-(lastpos+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - lastscan=scan-lenb; - lastpos=pos-lenb; - lastoffset=pos-scan; - }; - }; - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed ctrl data */ - if ((len = ftello(pf)) == -1) - err(1, "ftello"); - offtout(len-32, header + 8); - - /* Write compressed diff data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, db, dblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed diff data */ - if ((newsize = ftello(pf)) == -1) - err(1, "ftello"); - offtout(newsize - len, header + 16); - - /* Write compressed extra data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Seek to the beginning, write the header, and close the file */ - if (fseeko(pf, 0, SEEK_SET)) - err(1, "fseeko"); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - if (fclose(pf)) - err(1, "fclose"); - - /* Free the memory we used */ - free(db); - free(eb); - free(I); - free(old); - free(new); - - return 0; -} diff --git a/src/bsdiff-4.3/bspatch.1 b/src/bsdiff-4.3/bspatch.1 deleted file mode 100644 index 82a2781aa7dc..000000000000 --- a/src/bsdiff-4.3/bspatch.1 +++ /dev/null @@ -1,59 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSPATCH 1 -.Os FreeBSD -.Sh NAME -.Nm bspatch -.Nd apply a patch built with bsdiff(1) -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -generates -.Ao Ar newfile Ac -from -.Ao Ar oldfile Ac -and -.Ao Ar patchfile Ac -where -.Ao Ar patchfile Ac -is a binary patch built by bsdiff(1). -.Pp -.Nm -uses memory equal to the size of -.Ao Ar oldfile Ac -plus the size of -.Ao Ar newfile Ac , -but can tolerate a very small working set without a dramatic loss -of performance. -.Sh SEE ALSO -.Xr bsdiff 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bspatch.c b/src/bsdiff-4.3/bspatch.c deleted file mode 100644 index f9d33ddd64a2..000000000000 --- a/src/bsdiff-4.3/bspatch.c +++ /dev/null @@ -1,224 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $"); -#endif - -#include <bzlib.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <err.h> -#include <errno.h> -#include <unistd.h> -#include <fcntl.h> -#include <sys/types.h> - -static off_t offtin(u_char *buf) -{ - off_t y; - - y=buf[7]&0x7F; - y=y*256;y+=buf[6]; - y=y*256;y+=buf[5]; - y=y*256;y+=buf[4]; - y=y*256;y+=buf[3]; - y=y*256;y+=buf[2]; - y=y*256;y+=buf[1]; - y=y*256;y+=buf[0]; - - if(buf[7]&0x80) y=-y; - - return y; -} - - -void writeFull(const char * name, int fd, - const unsigned char * buf, size_t count) -{ - while (count) { - ssize_t res = write(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - err(1,"writing to %s",name); - } - count -= res; - buf += res; - } -} - - -int main(int argc,char * argv[]) -{ - FILE * f, * cpf, * dpf, * epf; - BZFILE * cpfbz2, * dpfbz2, * epfbz2; - int cbz2err, dbz2err, ebz2err; - int fd; - ssize_t oldsize,newsize; - ssize_t bzctrllen,bzdatalen; - u_char header[32],buf[8]; - u_char *old, *new; - off_t oldpos,newpos; - off_t ctrl[3]; - off_t lenread; - off_t i; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Open patch file */ - if ((f = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - - /* - File format: - 0 8 "BSDIFF40" - 8 8 X - 16 8 Y - 24 8 sizeof(newfile) - 32 X bzip2(control block) - 32+X Y bzip2(diff block) - 32+X+Y ??? bzip2(extra block) - with control block a set of triples (x,y,z) meaning "add x bytes - from oldfile to x bytes from the diff block; copy y bytes from the - extra block; seek forwards in oldfile by z bytes". - */ - - /* Read header */ - if (fread(header, 1, 32, f) < 32) { - if (feof(f)) - errx(1, "Corrupt patch\n"); - err(1, "fread(%s)", argv[3]); - } - - /* Check for appropriate magic */ - if (memcmp(header, "BSDIFF40", 8) != 0) - errx(1, "Corrupt patch\n"); - - /* Read lengths from header */ - bzctrllen=offtin(header+8); - bzdatalen=offtin(header+16); - newsize=offtin(header+24); - if((bzctrllen<0) || (bzdatalen<0) || (newsize<0)) - errx(1,"Corrupt patch\n"); - - /* Close patch file and re-open it via libbzip2 at the right places */ - if (fclose(f)) - err(1, "fclose(%s)", argv[3]); - if ((cpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(cpf, 32, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)32); - if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err); - if ((dpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(dpf, 32 + bzctrllen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen)); - if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err); - if ((epf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen + bzdatalen)); - if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err); - - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - if((new=malloc(newsize+1))==NULL) err(1,NULL); - - oldpos=0;newpos=0; - while(newpos<newsize) { - /* Read control data */ - for(i=0;i<=2;i++) { - lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8); - if ((lenread < 8) || ((cbz2err != BZ_OK) && - (cbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - ctrl[i]=offtin(buf); - }; - - /* Sanity-check */ - if(newpos+ctrl[0]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read diff string */ - lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]); - if ((lenread < ctrl[0]) || - ((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Add old data to diff string */ - for(i=0;i<ctrl[0];i++) - if((oldpos+i>=0) && (oldpos+i<oldsize)) - new[newpos+i]+=old[oldpos+i]; - - /* Adjust pointers */ - newpos+=ctrl[0]; - oldpos+=ctrl[0]; - - /* Sanity-check */ - if(newpos+ctrl[1]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read extra string */ - lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]); - if ((lenread < ctrl[1]) || - ((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Adjust pointers */ - newpos+=ctrl[1]; - oldpos+=ctrl[2]; - }; - - /* Clean up the bzip2 reads */ - BZ2_bzReadClose(&cbz2err, cpfbz2); - BZ2_bzReadClose(&dbz2err, dpfbz2); - BZ2_bzReadClose(&ebz2err, epfbz2); - if (fclose(cpf) || fclose(dpf) || fclose(epf)) - err(1, "fclose(%s)", argv[3]); - - /* Write the new file */ - if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) - err(1,"%s",argv[2]); - writeFull(argv[2], fd, new, newsize); - if(close(fd)==-1) - err(1,"%s",argv[2]); - - free(new); - free(old); - - return 0; -} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h deleted file mode 100644 index a851ded6f907..000000000000 --- a/src/bsdiff-4.3/compat-include/err.h +++ /dev/null @@ -1,12 +0,0 @@ -/* Simulate BSD's <err.h> functionality. */ - -#ifndef COMPAT_ERR_H_INCLUDED -#define COMPAT_ERR_H_INCLUDED 1 - -#include <stdio.h> -#include <stdlib.h> - -#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) -#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) - -#endif diff --git a/src/bsdiff-4.3/local.mk b/src/bsdiff-4.3/local.mk deleted file mode 100644 index c957ceab0c0f..000000000000 --- a/src/bsdiff-4.3/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += bsdiff bspatch - -bsdiff_DIR := $(d) -bsdiff_SOURCES := $(d)/bsdiff.c -bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bsdiff_INSTALL_DIR = $(libexecdir)/nix - -bspatch_DIR := $(d) -bspatch_SOURCES := $(d)/bspatch.c -bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bspatch_INSTALL_DIR = $(libexecdir)/nix diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index f71cf56507b8..ed551ac461fb 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -43,8 +43,7 @@ static std::pair<FdSink, FdSource> connect(const string & conn) static void substitute(std::pair<FdSink, FdSource> & pipes, Path storePath, Path destPath) { - writeInt(cmdDumpStorePath, pipes.first); - writeString(storePath, pipes.first); + pipes.first << cmdDumpStorePath << storePath; pipes.first.flush(); restorePath(destPath, pipes.second); std::cout << std::endl; @@ -58,17 +57,17 @@ static void query(std::pair<FdSink, FdSource> & pipes) string cmd = tokenized.front(); tokenized.pop_front(); if (cmd == "have") { - writeInt(cmdQueryValidPaths, pipes.first); - writeInt(0, pipes.first); // don't lock - writeInt(0, pipes.first); // don't substitute - writeStrings(tokenized, pipes.first); + pipes.first + << cmdQueryValidPaths + << 0 // don't lock + << 0 // don't substitute + << tokenized; pipes.first.flush(); PathSet paths = readStrings<PathSet>(pipes.second); - foreach (PathSet::iterator, i, paths) - std::cout << *i << std::endl; + for (auto & i : paths) + std::cout << i << std::endl; } else if (cmd == "info") { - writeInt(cmdQueryPathInfos, pipes.first); - writeStrings(tokenized, pipes.first); + pipes.first << cmdQueryPathInfos << tokenized; pipes.first.flush(); while (1) { Path path = readString(pipes.second); @@ -80,8 +79,8 @@ static void query(std::pair<FdSink, FdSource> & pipes) std::cout << deriver << std::endl; PathSet references = readStorePaths<PathSet>(pipes.second); std::cout << references.size() << std::endl; - foreach (PathSet::iterator, i, references) - std::cout << *i << std::endl; + for (auto & i : references) + std::cout << i << std::endl; std::cout << readLongLong(pipes.second) << std::endl; std::cout << readLongLong(pipes.second) << std::endl; } @@ -116,13 +115,13 @@ int main(int argc, char * * argv) std::pair<FdSink, FdSource> pipes = connect(host); /* Exchange the greeting */ - writeInt(SERVE_MAGIC_1, pipes.first); + pipes.first << SERVE_MAGIC_1; pipes.first.flush(); unsigned int magic = readInt(pipes.second); if (magic != SERVE_MAGIC_2) throw Error("protocol mismatch"); readInt(pipes.second); // Server version, unused for now - writeInt(SERVE_PROTOCOL_VERSION, pipes.first); + pipes.first << SERVE_PROTOCOL_VERSION; pipes.first.flush(); string arg = argv[1]; diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index fdd61a5fd375..55379f94b189 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -42,11 +42,10 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Value * v = &vIn; - foreach (Strings::iterator, i, tokens) { + for (auto & attr : tokens) { - /* Is *i an index (integer) or a normal attribute name? */ + /* Is i an index (integer) or a normal attribute name? */ enum { apAttr, apIndex } apType = apAttr; - string attr = *i; unsigned int attrIndex; if (string2Int(attr, attrIndex)) apType = apIndex; @@ -77,15 +76,15 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, else if (apType == apIndex) { - if (v->type != tList) + if (!v->isList()) throw TypeError( format("the expression selected by the selection path ‘%1%’ should be a list but is %2%") % attrPath % showType(*v)); - if (attrIndex >= v->list.length) + if (attrIndex >= v->listSize()) throw Error(format("list index %1% in selection path ‘%2%’ is out of range") % attrIndex % attrPath); - v = v->list.elems[attrIndex]; + v = v->listElems()[attrIndex]; } } diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc new file mode 100644 index 000000000000..910428c02686 --- /dev/null +++ b/src/libexpr/attr-set.cc @@ -0,0 +1,63 @@ +#include "attr-set.hh" +#include "eval.hh" + +#include <algorithm> + + +namespace nix { + + +static void * allocBytes(size_t n) +{ + void * p; +#if HAVE_BOEHMGC + p = GC_malloc(n); +#else + p = malloc(n); +#endif + if (!p) throw std::bad_alloc(); + return p; +} + + +/* Allocate a new array of attributes for an attribute set with a specific + capacity. The space is implicitly reserved after the Bindings + structure. */ +Bindings * EvalState::allocBindings(Bindings::size_t capacity) +{ + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity); +} + + +void EvalState::mkAttrs(Value & v, unsigned int capacity) +{ + if (capacity == 0) { + v = vEmptySet; + return; + } + clearValue(v); + v.type = tAttrs; + v.attrs = allocBindings(capacity); + nrAttrsets++; + nrAttrsInAttrsets += capacity; +} + + +/* Create a new attribute named 'name' on an existing attribute set stored + in 'vAttrs' and return the newly allocated Value which is associated with + this attribute. */ +Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) +{ + Value * v = allocValue(); + vAttrs.attrs->push_back(Attr(name, v)); + return v; +} + + +void Bindings::sort() +{ + std::sort(begin(), end()); +} + + +} diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh new file mode 100644 index 000000000000..7cf6a9c58086 --- /dev/null +++ b/src/libexpr/attr-set.hh @@ -0,0 +1,82 @@ +#pragma once + +#include "nixexpr.hh" +#include "symbol-table.hh" + +#include <algorithm> + +namespace nix { + + +class EvalState; +struct Value; + +/* Map one attribute name to its value. */ +struct Attr +{ + Symbol name; + Value * value; + Pos * pos; + Attr(Symbol name, Value * value, Pos * pos = &noPos) + : name(name), value(value), pos(pos) { }; + Attr() : pos(&noPos) { }; + bool operator < (const Attr & a) const + { + return name < a.name; + } +}; + +/* Bindings contains all the attributes of an attribute set. It is defined + by its size and its capacity, the capacity being the number of Attr + elements allocated after this structure, while the size corresponds to + the number of elements already inserted in this structure. */ +class Bindings +{ +public: + typedef uint32_t size_t; + +private: + size_t size_, capacity_; + Attr attrs[0]; + + Bindings(size_t capacity) : size_(0), capacity_(capacity) { } + Bindings(const Bindings & bindings) = delete; + +public: + size_t size() const { return size_; } + + bool empty() const { return !size_; } + + typedef Attr * iterator; + + void push_back(const Attr & attr) + { + assert(size_ < capacity_); + attrs[size_++] = attr; + } + + iterator find(const Symbol & name) + { + Attr key(name, 0); + iterator i = std::lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return i; + return end(); + } + + iterator begin() { return &attrs[0]; } + iterator end() { return &attrs[size_]; } + + Attr & operator[](size_t pos) + { + return attrs[pos]; + } + + void sort(); + + size_t capacity() { return capacity_; } + + friend class EvalState; +}; + + +} diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 13760490d9c4..8a7989aac663 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -55,7 +55,7 @@ bool parseSearchPathArg(Strings::iterator & i, Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return downloadFileCached(s, true); + return makeDownloader()->downloadCached(state.store, s, true); else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh index be0f40202430..cb2732d6fe7e 100644 --- a/src/libexpr/common-opts.hh +++ b/src/libexpr/common-opts.hh @@ -4,6 +4,8 @@ namespace nix { +class Store; + /* Some common option parsing between nix-env and nix-instantiate. */ bool parseAutoArgs(Strings::iterator & i, const Strings::iterator & argsEnd, std::map<string, string> & res); diff --git a/src/libexpr/download.hh b/src/libexpr/download.hh deleted file mode 100644 index 28c9117e4227..000000000000 --- a/src/libexpr/download.hh +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "types.hh" -#include <string> - -namespace nix { - -struct DownloadResult -{ - bool cached; - string data, etag; -}; - -DownloadResult downloadFile(string url, string expectedETag = ""); - -Path downloadFileCached(const string & url, bool unpack); - -MakeError(DownloadError, Error) - -bool isUri(const string & s); - -} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c275f7ba83e8..0748fbd3f3e1 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -7,9 +7,9 @@ namespace nix { -LocalNoInlineNoReturn(void throwEvalError(const char * s)) +LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) { - throw EvalError(s); + throw EvalError(format(s) % pos); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) @@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const } -void EvalState::forceValue(Value & v) +void EvalState::forceValue(Value & v, const Pos & pos) { if (v.type == tThunk) { Env * env = v.thunk.env; @@ -43,7 +43,7 @@ void EvalState::forceValue(Value & v) else if (v.type == tApp) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered"); + throwEvalError("infinite recursion encountered, at %1%", pos); } @@ -66,7 +66,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos) inline void EvalState::forceList(Value & v) { forceValue(v); - if (v.type != tList) + if (!v.isList()) throwTypeError("value is %1% while a list was expected", v); } @@ -74,9 +74,8 @@ inline void EvalState::forceList(Value & v) inline void EvalState::forceList(Value & v, const Pos & pos) { forceValue(v); - if (v.type != tList) + if (!v.isList()) throwTypeError("value is %1% while a list was expected, at %2%", v, pos); } - } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d61ee7e80795..5a6428ca6b6f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,6 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" +#include "download.hh" #include <algorithm> #include <cstring> @@ -55,14 +56,10 @@ static void * allocBytes(size_t n) } -void Bindings::sort() -{ - std::sort(begin(), end()); -} - - static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) { + checkInterrupt(); + if (active.find(&v) != active.end()) { str << "<CYCLE>"; return; @@ -96,8 +93,8 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con str << "{ "; typedef std::map<string, Value *> Sorted; Sorted sorted; - foreach (Bindings::iterator, i, *v.attrs) - sorted[i->name] = i->value; + for (auto & i : *v.attrs) + sorted[i.name] = i.value; for (auto & i : sorted) { str << i.first << " = "; printValue(str, active, *i.second); @@ -106,10 +103,12 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con str << "}"; break; } - case tList: + case tList1: + case tList2: + case tListN: str << "[ "; - for (unsigned int n = 0; n < v.list.length; ++n) { - printValue(str, active, *v.list.elems[n]); + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValue(str, active, *v.listElems()[n]); str << " "; } str << "]"; @@ -130,6 +129,9 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con case tExternal: str << *v.external; break; + case tFloat: + str << v.fpoint; + break; default: throw Error("invalid value"); } @@ -155,7 +157,7 @@ string showType(const Value & v) case tPath: return "a path"; case tNull: return "null"; case tAttrs: return "a set"; - case tList: return "a list"; + case tList1: case tList2: case tListN: return "a list"; case tThunk: return "a thunk"; case tApp: return "a function application"; case tLambda: return "a function"; @@ -163,6 +165,7 @@ string showType(const Value & v) case tPrimOp: return "a built-in function"; case tPrimOpApp: return "a partially applied built-in function"; case tExternal: return v.external->showType(); + case tFloat: return "a float"; } abort(); } @@ -236,17 +239,43 @@ void initGC() /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const string & in) +static Strings parseNixPath(const string & s) { - string marker = "\001//"; - auto res = tokenizeString<Strings>(replaceStrings(in, "://", marker), ":"); - for (auto & s : res) - s = replaceStrings(s, marker, "://"); + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + return res; } -EvalState::EvalState(const Strings & _searchPath) +EvalState::EvalState(const Strings & _searchPath, ref<Store> store) : sWith(symbols.create("<with>")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -263,14 +292,11 @@ EvalState::EvalState(const Strings & _searchPath) , sLine(symbols.create("line")) , sColumn(symbols.create("column")) , sFunctor(symbols.create("__functor")) - , repair(false) + , sToString(symbols.create("__toString")) + , store(store) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) - , baseEnvDispl(0) { - nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0; - nrAttrsets = nrAttrsInAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0; - nrListConcats = nrPrimOpCalls = nrFunctionCalls = 0; countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; restricted = settings.get("restrict-eval", false); @@ -279,10 +305,14 @@ EvalState::EvalState(const Strings & _searchPath) /* Initialise the Nix expression search path. */ Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i, true); + for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); + clearValue(vEmptySet); + vEmptySet.type = tAttrs; + vEmptySet.attrs = allocBindings(0); + createBaseEnv(); } @@ -298,11 +328,15 @@ Path EvalState::checkSourcePath(const Path & path_) if (!restricted) return path_; /* Resolve symlinks. */ + debug(format("checking access to ‘%s’") % path_); Path path = canonPath(path_, true); - for (auto & i : searchPath) - if (path == i.second || isInDir(path, i.second)) + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + if (path == r.second || isInDir(path, r.second)) return path; + } /* To support import-from-derivation, allow access to anything in the store. FIXME: only allow access to paths that have been @@ -361,11 +395,6 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) throw EvalError(format(s) % s2); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) -{ - throw EvalError(format(s) % pos); -} - LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) { throw EvalError(format(s) % s2 % pos); @@ -440,8 +469,8 @@ void mkString(Value & v, const string & s, const PathSet & context) unsigned int n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); - foreach (PathSet::const_iterator, i, context) - v.string.context[n++] = dupString(i->c_str()); + for (auto & i : context) + v.string.context[n++] = dupString(i.c_str()); v.string.context[n] = 0; } } @@ -503,37 +532,19 @@ Env & EvalState::allocEnv(unsigned int size) } -Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) -{ - Value * v = allocValue(); - vAttrs.attrs->push_back(Attr(name, v)); - return v; -} - - -Bindings * EvalState::allocBindings(Bindings::size_t capacity) -{ - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(capacity); -} - - -void EvalState::mkList(Value & v, unsigned int length) -{ - clearValue(v); - v.type = tList; - v.list.length = length; - v.list.elems = length ? (Value * *) allocBytes(length * sizeof(Value *)) : 0; - nrListElems += length; -} - - -void EvalState::mkAttrs(Value & v, unsigned int expected) +void EvalState::mkList(Value & v, unsigned int size) { clearValue(v); - v.type = tAttrs; - v.attrs = allocBindings(expected); - nrAttrsets++; - nrAttrsInAttrsets += expected; + if (size == 1) + v.type = tList1; + else if (size == 2) + v.type = tList2; + else { + v.type = tListN; + v.bigList.size = size; + v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; + } + nrListElems += size; } @@ -603,6 +614,12 @@ Value * ExprInt::maybeThunk(EvalState & state, Env & env) return &v; } +Value * ExprFloat::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + Value * ExprPath::maybeThunk(EvalState & state, Env & env) { nrAvoided++; @@ -624,7 +641,7 @@ void EvalState::evalFile(const Path & path, Value & v) return; } - startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path2); + Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2); Expr * e = parseExprFromFile(checkSourcePath(path2)); try { eval(e, v); @@ -690,6 +707,11 @@ void ExprInt::eval(EvalState & state, Env & env, Value & v) } +void ExprFloat::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + void ExprString::eval(EvalState & state, Env & env, Value & v) { v = this->v; @@ -721,15 +743,15 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) environment, while the inherited attributes are evaluated in the original environment. */ unsigned int displ = 0; - foreach (AttrDefs::iterator, i, attrs) { + for (auto & i : attrs) { Value * vAttr; - if (hasOverrides && !i->second.inherited) { + if (hasOverrides && !i.second.inherited) { vAttr = state.allocValue(); - mkThunk(*vAttr, env2, i->second.e); + mkThunk(*vAttr, env2, i.second.e); } else - vAttr = i->second.e->maybeThunk(state, i->second.inherited ? env : env2); + vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos)); + v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos)); } /* If the rec contains an attribute called `__overrides', then @@ -760,25 +782,25 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) } else - foreach (AttrDefs::iterator, i, attrs) - v.attrs->push_back(Attr(i->first, i->second.e->maybeThunk(state, env), &i->second.pos)); + for (auto & i : attrs) + v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); /* Dynamic attrs apply *after* rec and __overrides. */ - foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) { + for (auto & i : dynamicAttrs) { Value nameVal; - i->nameExpr->eval(state, *dynamicEnv, nameVal); - state.forceValue(nameVal); + i.nameExpr->eval(state, *dynamicEnv, nameVal); + state.forceValue(nameVal, i.pos); if (nameVal.type == tNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - throwEvalError("dynamic attribute ‘%1%’ at %2% already defined at %3%", nameSym, i->pos, *j->pos); + throwEvalError("dynamic attribute ‘%1%’ at %2% already defined at %3%", nameSym, i.pos, *j->pos); - i->valueExpr->setName(nameSym); + i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i->valueExpr->maybeThunk(state, *dynamicEnv), &i->pos)); + v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); v.attrs->sort(); // FIXME: inefficient } } @@ -795,8 +817,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) while the inherited attributes are evaluated in the original environment. */ unsigned int displ = 0; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - env2.values[displ++] = i->second.e->maybeThunk(state, i->second.inherited ? env : env2); + for (auto & i : attrs->attrs) + env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); body->eval(state, env2, v); } @@ -805,15 +827,15 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (unsigned int n = 0; n < v.list.length; ++n) - v.list.elems[n] = elems[n]->maybeThunk(state, env); + for (unsigned int n = 0; n < elems.size(); ++n) + v.listElems()[n] = elems[n]->maybeThunk(state, env); } void ExprVar::eval(EvalState & state, Env & env, Value & v) { Value * v2 = state.lookupVar(&env, *this, false); - state.forceValue(*v2); + state.forceValue(*v2, pos); v = *v2; } @@ -847,12 +869,12 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) try { - foreach (AttrPath::const_iterator, i, attrPath) { + for (auto & i : attrPath) { nrLookups++; Bindings::iterator j; - Symbol name = getName(*i, state, env); + Symbol name = getName(i, state, env); if (def) { - state.forceValue(*vAttrs); + state.forceValue(*vAttrs, pos); if (vAttrs->type != tAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { @@ -869,7 +891,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) if (state.countCalls && pos2) state.attrSelects[*pos2]++; } - state.forceValue(*vAttrs); + state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) ); } catch (Error & e) { if (pos2 && pos2->file != state.sDerivationNix) @@ -889,10 +911,10 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) e->eval(state, env, vTmp); - foreach (AttrPath::const_iterator, i, attrPath) { + for (auto & i : attrPath) { state.forceValue(*vAttrs); Bindings::iterator j; - Symbol name = getName(*i, state, env); + Symbol name = getName(i, state, env); if (vAttrs->type != tAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { @@ -971,10 +993,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po if (fun.type == tAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { - forceValue(*found->value); + forceValue(*found->value, pos); Value * v2 = allocValue(); callFunction(*found->value, fun, *v2, pos); - forceValue(*v2); + forceValue(*v2, pos); return callFunction(*v2, arg, v, pos); } } @@ -1005,12 +1027,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po there is no matching actual argument but the formal argument has a default, use the default. */ unsigned int attrsUsed = 0; - foreach (Formals::Formals_::iterator, i, lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i->name); + for (auto & i : lambda.formals->formals) { + Bindings::iterator j = arg.attrs->find(i.name); if (j == arg.attrs->end()) { - if (!i->def) throwTypeError("%1% called without required argument ‘%2%’, at %3%", - lambda, i->name, pos); - env2.values[displ++] = i->def->maybeThunk(*this, env2); + if (!i.def) throwTypeError("%1% called without required argument ‘%2%’, at %3%", + lambda, i.name, pos); + env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; env2.values[displ++] = j->value; @@ -1022,9 +1044,9 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { /* Nope, so show the first unexpected argument to the user. */ - foreach (Bindings::iterator, i, *arg.attrs) - if (lambda.formals->argNames.find(i->name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument ‘%2%’, at %3%", lambda, i->name, pos); + for (auto & i : *arg.attrs) + if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + throwTypeError("%1% called with unexpected argument ‘%2%’, at %3%", lambda, i.name, pos); abort(); // can't happen } } @@ -1058,6 +1080,17 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + forceValue(*found->value); + Value * v = allocValue(); + callFunction(*found->value, fun, *v, noPos); + forceValue(*v); + return autoCallFunction(args, *v, res); + } + } + if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { res = fun; return; @@ -1066,12 +1099,12 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) Value * actualArgs = allocValue(); mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); - foreach (Formals::Formals_::iterator, i, fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i->name); + for (auto & i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args.find(i.name); if (j != args.end()) actualArgs->attrs->push_back(*j); - else if (!i->def) - throwTypeError("cannot auto-call a function that has an argument without a default value (‘%1%’)", i->name); + else if (!i.def) + throwTypeError("cannot auto-call a function that has an argument without a default value (‘%1%’)", i.name); } actualArgs->attrs->sort(); @@ -1199,20 +1232,21 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co unsigned int len = 0; for (unsigned int n = 0; n < nrLists; ++n) { forceList(*lists[n], pos); - unsigned int l = lists[n]->list.length; + unsigned int l = lists[n]->listSize(); len += l; if (l) nonEmpty = lists[n]; } - if (nonEmpty && len == nonEmpty->list.length) { + if (nonEmpty && len == nonEmpty->listSize()) { v = *nonEmpty; return; } mkList(v, len); + auto out = v.listElems(); for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { - unsigned int l = lists[n]->list.length; - memcpy(v.list.elems + pos, lists[n]->list.elems, l * sizeof(Value *)); + unsigned int l = lists[n]->listSize(); + memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); pos += l; } } @@ -1223,13 +1257,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) PathSet context; std::ostringstream s; NixInt n = 0; + NixFloat nf = 0; bool first = !forceString; ValueType firstType = tString; - foreach (vector<Expr *>::iterator, i, *es) { + for (auto & i : *es) { Value vTmp; - (*i)->eval(state, env, vTmp); + i->eval(state, env, vTmp); /* If the first element is a path, then the result will also be a path, we don't copy anything (yet - that's done later, @@ -1241,15 +1276,30 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) } if (firstType == tInt) { - if (vTmp.type != tInt) + if (vTmp.type == tInt) { + n += vTmp.integer; + } else if (vTmp.type == tFloat) { + // Upgrade the type from int to float; + firstType = tFloat; + nf = n; + nf += vTmp.fpoint; + } else throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); - n += vTmp.integer; + } else if (firstType == tFloat) { + if (vTmp.type == tInt) { + nf += vTmp.integer; + } else if (vTmp.type == tFloat) { + nf += vTmp.fpoint; + } else + throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); } else s << state.coerceToString(pos, vTmp, context, false, firstType == tString); } if (firstType == tInt) mkInt(v, n); + else if (firstType == tFloat) + mkFloat(v, nf); else if (firstType == tPath) { if (!context.empty()) throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); @@ -1288,9 +1338,9 @@ void EvalState::forceValueDeep(Value & v) } } - else if (v.type == tList) { - for (unsigned int n = 0; n < v.list.length; ++n) - recurse(*v.list.elems[n]); + else if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) + recurse(*v.listElems()[n]); } }; @@ -1300,13 +1350,24 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tInt) throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); return v.integer; } +NixFloat EvalState::forceFloat(Value & v, const Pos & pos) +{ + forceValue(v, pos); + if (v.type == tInt) + return v.integer; + else if (v.type != tFloat) + throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + return v.fpoint; +} + + bool EvalState::forceBool(Value & v) { forceValue(v); @@ -1316,17 +1377,23 @@ bool EvalState::forceBool(Value & v) } +bool EvalState::isFunctor(Value & fun) +{ + return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); +} + + void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp) + if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) throwTypeError("value is %1% while a function was expected, at %2%", v, pos); } string EvalState::forceString(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tString) { if (pos) throwTypeError("value is %1% while a string was expected, at %2%", v, pos); @@ -1397,7 +1464,14 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } if (v.type == tAttrs) { - Bindings::iterator i = v.attrs->find(sOutPath); + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + forceValue(*i->value, pos); + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + i = v.attrs->find(sOutPath); if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } @@ -1411,17 +1485,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, shell scripting convenience, just like `null'. */ if (v.type == tBool && v.boolean) return "1"; if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return int2String(v.integer); + if (v.type == tInt) return std::to_string(v.integer); + if (v.type == tFloat) return std::to_string(v.fpoint); if (v.type == tNull) return ""; - if (v.type == tList) { + if (v.isList()) { string result; - for (unsigned int n = 0; n < v.list.length; ++n) { - result += coerceToString(pos, *v.list.elems[n], + for (unsigned int n = 0; n < v.listSize(); ++n) { + result += coerceToString(pos, *v.listElems()[n], context, coerceMore, copyToStore); - if (n < v.list.length - 1 + if (n < v.listSize() - 1 /* !!! not quite correct */ - && (v.list.elems[n]->type != tList || v.list.elems[n]->list.length != 0)) + && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) result += " "; } return result; @@ -1473,6 +1548,13 @@ bool EvalState::eqValues(Value & v1, Value & v2) uniqList on a list of sets.) Will remove this eventually. */ if (&v1 == &v2) return true; + // Special case type-compatibility between float and int + if (v1.type == tInt && v2.type == tFloat) + return v1.integer == v2.fpoint; + if (v1.type == tFloat && v2.type == tInt) + return v1.fpoint == v2.integer; + + // All other types are not compatible with each other. if (v1.type != v2.type) return false; switch (v1.type) { @@ -1492,10 +1574,12 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tNull: return true; - case tList: - if (v1.list.length != v2.list.length) return false; - for (unsigned int n = 0; n < v1.list.length; ++n) - if (!eqValues(*v1.list.elems[n], *v2.list.elems[n])) return false; + case tList1: + case tList2: + case tListN: + if (v1.listSize() != v2.listSize()) return false; + for (unsigned int n = 0; n < v1.listSize(); ++n) + if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; case tAttrs: { @@ -1528,6 +1612,9 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tExternal: return *v1.external == *v2.external; + case tFloat: + return v1.fpoint == v2.fpoint; + default: throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); } @@ -1581,25 +1668,25 @@ void EvalState::printStats() printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); typedef std::multimap<unsigned int, Symbol> PrimOpCalls_; PrimOpCalls_ primOpCalls_; - foreach (PrimOpCalls::iterator, i, primOpCalls) - primOpCalls_.insert(std::pair<unsigned int, Symbol>(i->second, i->first)); - foreach_reverse (PrimOpCalls_::reverse_iterator, i, primOpCalls_) + for (auto & i : primOpCalls) + primOpCalls_.insert(std::pair<unsigned int, Symbol>(i.second, i.first)); + for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i) printMsg(v, format("%1$10d %2%") % i->first % i->second); printMsg(v, format("calls to %1% functions:") % functionCalls.size()); typedef std::multimap<unsigned int, ExprLambda *> FunctionCalls_; FunctionCalls_ functionCalls_; - foreach (FunctionCalls::iterator, i, functionCalls) - functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i->second, i->first)); - foreach_reverse (FunctionCalls_::reverse_iterator, i, functionCalls_) + for (auto & i : functionCalls) + functionCalls_.insert(std::pair<unsigned int, ExprLambda *>(i.second, i.first)); + for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i) printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos()); printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); typedef std::multimap<unsigned int, Pos> AttrSelects_; AttrSelects_ attrSelects_; - foreach (AttrSelects::iterator, i, attrSelects) - attrSelects_.insert(std::pair<unsigned int, Pos>(i->second, i->first)); - foreach_reverse (AttrSelects_::reverse_iterator, i, attrSelects_) + for (auto & i : attrSelects) + attrSelects_.insert(std::pair<unsigned int, Pos>(i.second, i.first)); + for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i) printMsg(v, format("%1$10d %2%") % i->first % i->second); } @@ -1643,12 +1730,14 @@ size_t valueSize(Value & v) sz += doValue(*i.value); } break; - case tList: - if (seen.find(v.list.elems) == seen.end()) { - seen.insert(v.list.elems); - sz += v.list.length * sizeof(Value *); - for (unsigned int n = 0; n < v.list.length; ++n) - sz += doValue(*v.list.elems[n]); + case tList1: + case tList2: + case tListN: + if (seen.find(v.listElems()) == seen.end()) { + seen.insert(v.listElems()); + sz += v.listSize() * sizeof(Value *); + for (unsigned int n = 0; n < v.listSize(); ++n) + sz += doValue(*v.listElems()[n]); } break; case tThunk: diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 627fae3ff363..80e369f2d68f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,5 +1,6 @@ #pragma once +#include "attr-set.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" @@ -15,82 +16,19 @@ namespace nix { +class Store; class EvalState; -struct Attr -{ - Symbol name; - Value * value; - Pos * pos; - Attr(Symbol name, Value * value, Pos * pos = &noPos) - : name(name), value(value), pos(pos) { }; - Attr() : pos(&noPos) { }; - bool operator < (const Attr & a) const - { - return name < a.name; - } -}; - - -class Bindings -{ -public: - typedef uint32_t size_t; - -private: - size_t size_, capacity_; - Attr attrs[0]; - - Bindings(size_t capacity) : size_(0), capacity_(capacity) { } - Bindings(const Bindings & bindings) = delete; - -public: - size_t size() const { return size_; } - - bool empty() const { return !size_; } - - typedef Attr * iterator; - - void push_back(const Attr & attr) - { - assert(size_ < capacity_); - attrs[size_++] = attr; - } - - iterator find(const Symbol & name) - { - Attr key(name, 0); - iterator i = std::lower_bound(begin(), end(), key); - if (i != end() && i->name == name) return i; - return end(); - } - - iterator begin() { return &attrs[0]; } - iterator end() { return &attrs[size_]; } - - Attr & operator[](size_t pos) - { - return attrs[pos]; - } - - void sort(); - - size_t capacity() { return capacity_; } - - friend class EvalState; -}; - - typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); struct PrimOp { PrimOpFun fun; - unsigned int arity; + size_t arity; Symbol name; - PrimOp(PrimOpFun fun, unsigned int arity, Symbol name) + PrimOp(PrimOpFun fun, size_t arity, Symbol name) : fun(fun), arity(arity), name(name) { } }; @@ -118,7 +56,8 @@ typedef std::map<Path, Path> SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); -typedef list<std::pair<string, Path> > SearchPath; +typedef std::pair<std::string, std::string> SearchPathElem; +typedef std::list<SearchPathElem> SearchPath; /* Initialise the Boehm GC, if applicable. */ @@ -132,17 +71,21 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, - sFile, sLine, sColumn, sFunctor; + sFile, sLine, sColumn, sFunctor, sToString; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they already exist there. */ - bool repair; + bool repair = false; /* If set, don't allow access to files outside of the Nix search path or to environment variables. */ bool restricted; + Value vEmptySet; + + const ref<Store> store; + private: SrcToStore srcToStore; @@ -156,12 +99,14 @@ private: SearchPath searchPath; + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + public: - EvalState(const Strings & _searchPath); + EvalState(const Strings & _searchPath, ref<Store> store); ~EvalState(); - void addToSearchPath(const string & s, bool warn = false); + void addToSearchPath(const string & s); Path checkSourcePath(const Path & path); @@ -183,6 +128,9 @@ public: Path findFile(const string & path); Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); + /* Evaluate an expression to normal form, storing the result in value `v'. */ void eval(Expr * e, Value & v); @@ -197,7 +145,7 @@ public: of the evaluation of the thunk. If `v' is a delayed function application, call the function and overwrite `v' with the result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v); + inline void forceValue(Value & v, const Pos & pos = noPos); /* Force a value, then recursively force list elements and attributes. */ @@ -205,6 +153,7 @@ public: /* Force `v', and then verify that it has the expected type. */ NixInt forceInt(Value & v, const Pos & pos); + NixFloat forceFloat(Value & v, const Pos & pos); bool forceBool(Value & v); inline void forceAttrs(Value & v); inline void forceAttrs(Value & v, const Pos & pos); @@ -244,7 +193,7 @@ public: private: - unsigned int baseEnvDispl; + unsigned int baseEnvDispl = 0; void createBaseEnv(); @@ -274,6 +223,8 @@ public: elements and attributes are compared recursively. */ bool eqValues(Value & v1, Value & v2); + bool isFunctor(Value & fun); + void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); @@ -290,7 +241,7 @@ public: Bindings * allocBindings(Bindings::size_t capacity); void mkList(Value & v, unsigned int length); - void mkAttrs(Value & v, unsigned int expected); + void mkAttrs(Value & v, unsigned int capacity); void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, Pos * pos); @@ -299,19 +250,21 @@ public: /* Print statistics. */ void printStats(); + void realiseContext(const PathSet & context); + private: - unsigned long nrEnvs; - unsigned long nrValuesInEnvs; - unsigned long nrValues; - unsigned long nrListElems; - unsigned long nrAttrsets; - unsigned long nrAttrsInAttrsets; - unsigned long nrOpUpdates; - unsigned long nrOpUpdateValuesCopied; - unsigned long nrListConcats; - unsigned long nrPrimOpCalls; - unsigned long nrFunctionCalls; + unsigned long nrEnvs = 0; + unsigned long nrValuesInEnvs = 0; + unsigned long nrValues = 0; + unsigned long nrListElems = 0; + unsigned long nrAttrsets = 0; + unsigned long nrAttrsInAttrsets = 0; + unsigned long nrOpUpdates = 0; + unsigned long nrOpUpdateValuesCopied = 0; + unsigned long nrListConcats = 0; + unsigned long nrPrimOpCalls = 0; + unsigned long nrFunctionCalls = 0; bool countCalls; @@ -349,7 +302,4 @@ struct InvalidPathError : EvalError #endif }; -/* Realise all paths in `context' */ -void realiseContext(const PathSet & context); - } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1c9fa02a366a..b06c539de0fb 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -30,7 +30,7 @@ string DrvInfo::queryOutPath() } -DrvInfo::Outputs DrvInfo::queryOutputs() +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ @@ -39,9 +39,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs() state->forceList(*i->value, *i->pos); /* For each output... */ - for (unsigned int j = 0; j < i->value->list.length; ++j) { + for (unsigned int j = 0; j < i->value->listSize(); ++j) { /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->list.elems[j], *i->pos); + string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); Bindings::iterator out = attrs->find(state->symbols.create(name)); if (out == attrs->end()) continue; // FIXME: throw error? state->forceAttrs(*out->value); @@ -55,7 +55,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs() } else outputs["out"] = queryOutPath(); } - return outputs; + if (!onlyOutputsToInstall || !attrs) + return outputs; + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value * outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + const auto errMsg = Error("this derivation has bad ‘meta.outputsToInstall’"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { + if ((*i)->type != tString) throw errMsg; + auto out = outputs.find((*i)->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); + } + return result; } @@ -85,8 +101,8 @@ StringSet DrvInfo::queryMetaNames() { StringSet res; if (!getMeta()) return res; - foreach (Bindings::iterator, i, *meta) - res.insert(i->name); + for (auto & i : *meta) + res.insert(i.name); return res; } @@ -94,19 +110,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.type == tList) { - for (unsigned int n = 0; n < v.list.length; ++n) - if (!checkMeta(*v.list.elems[n])) return false; + if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) + if (!checkMeta(*v.listElems()[n])) return false; return true; } else if (v.type == tAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; - foreach (Bindings::iterator, i, *v.attrs) - if (!checkMeta(*i->value)) return false; + for (auto & i : *v.attrs) + if (!checkMeta(*i.value)) return false; return true; } - else return v.type == tInt || v.type == tBool || v.type == tString; + else return v.type == tInt || v.type == tBool || v.type == tString || + v.type == tFloat; } @@ -127,7 +144,7 @@ string DrvInfo::queryMetaString(const string & name) } -int DrvInfo::queryMetaInt(const string & name, int def) +NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; @@ -135,12 +152,26 @@ int DrvInfo::queryMetaInt(const string & name, int def) if (v->type == tString) { /* Backwards compatibility with before we had support for integer meta fields. */ - int n; + NixInt n; if (string2Int(v->string.s, n)) return n; } return def; } +NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) +{ + Value * v = queryMeta(name); + if (!v) return def; + if (v->type == tFloat) return v->fpoint; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + float meta fields. */ + NixFloat n; + if (string2Float(v->string.s, n)) return n; + } + return def; +} + bool DrvInfo::queryMetaBool(const string & name, bool def) { @@ -177,8 +208,8 @@ typedef set<Bindings *> Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', - then put information about it in `drvs' (unless it's already in - `doneExprs'). The result boolean indicates whether it makes sense + then put information about it in `drvs' (unless it's already in `done'). + The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ static bool getDerivation(EvalState & state, Value & v, const string & attrPath, DrvInfos & drvs, Done & done, @@ -255,13 +286,13 @@ static void getDerivations(EvalState & state, Value & vIn, precedence). */ typedef std::map<string, Symbol> SortedSymbols; SortedSymbols attrs; - foreach (Bindings::iterator, i, *v.attrs) - attrs.insert(std::pair<string, Symbol>(i->name, i->name)); + for (auto & i : *v.attrs) + attrs.insert(std::pair<string, Symbol>(i.name, i.name)); - foreach (SortedSymbols::iterator, i, attrs) { - startNest(nest, lvlDebug, format("evaluating attribute ‘%1%’") % i->first); - string pathPrefix2 = addToPath(pathPrefix, i->first); - Value & v2(*v.attrs->find(i->second)->value); + for (auto & i : attrs) { + Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); + string pathPrefix2 = addToPath(pathPrefix, i.first); + Value & v2(*v.attrs->find(i.second)->value); if (combineChannels) getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) { @@ -277,13 +308,12 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.type == tList) { - for (unsigned int n = 0; n < v.list.length; ++n) { - startNest(nest, lvlDebug, - format("evaluating list element")); + else if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + Activity act(*logger, lvlDebug, "evaluating list element"); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.list.elems[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.list.elems[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 98f762494aa5..37fcbe829d3c 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -42,12 +42,14 @@ public: string queryDrvPath(); string queryOutPath(); string queryOutputName(); - Outputs queryOutputs(); + /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); Value * queryMeta(const string & name); string queryMetaString(const string & name); - int queryMetaInt(const string & name, int def); + NixInt queryMetaInt(const string & name, NixInt def); + NixFloat queryMetaFloat(const string & name, NixFloat def); bool queryMetaBool(const string & name, bool def); void setMeta(const string & name, Value * v); diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 1892b0bac1af..1daf84600dca 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -73,7 +73,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) s++; state.mkList(v, values.size()); for (size_t n = 0; n < values.size(); ++n) - v.list.elems[n] = values[n]; + v.listElems()[n] = values[n]; } else if (*s == '{') { @@ -105,17 +105,21 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) mkString(v, parseJSONString(s)); } - else if (isdigit(*s) || *s == '-') { - bool neg = false; - if (*s == '-') { - neg = true; - if (!*++s) throw JSONParseError("unexpected end of JSON number"); + else if (isdigit(*s) || *s == '-' || *s == '.' ) { + // Buffer into a string first, then use built-in C++ conversions + std::string tmp_number; + ValueType number_type = tInt; + + while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { + if (*s == '.' || *s == 'e' || *s == 'E') + number_type = tFloat; + tmp_number += *s++; } - NixInt n = 0; - // FIXME: detect overflow - while (isdigit(*s)) n = n * 10 + (*s++ - '0'); - if (*s == '.' || *s == 'e') throw JSONParseError("floating point JSON numbers are not supported"); - mkInt(v, neg ? -n : n); + + if (number_type == tFloat) + mkFloat(v, stod(tmp_number)); + else + mkInt(v, stoi(tmp_number)); } else if (strncmp(s, "true", 4) == 0) { diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 7051909008d1..f3660ab43723 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -1,10 +1,14 @@ %option reentrant bison-bridge bison-locations %option noyywrap %option never-interactive +%option stack +%option nodefault +%option nounput noyy_top_state %x STRING %x IND_STRING +%x INSIDE_DOLLAR_CURLY %{ @@ -74,11 +78,15 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s) #define YY_USER_INIT initLoc(yylloc) #define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); +#define PUSH_STATE(state) yy_push_state(state, yyscanner) +#define POP_STATE() yy_pop_state(yyscanner) + %} ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* INT [0-9]+ +FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+ HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+ SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> @@ -87,6 +95,8 @@ URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~ %% +<INITIAL,INSIDE_DOLLAR_CURLY>{ + if { return IF; } then { return THEN; } @@ -117,24 +127,35 @@ or { return OR_KW; } throw ParseError(format("invalid integer ‘%1%’") % yytext); return INT; } +{FLOAT} { errno = 0; + yylval->nf = strtod(yytext, 0); + if (errno != 0) + throw ParseError(format("invalid float ‘%1%’") % yytext); + return FLOAT; + } -\$\{ { return DOLLAR_CURLY; } +\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +} -\" { BEGIN(STRING); return '"'; } -<STRING>([^\$\"\\]|\$[^\{\"]|\\.)+ { - /* !!! Not quite right: we want a follow restriction on - "$", it shouldn't be followed by a "{". Right now - "$\"" will be consumed as part of a string, rather - than a "$" followed by the string terminator. - Disallow "$\"" for now. */ +\} { return '}'; } +<INSIDE_DOLLAR_CURLY>\} { POP_STATE(); return '}'; } +\{ { return '{'; } +<INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } + +<INITIAL,INSIDE_DOLLAR_CURLY>\" { PUSH_STATE(STRING); return '"'; } +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ yylval->e = unescapeStr(data->symbols, yytext); return STR; } -<STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; } -<STRING>\" { BEGIN(INITIAL); return '"'; } +<STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<STRING>\" { POP_STATE(); return '"'; } <STRING>. return yytext[0]; /* just in case: shouldn't be reached */ -\'\'(\ *\n)? { BEGIN(IND_STRING); return IND_STRING_OPEN; } +<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; @@ -151,14 +172,16 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext + 2); return IND_STR; } -<IND_STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; } -<IND_STRING>\'\' { BEGIN(INITIAL); return IND_STRING_CLOSE; } +<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } <IND_STRING>\' { yylval->e = new ExprIndStr("'"); return IND_STR; } <IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ +<INITIAL,INSIDE_DOLLAR_CURLY>{ + {PATH} { yylval->path = strdup(yytext); return PATH; } {HPATH} { yylval->path = strdup(yytext); return HPATH; } {SPATH} { yylval->path = strdup(yytext); return SPATH; } @@ -170,26 +193,9 @@ or { return OR_KW; } . return yytext[0]; - -%% - - -namespace nix { - -/* Horrible, disgusting hack: allow the parser to set the scanner - start condition back to STRING. Necessary in interpolations like - "foo${expr}bar"; after the close brace we have to go back to the - STRING state. */ -void backToString(yyscan_t scanner) -{ - struct yyguts_t * yyg = (struct yyguts_t *) scanner; - BEGIN(STRING); } -void backToIndString(yyscan_t scanner) -{ - struct yyguts_t * yyg = (struct yyguts_t *) scanner; - BEGIN(IND_STRING); -} +<<EOF>> { data->atEnd = true; return 0; } + +%% -} diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 35e84980a6dd..620050a13b05 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,11 +4,16 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc + +libexpr_CXXFLAGS := -Wno-deprecated-register libexpr_LIBS = libutil libstore libformat -libexpr_LDFLAGS = -ldl -lcurl +libexpr_LDFLAGS = +ifneq ($(OS), FreeBSD) + libexpr_LDFLAGS += -ldl +endif # The dependency on libgc must be propagated (i.e. meaning that # programs/libraries that use libexpr must explicitly pass -lgc), diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc index cda5aa1952ea..7bca9b6550be 100644 --- a/src/libexpr/names.cc +++ b/src/libexpr/names.cc @@ -98,8 +98,8 @@ int compareVersions(const string & v1, const string & v2) DrvNames drvNamesFromArgs(const Strings & opArgs) { DrvNames result; - foreach (Strings::const_iterator, i, opArgs) - result.push_back(DrvName(*i)); + for (auto & i : opArgs) + result.push_back(DrvName(i)); return result; } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 43f3161f8baf..b2c9f0528ca9 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -30,8 +30,9 @@ static void showString(std::ostream & str, const string & s) static void showId(std::ostream & str, const string & s) { - assert(!s.empty()); - if (s == "if") + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords str << '"' << s << '"'; else { char c = s[0]; @@ -67,6 +68,11 @@ void ExprInt::show(std::ostream & str) str << n; } +void ExprFloat::show(std::ostream & str) +{ + str << nf; +} + void ExprString::show(std::ostream & str) { showString(str, s); @@ -97,21 +103,21 @@ void ExprAttrs::show(std::ostream & str) { if (recursive) str << "rec "; str << "{ "; - foreach (AttrDefs::iterator, i, attrs) - if (i->second.inherited) - str << "inherit " << i->first << " " << "; "; + for (auto & i : attrs) + if (i.second.inherited) + str << "inherit " << i.first << " " << "; "; else - str << i->first << " = " << *i->second.e << "; "; - foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) - str << "\"${" << *i->nameExpr << "}\" = " << *i->valueExpr << "; "; + str << i.first << " = " << *i.second.e << "; "; + for (auto & i : dynamicAttrs) + str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; str << "}"; } void ExprList::show(std::ostream & str) { str << "[ "; - foreach (vector<Expr *>::iterator, i, elems) - str << "(" << **i << ") "; + for (auto & i : elems) + str << "(" << *i << ") "; str << "]"; } @@ -121,10 +127,10 @@ void ExprLambda::show(std::ostream & str) if (matchAttrs) { str << "{ "; bool first = true; - foreach (Formals::Formals_::iterator, i, formals->formals) { + for (auto & i : formals->formals) { if (first) first = false; else str << ", "; - str << i->name; - if (i->def) str << " ? " << *i->def; + str << i.name; + if (i.def) str << " ? " << *i.def; } if (formals->ellipsis) { if (!first) str << ", "; @@ -140,12 +146,12 @@ void ExprLambda::show(std::ostream & str) void ExprLet::show(std::ostream & str) { str << "(let "; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - if (i->second.inherited) { - str << "inherit " << i->first << "; "; + for (auto & i : attrs->attrs) + if (i.second.inherited) { + str << "inherit " << i.first << "; "; } else - str << i->first << " = " << *i->second.e << "; "; + str << i.first << " = " << *i.second.e << "; "; str << "in " << *body << ")"; } @@ -173,9 +179,9 @@ void ExprConcatStrings::show(std::ostream & str) { bool first = true; str << "("; - foreach (vector<Expr *>::iterator, i, *es) { + for (auto & i : *es) { if (first) first = false; else str << " + "; - str << **i; + str << *i; } str << ")"; } @@ -225,6 +231,10 @@ void ExprInt::bindVars(const StaticEnv & env) { } +void ExprFloat::bindVars(const StaticEnv & env) +{ +} + void ExprString::bindVars(const StaticEnv & env) { } @@ -267,17 +277,17 @@ void ExprSelect::bindVars(const StaticEnv & env) { e->bindVars(env); if (def) def->bindVars(env); - foreach (AttrPath::iterator, i, attrPath) - if (!i->symbol.set()) - i->expr->bindVars(env); + for (auto & i : attrPath) + if (!i.symbol.set()) + i.expr->bindVars(env); } void ExprOpHasAttr::bindVars(const StaticEnv & env) { e->bindVars(env); - foreach (AttrPath::iterator, i, attrPath) - if (!i->symbol.set()) - i->expr->bindVars(env); + for (auto & i : attrPath) + if (!i.symbol.set()) + i.expr->bindVars(env); } void ExprAttrs::bindVars(const StaticEnv & env) @@ -289,27 +299,27 @@ void ExprAttrs::bindVars(const StaticEnv & env) dynamicEnv = &newEnv; unsigned int displ = 0; - foreach (AttrDefs::iterator, i, attrs) - newEnv.vars[i->first] = i->second.displ = displ++; + for (auto & i : attrs) + newEnv.vars[i.first] = i.second.displ = displ++; - foreach (AttrDefs::iterator, i, attrs) - i->second.e->bindVars(i->second.inherited ? env : newEnv); + for (auto & i : attrs) + i.second.e->bindVars(i.second.inherited ? env : newEnv); } else - foreach (AttrDefs::iterator, i, attrs) - i->second.e->bindVars(env); + for (auto & i : attrs) + i.second.e->bindVars(env); - foreach (DynamicAttrDefs::iterator, i, dynamicAttrs) { - i->nameExpr->bindVars(*dynamicEnv); - i->valueExpr->bindVars(*dynamicEnv); + for (auto & i : dynamicAttrs) { + i.nameExpr->bindVars(*dynamicEnv); + i.valueExpr->bindVars(*dynamicEnv); } } void ExprList::bindVars(const StaticEnv & env) { - foreach (vector<Expr *>::iterator, i, elems) - (*i)->bindVars(env); + for (auto & i : elems) + i->bindVars(env); } void ExprLambda::bindVars(const StaticEnv & env) @@ -321,11 +331,11 @@ void ExprLambda::bindVars(const StaticEnv & env) if (!arg.empty()) newEnv.vars[arg] = displ++; if (matchAttrs) { - foreach (Formals::Formals_::iterator, i, formals->formals) - newEnv.vars[i->name] = displ++; + for (auto & i : formals->formals) + newEnv.vars[i.name] = displ++; - foreach (Formals::Formals_::iterator, i, formals->formals) - if (i->def) i->def->bindVars(newEnv); + for (auto & i : formals->formals) + if (i.def) i.def->bindVars(newEnv); } body->bindVars(newEnv); @@ -336,11 +346,11 @@ void ExprLet::bindVars(const StaticEnv & env) StaticEnv newEnv(false, &env); unsigned int displ = 0; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - newEnv.vars[i->first] = i->second.displ = displ++; + for (auto & i : attrs->attrs) + newEnv.vars[i.first] = i.second.displ = displ++; - foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - i->second.e->bindVars(i->second.inherited ? env : newEnv); + for (auto & i : attrs->attrs) + i.second.e->bindVars(i.second.inherited ? env : newEnv); body->bindVars(newEnv); } @@ -384,8 +394,8 @@ void ExprOpNot::bindVars(const StaticEnv & env) void ExprConcatStrings::bindVars(const StaticEnv & env) { - foreach (vector<Expr *>::iterator, i, *es) - (*i)->bindVars(env); + for (auto & i : *es) + i->bindVars(env); } void ExprPos::bindVars(const StaticEnv & env) @@ -419,8 +429,8 @@ string ExprLambda::showNamePos() const size_t SymbolTable::totalSize() const { size_t n = 0; - foreach (Symbols::const_iterator, i, symbols) - n += i->size(); + for (auto & i : symbols) + n += i.size(); return n; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index ef07d4557fe8..d2ca09b3a5bb 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -11,6 +11,7 @@ namespace nix { MakeError(EvalError, Error) MakeError(ParseError, Error) +MakeError(IncompleteParseError, ParseError) MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError) MakeError(Abort, EvalError) @@ -98,6 +99,15 @@ struct ExprInt : Expr Value * maybeThunk(EvalState & state, Env & env); }; +struct ExprFloat : Expr +{ + NixFloat nf; + Value v; + ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + struct ExprString : Expr { Symbol s; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 26168b2ed420..776e5cb39b81 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,10 +31,12 @@ namespace nix { Path basePath; Symbol path; string error; + bool atEnd; Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) + , atEnd(false) , sLetBody(symbols.create("<let-body>")) { }; }; @@ -136,8 +138,8 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex bool atStartOfLine = true; /* = seen only whitespace in the current line */ unsigned int minIndent = 1000000; unsigned int curIndent = 0; - foreach (vector<Expr *>::iterator, i, es) { - ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); + for (auto & i : es) { + ExprIndStr * e = dynamic_cast<ExprIndStr *>(i); if (!e) { /* Anti-quotations end the current start-of-line whitespace. */ if (atStartOfLine) { @@ -216,10 +218,6 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex } -void backToString(yyscan_t scanner); -void backToIndString(yyscan_t scanner); - - static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) { return Pos(data->path, loc.first_line, loc.first_column); @@ -248,6 +246,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err nix::Formals * formals; nix::Formal * formal; nix::NixInt n; + nix::NixFloat nf; const char * id; // !!! -> Symbol char * path; char * uri; @@ -268,6 +267,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %token <id> ID ATTRPATH %token <e> STR IND_STR %token <n> INT +%token <nf> FLOAT %token <path> PATH HPATH SPATH %token <uri> URI %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW @@ -370,6 +370,7 @@ expr_simple $$ = new ExprVar(CUR_POS, data->symbols.create($1)); } | INT { $$ = new ExprInt($1); } + | FLOAT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { $$ = stripIndentation(CUR_POS, data->symbols, *$2); @@ -404,25 +405,18 @@ string_parts string_parts_interpolated : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { backToString(scanner); $$ = $1; $1->push_back($3); } - | STR DOLLAR_CURLY expr '}' - { - backToString(scanner); + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); } + | STR DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($1); $$->push_back($3); } - | DOLLAR_CURLY expr '}' - { - backToString(scanner); - $$ = new vector<Expr *>; - $$->push_back($2); - } ; ind_string_parts : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } - | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = $1; $1->push_back($3); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } | { $$ = new vector<Expr *>; } ; @@ -430,20 +424,20 @@ binds : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } | binds INHERIT attrs ';' { $$ = $1; - foreach (AttrPath::iterator, i, *$3) { - if ($$->attrs.find(i->symbol) != $$->attrs.end()) - dupAttr(i->symbol, makeCurPos(@3, data), $$->attrs[i->symbol].pos); + for (auto & i : *$3) { + if ($$->attrs.find(i.symbol) != $$->attrs.end()) + dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); Pos pos = makeCurPos(@3, data); - $$->attrs[i->symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i->symbol), pos, true); + $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); } } | binds INHERIT '(' expr ')' attrs ';' { $$ = $1; /* !!! Should ensure sharing of the expression in $4. */ - foreach (AttrPath::iterator, i, *$6) { - if ($$->attrs.find(i->symbol) != $$->attrs.end()) - dupAttr(i->symbol, makeCurPos(@6, data), $$->attrs[i->symbol].pos); - $$->attrs[i->symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i->symbol), makeCurPos(@6, data)); + for (auto & i : *$6) { + if ($$->attrs.find(i.symbol) != $$->attrs.end()) + dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); + $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); } } | { $$ = new ExprAttrs; } @@ -526,9 +520,10 @@ formal #include <fcntl.h> #include <unistd.h> -#include <eval.hh> -#include <download.hh> -#include <store-api.hh> +#include "eval.hh" +#include "download.hh" +#include "store-api.hh" +#include "primops/fetchgit.hh" namespace nix { @@ -547,7 +542,12 @@ Expr * EvalState::parse(const char * text, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) throw ParseError(data.error); + if (res) { + if (data.atEnd) + throw IncompleteParseError(data.error); + else + throw ParseError(data.error); + } data.result->bindVars(staticEnv); @@ -601,7 +601,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) } -void EvalState::addToSearchPath(const string & s, bool warn) +void EvalState::addToSearchPath(const string & s) { size_t pos = s.find('='); string prefix; @@ -613,16 +613,7 @@ void EvalState::addToSearchPath(const string & s, bool warn) path = string(s, pos + 1); } - if (isUri(path)) - path = downloadFileCached(path, true); - - path = absPath(path); - if (pathExists(path)) { - debug(format("adding path ‘%1%’ to the search path") % path); - /* Resolve symlinks in the path to support restricted mode. */ - searchPath.push_back(std::pair<string, Path>(prefix, canonPath(path, true))); - } else if (warn) - printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % path); + searchPath.emplace_back(prefix, path); } @@ -635,17 +626,19 @@ Path EvalState::findFile(const string & path) Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) { for (auto & i : searchPath) { - assert(!isUri(i.second)); - Path res; + std::string suffix; if (i.first.empty()) - res = i.second + "/" + path; + suffix = "/" + path; else { - if (path.compare(0, i.first.size(), i.first) != 0 || - (path.size() > i.first.size() && path[i.first.size()] != '/')) + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) continue; - res = i.second + - (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size())); + suffix = path.size() == s ? "" : "/" + string(path, s); } + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } format f = format( @@ -656,4 +649,39 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos } +std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem) +{ + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) return i->second; + + std::pair<bool, std::string> res; + + if (isUri(elem.second)) { + try { + if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git")) + // FIXME: support specifying revision/branch + res = { true, exportGit(store, elem.second, "master") }; + else + res = { true, makeDownloader()->downloadCached(store, elem.second, true) }; + } catch (DownloadError & e) { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second); + res = { false, "" }; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) + res = { true, path }; + else { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second); + res = { false, "" }; + } + } + + debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second); + + searchPathResolved[elem.second] = res; + return res; +} + + } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 355b81adf76d..d7245fca52e1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,15 +1,16 @@ +#include "archive.hh" +#include "derivations.hh" +#include "download.hh" +#include "eval-inline.hh" #include "eval.hh" -#include "misc.hh" #include "globals.hh" +#include "json-to-value.hh" +#include "names.hh" #include "store-api.hh" #include "util.hh" -#include "archive.hh" -#include "value-to-xml.hh" #include "value-to-json.hh" -#include "json-to-value.hh" -#include "names.hh" -#include "eval-inline.hh" -#include "download.hh" +#include "value-to-xml.hh" +#include "primops.hh" #include <sys/types.h> #include <sys/stat.h> @@ -43,7 +44,7 @@ std::pair<string, string> decodeContext(const string & s) InvalidPathError::InvalidPathError(const Path & path) : EvalError(format("path ‘%1%’ is not valid") % path), path(path) {} -void realiseContext(const PathSet & context) +void EvalState::realiseContext(const PathSet & context) { PathSet drvs; for (auto & i : context) { @@ -52,16 +53,14 @@ void realiseContext(const PathSet & context) assert(isStorePath(ctx)); if (!store->isValidPath(ctx)) throw InvalidPathError(ctx); - if (!decoded.second.empty() && isDerivation(ctx)) + if (!decoded.second.empty() && nix::isDerivation(ctx)) drvs.insert(decoded.first + "!" + decoded.second); } if (!drvs.empty()) { /* For performance, prefetch all substitute info. */ PathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; - queryMissing(*store, drvs, - willBuild, willSubstitute, unknown, downloadSize, narSize); - + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); store->buildPaths(drvs); } } @@ -75,7 +74,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path path = state.coerceToPath(pos, *args[1], context); try { - realiseContext(context); + state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError(format("cannot import ‘%1%’, since path ‘%2%’ is not valid, at %3%") % path % e.path % pos); @@ -83,12 +82,14 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args path = state.checkSourcePath(path); - if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { + if (isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) { Derivation drv = readDerivation(path); Value & w = *state.allocValue(); - state.mkAttrs(w, 2 + drv.outputs.size()); + state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); - mkString(*v2, path, singleton<PathSet>("=" + path)); + mkString(*v2, path, {"=" + path}); + v2 = state.allocAttr(w, state.sName); + mkString(*v2, drv.env["name"]); Value * outputsVal = state.allocAttr(w, state.symbols.create("outputs")); state.mkList(*outputsVal, drv.outputs.size()); @@ -96,10 +97,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args for (const auto & o : drv.outputs) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, o.second.path, - singleton<PathSet>("!" + o.first + "!" + path)); - outputsVal->list.elems[outputs_index] = state.allocValue(); - mkString(*(outputsVal->list.elems[outputs_index++]), o.first); + mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + outputsVal->listElems()[outputs_index] = state.allocValue(); + mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } w.attrs->sort(); Value fun; @@ -123,7 +123,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path); Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); @@ -143,7 +143,7 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args Path path = state.coerceToPath(pos, *args[0], context); try { - realiseContext(context); + state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError(format("cannot import ‘%1%’, since path ‘%2%’ is not valid, at %3%") % path % e.path % pos); @@ -186,7 +186,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu case tPath: t = "path"; break; case tNull: t = "null"; break; case tAttrs: t = "set"; break; - case tList: t = "list"; break; + case tList1: case tList2: case tListN: t = "list"; break; case tLambda: case tPrimOp: case tPrimOpApp: @@ -195,6 +195,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu case tExternal: t = args[0]->external->typeOf(); break; + case tFloat: t = "float"; break; default: abort(); } mkString(v, state.symbols.create(t)); @@ -224,6 +225,12 @@ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value mkBool(v, args[0]->type == tInt); } +/* Determine whether the argument is a float. */ +static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tFloat); +} /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -245,11 +252,17 @@ struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { + if (v1->type == tFloat && v2->type == tInt) + return v1->fpoint < v2->integer; + if (v1->type == tInt && v2->type == tFloat) + return v1->integer < v2->fpoint; if (v1->type != v2->type) - throw EvalError("cannot compare values of different types"); + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); switch (v1->type) { case tInt: return v1->integer < v2->integer; + case tFloat: + return v1->fpoint < v2->fpoint; case tString: return strcmp(v1->string.s, v2->string.s) < 0; case tPath: @@ -270,7 +283,7 @@ typedef list<Value *> ValueList; static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { - startNest(nest, lvlDebug, "finding dependencies"); + Activity act(*logger, lvlDebug, "finding dependencies"); state.forceAttrs(*args[0], pos); @@ -282,8 +295,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(*startSet->value, pos); ValueList workSet; - for (unsigned int n = 0; n < startSet->value->list.length; ++n) - workSet.push_back(startSet->value->list.elems[n]); + for (unsigned int n = 0; n < startSet->value->listSize(); ++n) + workSet.push_back(startSet->value->listElems()[n]); /* Get the operator. */ Bindings::iterator op = @@ -321,17 +334,17 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(call, pos); /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.list.length; ++n) { - state.forceValue(*call.list.elems[n]); - workSet.push_back(call.list.elems[n]); + for (unsigned int n = 0; n < call.listSize(); ++n) { + state.forceValue(*call.listElems()[n]); + workSet.push_back(call.listElems()[n]); } } /* Create the result list. */ state.mkList(v, res.size()); unsigned int n = 0; - foreach (ValueList::iterator, i, res) - v.list.elems[n++] = *i; + for (auto & i : res) + v.listElems()[n++] = i; } @@ -443,7 +456,7 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation. */ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) { - startNest(nest, lvlVomit, "evaluating derivation"); + Activity act(*logger, lvlVomit, "evaluating derivation"); state.forceAttrs(*args[0], pos); @@ -477,24 +490,24 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * StringSet outputs; outputs.insert("out"); - foreach (Bindings::iterator, i, *args[0]->attrs) { - if (i->name == state.sIgnoreNulls) continue; - string key = i->name; - startNest(nest, lvlVomit, format("processing attribute ‘%1%’") % key); + for (auto & i : *args[0]->attrs) { + if (i.name == state.sIgnoreNulls) continue; + string key = i.name; + Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); try { if (ignoreNulls) { - state.forceValue(*i->value); - if (i->value->type == tNull) continue; + state.forceValue(*i.value); + if (i.value->type == tNull) continue; } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ if (key == "args") { - state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->list.length; ++n) { - string s = state.coerceToString(posDrvName, *i->value->list.elems[n], context, true); + state.forceList(*i.value, pos); + for (unsigned int n = 0; n < i.value->listSize(); ++n) { + string s = state.coerceToString(posDrvName, *i.value->listElems()[n], context, true); drv.args.push_back(s); } } @@ -502,11 +515,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* All other attributes are passed to the builder through the environment. */ else { - string s = state.coerceToString(posDrvName, *i->value, context, true); + string s = state.coerceToString(posDrvName, *i.value, context, true); drv.env[key] = s; if (key == "builder") drv.builder = s; - else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sName) { + else if (i.name == state.sSystem) drv.platform = s; + else if (i.name == state.sName) { drvName = s; printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName); } @@ -520,17 +533,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * else if (key == "outputs") { Strings tmp = tokenizeString<Strings>(s); outputs.clear(); - foreach (Strings::iterator, j, tmp) { - if (outputs.find(*j) != outputs.end()) - throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % *j % posDrvName); - /* !!! Check whether *j is a valid attribute + for (auto & j : tmp) { + if (outputs.find(j) != outputs.end()) + throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName); + /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ - if (*j == "drv") + if (j == "drv") throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName); - outputs.insert(*j); + outputs.insert(j); } if (outputs.empty()) throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); @@ -547,8 +560,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - foreach (PathSet::iterator, i, context) { - Path path = *i; + for (auto & path : context) { /* Paths marked with `=' denote that the path of a derivation is explicitly passed to the builder. Since that allows the @@ -559,11 +571,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * runs. */ if (path.at(0) == '=') { /* !!! This doesn't work if readOnlyMode is set. */ - PathSet refs; computeFSClosure(*store, string(path, 1), refs); - foreach (PathSet::iterator, j, refs) { - drv.inputSrcs.insert(*j); - if (isDerivation(*j)) - drv.inputDrvs[*j] = store->queryDerivationOutputNames(*j); + PathSet refs; + state.store->computeFSClosure(string(path, 1), refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (isDerivation(j)) + drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); } } @@ -580,7 +593,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Handle derivation contexts returned by ‘builtins.storePath’. */ else if (isDerivation(path)) - drv.inputDrvs[path] = store->queryDerivationOutputNames(path); + drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path); /* Otherwise it's a source file. */ else @@ -622,25 +635,25 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * are empty, and the corresponding environment variables have an empty value. This ensures that changes in the set of output names do get reflected in the hash. */ - foreach (StringSet::iterator, i, outputs) { - drv.env[*i] = ""; - drv.outputs[*i] = DerivationOutput("", "", ""); + for (auto & i : outputs) { + drv.env[i] = ""; + drv.outputs[i] = DerivationOutput("", "", ""); } /* Use the masked derivation expression to compute the output path. */ - Hash h = hashDerivationModulo(*store, drv); + Hash h = hashDerivationModulo(*state.store, drv); - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (i->second.path == "") { - Path outPath = makeOutputPath(i->first, h, drvName); - drv.env[i->first] = outPath; - i->second.path = outPath; + for (auto & i : drv.outputs) + if (i.second.path == "") { + Path outPath = makeOutputPath(i.first, h, drvName); + drv.env[i.first] = outPath; + i.second.path = outPath; } } /* Write the resulting term into the Nix store directory. */ - Path drvPath = writeDerivation(*store, drv, drvName, state.repair); + Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); printMsg(lvlChatty, format("instantiated ‘%1%’ -> ‘%2%’") % drvName % drvPath); @@ -648,13 +661,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't read them later. */ - drvHashes[drvPath] = hashDerivationModulo(*store, drv); + drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton<PathSet>("=" + drvPath)); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - mkString(*state.allocAttr(v, state.symbols.create(i->first)), - i->second.path, singleton<PathSet>("!" + i->first + "!" + drvPath)); + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + for (auto & i : drv.outputs) { + mkString(*state.allocAttr(v, state.symbols.create(i.first)), + i.second.path, {"!" + i.first + "!" + drvPath}); } v.attrs->sort(); } @@ -694,7 +707,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V throw EvalError(format("path ‘%1%’ is not in the Nix store, at %2%") % path % pos); Path path2 = toStorePath(path); if (!settings.readOnlyMode) - store->ensurePath(path2); + state.store->ensurePath(path2); context.insert(path2); mkString(v, path, context); } @@ -744,7 +757,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va PathSet context; Path path = state.coerceToPath(pos, *args[0], context); try { - realiseContext(context); + state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%") % path % e.path % pos); @@ -752,7 +765,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va string s = readFile(state.checkSourcePath(path)); if (s.find((char) 0) != string::npos) throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path); - mkString(v, s.c_str()); + mkString(v, s.c_str(), context); } @@ -764,9 +777,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - PathSet context; - for (unsigned int n = 0; n < args[0]->list.length; ++n) { - Value & v2(*args[0]->list.elems[n]); + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + Value & v2(*args[0]->listElems()[n]); state.forceAttrs(v2, pos); string prefix; @@ -777,21 +789,23 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError(format("attribute ‘path’ missing, at %1%") % pos); - string path = state.coerceToPath(pos, *i->value, context); - searchPath.push_back(std::pair<string, Path>(prefix, state.checkSourcePath(path))); - } + PathSet context; + string path = state.coerceToString(pos, *i->value, context, false, false); - string path = state.forceStringNoCtx(*args[1], pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") + % path % e.path % pos); + } - try { - realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + searchPath.emplace_back(prefix, path); } - mkPath(v, state.findFile(searchPath, path, pos).c_str()); + string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } /* Read a directory (without . or ..) */ @@ -800,7 +814,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val PathSet ctx; Path path = state.coerceToPath(pos, *args[0], ctx); try { - realiseContext(ctx); + state.realiseContext(ctx); } catch (InvalidPathError & e) { throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%") % path % e.path % pos); @@ -871,23 +885,26 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu PathSet refs; - foreach (PathSet::iterator, i, context) { - Path path = *i; + for (auto path : context) { if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) - throw EvalError(format("in ‘toFile’: the file ‘%1%’ cannot refer to derivation outputs, at %2%") % name % pos); + if (isDerivation(path)) { + /* See prim_unsafeDiscardOutputDependency. */ + if (path.at(0) != '~') + throw EvalError(format("in ‘toFile’: the file ‘%1%’ cannot refer to derivation outputs, at %2%") % name % pos); + path = string(path, 1); + } refs.insert(path); } Path storePath = settings.readOnlyMode ? computeStorePathForText(name, contents, refs) - : store->addTextToStore(name, contents, refs, state.repair); + : state.store->addTextToStore(name, contents, refs, state.repair); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths used in args[1]. */ - mkString(v, storePath, singleton<PathSet>(storePath)); + mkString(v, storePath, {storePath}); } @@ -947,9 +964,9 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args Path dstPath = settings.readOnlyMode ? computeStorePathForPath(path, true, htSHA256, filter).first - : store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); - mkString(v, dstPath, singleton<PathSet>(dstPath)); + mkString(v, dstPath, {dstPath}); } @@ -968,9 +985,9 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V unsigned int n = 0; for (auto & i : *args[0]->attrs) - mkString(*(v.list.elems[n++] = state.allocValue()), i.name); + mkString(*(v.listElems()[n++] = state.allocValue()), i.name); - std::sort(v.list.elems, v.list.elems + n, + std::sort(v.listElems(), v.listElems() + n, [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } @@ -985,13 +1002,13 @@ static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, unsigned int n = 0; for (auto & i : *args[0]->attrs) - v.list.elems[n++] = (Value *) &i; + v.listElems()[n++] = (Value *) &i; - std::sort(v.list.elems, v.list.elems + n, + std::sort(v.listElems(), v.listElems() + n, [](Value * v1, Value * v2) { return (string) ((Attr *) v1)->name < (string) ((Attr *) v2)->name; }); for (unsigned int i = 0; i < n; ++i) - v.list.elems[i] = ((Attr *) v.list.elems[i])->value; + v.listElems()[i] = ((Attr *) v.listElems()[i])->value; } @@ -1048,18 +1065,18 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, /* Get the attribute names to be removed. */ std::set<Symbol> names; - for (unsigned int i = 0; i < args[1]->list.length; ++i) { - state.forceStringNoCtx(*args[1]->list.elems[i], pos); - names.insert(state.symbols.create(args[1]->list.elems[i]->string.s)); + for (unsigned int i = 0; i < args[1]->listSize(); ++i) { + state.forceStringNoCtx(*args[1]->listElems()[i], pos); + names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); } /* Copy all attributes not in that set. Note that we don't need to sort v.attrs because it's a subset of an already sorted vector. */ state.mkAttrs(v, args[0]->attrs->size()); - foreach (Bindings::iterator, i, *args[0]->attrs) { - if (names.find(i->name) == names.end()) - v.attrs->push_back(*i); + for (auto & i : *args[0]->attrs) { + if (names.find(i.name) == names.end()) + v.attrs->push_back(i); } } @@ -1073,12 +1090,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, { state.forceList(*args[0], pos); - state.mkAttrs(v, args[0]->list.length); + state.mkAttrs(v, args[0]->listSize()); std::set<Symbol> seen; - for (unsigned int i = 0; i < args[0]->list.length; ++i) { - Value & v2(*args[0]->list.elems[i]); + for (unsigned int i = 0; i < args[0]->listSize(); ++i) { + Value & v2(*args[0]->listElems()[i]); state.forceAttrs(v2, pos); Bindings::iterator j = v2.attrs->find(state.sName); @@ -1111,8 +1128,8 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); - foreach (Bindings::iterator, i, *args[0]->attrs) { - Bindings::iterator j = args[1]->attrs->find(i->name); + for (auto & i : *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i.name); if (j != args[1]->attrs->end()) v.attrs->push_back(*j); } @@ -1131,11 +1148,11 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); state.forceList(*args[1], pos); - Value * res[args[1]->list.length]; + Value * res[args[1]->listSize()]; unsigned int found = 0; - for (unsigned int n = 0; n < args[1]->list.length; ++n) { - Value & v2(*args[1]->list.elems[n]); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value & v2(*args[1]->listElems()[n]); state.forceAttrs(v2, pos); Bindings::iterator i = v2.attrs->find(attrName); if (i != v2.attrs->end()) @@ -1144,13 +1161,13 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va state.mkList(v, found); for (unsigned int n = 0; n < found; ++n) - v.list.elems[n] = res[n]; + v.listElems()[n] = res[n]; } /* Return a set containing the names of the formal arguments expected by the function `f'. The value of each attribute is a Boolean - denoting whether has a default value. For instance, + denoting whether the corresponding argument has a default value. For instance, functionArgs ({ x, y ? 123}: ...) => { x = false; y = true; } @@ -1173,9 +1190,9 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - foreach (Formals::Formals_::iterator, i, args[0]->lambda.fun->formals->formals) + for (auto & i : args[0]->lambda.fun->formals->formals) // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i->name), i->def); + mkBool(*state.allocAttr(v, i.name), i.def); v.attrs->sort(); } @@ -1189,17 +1206,17 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0]); - mkBool(v, args[0]->type == tList); + mkBool(v, args[0]->isList()); } static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) { state.forceList(list, pos); - if (n < 0 || (unsigned int) n >= list.list.length) + if (n < 0 || (unsigned int) n >= list.listSize()) throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); - state.forceValue(*list.list.elems[n]); - v = *list.list.elems[n]; + state.forceValue(*list.listElems()[n]); + v = *list.listElems()[n]; } @@ -1223,11 +1240,11 @@ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); - if (args[0]->list.length == 0) + if (args[0]->listSize() == 0) throw Error(format("‘tail’ called on an empty list, at %1%") % pos); - state.mkList(v, args[0]->list.length - 1); - for (unsigned int n = 0; n < v.list.length; ++n) - v.list.elems[n] = args[0]->list.elems[n + 1]; + state.mkList(v, args[0]->listSize() - 1); + for (unsigned int n = 0; n < v.listSize(); ++n) + v.listElems()[n] = args[0]->listElems()[n + 1]; } @@ -1237,11 +1254,11 @@ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & state.forceFunction(*args[0], pos); state.forceList(*args[1], pos); - state.mkList(v, args[1]->list.length); + state.mkList(v, args[1]->listSize()); - for (unsigned int n = 0; n < v.list.length; ++n) - mkApp(*(v.list.elems[n] = state.allocValue()), - *args[0], *args[1]->list.elems[n]); + for (unsigned int n = 0; n < v.listSize(); ++n) + mkApp(*(v.listElems()[n] = state.allocValue()), + *args[0], *args[1]->listElems()[n]); } @@ -1254,15 +1271,15 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu state.forceList(*args[1], pos); // FIXME: putting this on the stack is risky. - Value * vs[args[1]->list.length]; + Value * vs[args[1]->listSize()]; unsigned int k = 0; bool same = true; - for (unsigned int n = 0; n < args[1]->list.length; ++n) { + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { Value res; - state.callFunction(*args[0], *args[1]->list.elems[n], res, noPos); + state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); if (state.forceBool(res)) - vs[k++] = args[1]->list.elems[n]; + vs[k++] = args[1]->listElems()[n]; else same = false; } @@ -1271,7 +1288,7 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu v = *args[1]; else { state.mkList(v, k); - for (unsigned int n = 0; n < k; ++n) v.list.elems[n] = vs[n]; + for (unsigned int n = 0; n < k; ++n) v.listElems()[n] = vs[n]; } } @@ -1281,8 +1298,8 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value { bool res = false; state.forceList(*args[1], pos); - for (unsigned int n = 0; n < args[1]->list.length; ++n) - if (state.eqValues(*args[0], *args[1]->list.elems[n])) { + for (unsigned int n = 0; n < args[1]->listSize(); ++n) + if (state.eqValues(*args[0], *args[1]->listElems()[n])) { res = true; break; } @@ -1294,7 +1311,7 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); - state.concatLists(v, args[0]->list.length, args[0]->list.elems, pos); + state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); } @@ -1302,7 +1319,114 @@ static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); - mkInt(v, args[0]->list.length); + mkInt(v, args[0]->listSize()); +} + + +/* Reduce a list by applying a binary operator, from left to + right. The operator is applied strictly. */ +static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[2], pos); + + Value * vCur = args[1]; + + if (args[2]->listSize()) + for (unsigned int n = 0; n < args[2]->listSize(); ++n) { + Value vTmp; + state.callFunction(*args[0], *vCur, vTmp, pos); + vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); + state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + } + else + v = *vCur; + + state.forceValue(v); +} + + +static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + Value vTmp; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + bool res = state.forceBool(vTmp); + if (res == any) { + mkBool(v, any); + return; + } + } + + mkBool(v, !any); +} + + +static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + anyOrAll(true, state, pos, args, v); +} + + +static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + anyOrAll(false, state, pos, args, v); +} + + +static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + auto len = state.forceInt(*args[1], pos); + + if (len < 0) + throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos); + + state.mkList(v, len); + + for (unsigned int n = 0; n < (unsigned int) len; ++n) { + Value * arg = state.allocValue(); + mkInt(*arg, n); + mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg); + } +} + + +static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v); + + +static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + auto len = args[1]->listSize(); + state.mkList(v, len); + for (unsigned int n = 0; n < len; ++n) { + state.forceValue(*args[1]->listElems()[n]); + v.listElems()[n] = args[1]->listElems()[n]; + } + + + auto comparator = [&](Value * a, Value * b) { + /* Optimization: if the comparator is lessThan, bypass + callFunction. */ + if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + return CompareValues()(a, b); + + Value vTmp1, vTmp2; + state.callFunction(*args[0], *a, vTmp1, pos); + state.callFunction(vTmp1, *b, vTmp2, pos); + return state.forceBool(vTmp2); + }; + + /* FIXME: std::sort can segfault if the comparator is not a strict + weak ordering. What to do? std::stable_sort() seems more + resilient, but no guarantees... */ + std::stable_sort(v.listElems(), v.listElems() + len, comparator); } @@ -1313,27 +1437,40 @@ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Valu static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); } static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); } static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); } static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) { - NixInt i2 = state.forceInt(*args[1], pos); - if (i2 == 0) throw EvalError(format("division by zero, at %1%") % pos); - mkInt(v, state.forceInt(*args[0], pos) / i2); + NixFloat f2 = state.forceFloat(*args[1], pos); + if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); + + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) / state.forceInt(*args[1], pos)); } @@ -1407,11 +1544,8 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po string s = state.coerceToString(pos, *args[0], context); PathSet context2; - foreach (PathSet::iterator, i, context) { - Path p = *i; - if (p.at(0) == '=') p = "~" + string(p, 1); - context2.insert(p); - } + for (auto & p : context) + context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p); mkString(v, s, context2); } @@ -1452,13 +1586,68 @@ static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value for (unsigned int n = 0; n < len; ++n) { auto i = subs.find(n); if (i == subs.end()) - mkNull(*(v.list.elems[n] = state.allocValue())); + mkNull(*(v.listElems()[n] = state.allocValue())); else - mkString(*(v.list.elems[n] = state.allocValue()), i->second); + mkString(*(v.listElems()[n] = state.allocValue()), i->second); } } +static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + + auto sep = state.forceString(*args[0], context, pos); + state.forceList(*args[1], pos); + + string res; + res.reserve((args[1]->listSize() + 32) * sep.size()); + bool first = true; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (first) first = false; else res += sep; + res += state.coerceToString(pos, *args[1]->listElems()[n], context); + } + + mkString(v, res, context); +} + + +static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + state.forceList(*args[1], pos); + if (args[0]->listSize() != args[1]->listSize()) + throw EvalError(format("‘from’ and ‘to’ arguments to ‘replaceStrings’ have different lengths, at %1%") % pos); + + Strings from; + for (unsigned int n = 0; n < args[0]->listSize(); ++n) + from.push_back(state.forceStringNoCtx(*args[0]->listElems()[n], pos)); + + Strings to; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) + to.push_back(state.forceStringNoCtx(*args[1]->listElems()[n], pos)); + + PathSet context; + auto s = state.forceString(*args[2], context, pos); + + string res; + for (size_t p = 0; p < s.size(); ) { + bool found = false; + for (auto i = from.begin(), j = to.begin(); i != from.end(); ++i, ++j) + if (s.compare(p, i->size(), *i) == 0) { + found = true; + p += i->size(); + res += *j; + break; + } + if (!found) res += s[p++]; + } + + mkString(v, res, context); +} + + /************************************************************* * Versions *************************************************************/ @@ -1515,7 +1704,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, } else url = state.forceStringNoCtx(*args[0], pos); - Path res = downloadFileCached(url, unpack); + Path res = makeDownloader()->downloadCached(state.store, url, unpack); mkString(v, res, PathSet({res})); } @@ -1537,6 +1726,16 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args *************************************************************/ +RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; + + +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +{ + if (!primOps) primOps = new PrimOps; + primOps->emplace_back(name, arity, fun); +} + + void EvalState::createBaseEnv() { baseEnv.up = 0; @@ -1573,7 +1772,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 3); + mkInt(v, 4); addConstant("__langVersion", v); // Miscellaneous @@ -1590,6 +1789,7 @@ void EvalState::createBaseEnv() addPrimOp("__isFunction", 1, prim_isFunction); addPrimOp("__isString", 1, prim_isString); addPrimOp("__isInt", 1, prim_isInt); + addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); @@ -1646,6 +1846,11 @@ void EvalState::createBaseEnv() addPrimOp("__elem", 2, prim_elem); addPrimOp("__concatLists", 1, prim_concatLists); addPrimOp("__length", 1, prim_length); + addPrimOp("__foldl'", 3, prim_foldlStrict); + addPrimOp("__any", 2, prim_any); + addPrimOp("__all", 2, prim_all); + addPrimOp("__genList", 2, prim_genList); + addPrimOp("__sort", 2, prim_sort); // Integer arithmetic addPrimOp("__add", 2, prim_add); @@ -1662,6 +1867,8 @@ void EvalState::createBaseEnv() addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); addPrimOp("__match", 2, prim_match); + addPrimOp("__concatStringsSep", 2, prim_concatStringSep); + addPrimOp("__replaceStrings", 3, prim_replaceStrings); // Versions addPrimOp("__parseDrvName", 1, prim_parseDrvName); @@ -1685,7 +1892,7 @@ void EvalState::createBaseEnv() mkList(v, searchPath.size()); int n = 0; for (auto & i : searchPath) { - v2 = v.list.elems[n++] = allocValue(); + v2 = v.listElems()[n++] = allocValue(); mkAttrs(*v2, 2); mkString(*allocAttr(*v2, symbols.create("path")), i.second); mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); @@ -1693,6 +1900,10 @@ void EvalState::createBaseEnv() } addConstant("__nixPath", v); + if (RegisterPrimOp::primOps) + for (auto & primOp : *RegisterPrimOp::primOps) + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh new file mode 100644 index 000000000000..39d23b04a5ce --- /dev/null +++ b/src/libexpr/primops.hh @@ -0,0 +1,15 @@ +#include "eval.hh" + +#include <tuple> +#include <vector> + +namespace nix { + +struct RegisterPrimOp +{ + typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; + static PrimOps * primOps; + RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); +}; + +} diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc new file mode 100644 index 000000000000..bd440c8c62ad --- /dev/null +++ b/src/libexpr/primops/fetchgit.cc @@ -0,0 +1,82 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "store-api.hh" + +namespace nix { + +Path exportGit(ref<Store> store, const std::string & uri, const std::string & rev) +{ + if (!isUri(uri)) + throw EvalError(format("‘%s’ is not a valid URI") % uri); + + Path cacheDir = getCacheDir() + "/nix/git"; + + if (!pathExists(cacheDir)) { + createDirs(cacheDir); + runProgram("git", true, { "init", "--bare", cacheDir }); + } + + Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); + + std::string localRef = "pid-" + std::to_string(getpid()); + Path localRefFile = cacheDir + "/refs/heads/" + localRef; + + runProgram("git", true, { "-C", cacheDir, "fetch", uri, rev + ":" + localRef }); + + std::string commitHash = chomp(readFile(localRefFile)); + + unlink(localRefFile.c_str()); + + debug(format("got revision ‘%s’") % commitHash); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, { "-C", cacheDir, "archive", commitHash }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("tar", true, { "x", "-C", tmpDir }, tar); + + return store->addToStore("git-export", tmpDir); +} + +static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + // FIXME: cut&paste from fetch(). + if (state.restricted) throw Error("‘fetchgit’ is not allowed in restricted mode"); + + std::string url; + std::string rev = "master"; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string name(attr.name); + if (name == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (name == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError(format("unsupported argument ‘%1%’ to ‘fetchgit’, at %3%") % attr.name % attr.pos); + } + + if (url.empty()) + throw EvalError(format("‘url’ argument required, at %1%") % pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + Path storePath = exportGit(state.store, url, rev); + + mkString(v, storePath, PathSet({storePath})); +} + +static RegisterPrimOp r("__fetchgit", 1, prim_fetchgit); + +} diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh new file mode 100644 index 000000000000..6ffb21a96daa --- /dev/null +++ b/src/libexpr/primops/fetchgit.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <string> + +#include "ref.hh" + +namespace nix { + +class Store; + +Path exportGit(ref<Store> store, + const std::string & uri, const std::string & rev); + +} diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index cdb71341875a..47ee324a6e4f 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -12,14 +12,14 @@ namespace nix { void escapeJSON(std::ostream & str, const string & s) { str << "\""; - foreach (string::const_iterator, i, s) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i >= 0 && *i < 32) - str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec; - else str << *i; + for (auto & i : s) + if (i == '\"' || i == '\\') str << "\\" << i; + else if (i == '\n') str << "\\n"; + else if (i == '\r') str << "\\r"; + else if (i == '\t') str << "\\t"; + else if (i >= 0 && i < 32) + str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) i << std::dec; + else str << i; str << "\""; } @@ -59,11 +59,11 @@ void printValueAsJSON(EvalState & state, bool strict, if (i == v.attrs->end()) { JSONObject json(str); StringSet names; - foreach (Bindings::iterator, i, *v.attrs) - names.insert(i->name); - foreach (StringSet::iterator, i, names) { - Attr & a(*v.attrs->find(state.symbols.create(*i))); - json.attr(*i); + for (auto & j : *v.attrs) + names.insert(j.name); + for (auto & j : names) { + Attr & a(*v.attrs->find(state.symbols.create(j))); + json.attr(j); printValueAsJSON(state, strict, *a.value, str, context); } } else @@ -71,19 +71,23 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tList: { + case tList1: case tList2: case tListN: { JSONList json(str); - for (unsigned int n = 0; n < v.list.length; ++n) { + for (unsigned int n = 0; n < v.listSize(); ++n) { json.elem(); - printValueAsJSON(state, strict, *v.list.elems[n], str, context); + printValueAsJSON(state, strict, *v.listElems()[n], str, context); } break; } - case tExternal: + case tExternal: v.external->printValueAsJSON(state, strict, str, context); break; + case tFloat: + str << v.fpoint; + break; + default: throw TypeError(format("cannot convert %1% to JSON") % showType(v)); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index f6796f2053e9..c59caf5641bc 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -36,7 +36,18 @@ struct JSONObject attr(s); escapeJSON(str, t); } - void attr(const string & s, int n) + void attr(const string & s, const char * t) + { + attr(s); + escapeJSON(str, t); + } + void attr(const string & s, bool b) + { + attr(s); + str << (b ? "true" : "false"); + } + template<typename T> + void attr(const string & s, const T & n) { attr(s); str << n; diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index bbbb7039bf70..00b1918a82aa 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -8,7 +8,7 @@ namespace nix { - + static XMLAttrs singletonAttrs(const string & name, const string & value) { XMLAttrs attrs; @@ -33,17 +33,17 @@ static void showAttrs(EvalState & state, bool strict, bool location, Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) { StringSet names; - - foreach (Bindings::iterator, i, attrs) - names.insert(i->name); - - foreach (StringSet::iterator, i, names) { - Attr & a(*attrs.find(state.symbols.create(*i))); - + + for (auto & i : attrs) + names.insert(i.name); + + for (auto & i : names) { + Attr & a(*attrs.find(state.symbols.create(i))); + XMLAttrs xmlAttrs; - xmlAttrs["name"] = *i; + xmlAttrs["name"] = i; if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); - + XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen); @@ -57,7 +57,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, checkInterrupt(); if (strict) state.forceValue(v); - + switch (v.type) { case tInt: @@ -85,7 +85,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case tAttrs: if (state.isDerivation(v)) { XMLAttrs xmlAttrs; - + Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); Path drvPath; @@ -95,7 +95,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (a->value->type == tString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } - + a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); @@ -116,13 +116,13 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "attrs"); showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); } - + break; - case tList: { + case tList1: case tList2: case tListN: { XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.list.length; ++n) - printValueAsXML(state, strict, location, *v.list.elems[n], doc, context, drvsSeen); + for (unsigned int n = 0; n < v.listSize(); ++n) + printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); break; } @@ -130,17 +130,17 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLAttrs xmlAttrs; if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); - + if (v.lambda.fun->matchAttrs) { XMLAttrs attrs; if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; XMLOpenElement _(doc, "attrspat", attrs); - foreach (Formals::Formals_::iterator, i, v.lambda.fun->formals->formals) - doc.writeEmptyElement("attr", singletonAttrs("name", i->name)); + for (auto & i : v.lambda.fun->formals->formals) + doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); } else doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); - + break; } @@ -148,6 +148,10 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); break; + case tFloat: + doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + break; + default: doc.writeEmptyElement("unevaluated"); } @@ -166,9 +170,9 @@ void printValueAsXML(EvalState & state, bool strict, bool location, { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); - PathSet drvsSeen; + PathSet drvsSeen; printValueAsXML(state, strict, location, v, doc, context, drvsSeen); } - + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c06b5a6d1153..62bdd9281f08 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -12,7 +12,9 @@ typedef enum { tPath, tNull, tAttrs, - tList, + tList1, + tList2, + tListN, tThunk, tApp, tLambda, @@ -20,6 +22,7 @@ typedef enum { tPrimOp, tPrimOpApp, tExternal, + tFloat } ValueType; @@ -36,6 +39,7 @@ class XMLWriter; typedef long NixInt; +typedef float NixFloat; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented @@ -119,9 +123,10 @@ struct Value const char * path; Bindings * attrs; struct { - unsigned int length; + unsigned int size; Value * * elems; - } list; + } bigList; + Value * smallList[2]; struct { Env * env; Expr * expr; @@ -138,7 +143,28 @@ struct Value Value * left, * right; } primOpApp; ExternalValueBase * external; + NixFloat fpoint; }; + + bool isList() const + { + return type == tList1 || type == tList2 || type == tListN; + } + + Value * * listElems() + { + return type == tList1 || type == tList2 ? smallList : bigList.elems; + } + + const Value * const * listElems() const + { + return type == tList1 || type == tList2 ? smallList : bigList.elems; + } + + unsigned int listSize() const + { + return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + } }; @@ -158,6 +184,14 @@ static inline void mkInt(Value & v, NixInt n) } +static inline void mkFloat(Value & v, NixFloat n) +{ + clearValue(v); + v.type = tFloat; + v.fpoint = n; +} + + static inline void mkBool(Value & v, bool b) { clearValue(v); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc new file mode 100644 index 000000000000..98693d78a7f4 --- /dev/null +++ b/src/libmain/common-args.cc @@ -0,0 +1,29 @@ +#include "common-args.hh" +#include "globals.hh" + +namespace nix { + +MixCommonArgs::MixCommonArgs(const string & programName) + : programName(programName) +{ + mkFlag('v', "verbose", "increase verbosity level", []() { + verbosity = (Verbosity) (verbosity + 1); + }); + + mkFlag(0, "quiet", "decrease verbosity level", []() { + verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; + }); + + mkFlag(0, "debug", "enable debug output", []() { + verbosity = lvlDebug; + }); + + mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2, + [](Strings ss) { + auto name = ss.front(); ss.pop_front(); + auto value = ss.front(); + settings.set(name, value); + }); +} + +} diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh new file mode 100644 index 000000000000..2c0d71edd815 --- /dev/null +++ b/src/libmain/common-args.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct MixCommonArgs : virtual Args +{ + string programName; + MixCommonArgs(const string & programName); +}; + +struct MixDryRun : virtual Args +{ + bool dryRun; + + MixDryRun() + { + mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + } +}; + +} diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 16dbf752823d..f1fd3eb72424 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -6,6 +6,8 @@ libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +libmain_LDFLAGS = $(OPENSSL_LIBS) + libmain_LIBS = libstore libutil libformat libmain_ALLOW_UNDEFINED = 1 diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f300fbf1b463..0b6311516ad4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,15 +1,16 @@ #include "config.h" -#include "shared.hh" +#include "common-args.hh" #include "globals.hh" +#include "shared.hh" #include "store-api.hh" #include "util.hh" -#include "misc.hh" -#include <iostream> +#include <algorithm> #include <cctype> #include <exception> -#include <algorithm> +#include <iostream> +#include <mutex> #include <cstdlib> #include <sys/time.h> @@ -17,21 +18,15 @@ #include <unistd.h> #include <signal.h> -extern char * * environ; +#include <openssl/crypto.h> namespace nix { -volatile sig_atomic_t blockInt = 0; - - static void sigintHandler(int signo) { - if (!blockInt) { - _isInterrupted = 1; - blockInt = 1; - } + _isInterrupted = 1; } @@ -47,22 +42,22 @@ void printGCWarning() } -void printMissing(StoreAPI & store, const PathSet & paths) +void printMissing(ref<Store> store, const PathSet & paths) { unsigned long long downloadSize, narSize; PathSet willBuild, willSubstitute, unknown; - queryMissing(store, paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize); + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize); } -void printMissing(const PathSet & willBuild, +void printMissing(ref<Store> store, const PathSet & willBuild, const PathSet & willSubstitute, const PathSet & unknown, unsigned long long downloadSize, unsigned long long narSize) { if (!willBuild.empty()) { printMsg(lvlInfo, format("these derivations will be built:")); - Paths sorted = topoSortPaths(*store, willBuild); + Paths sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) printMsg(lvlInfo, format(" %1%") % i); @@ -85,15 +80,6 @@ void printMissing(const PathSet & willBuild, } -static void setLogType(string lt) -{ - if (lt == "pretty") logType = ltPretty; - else if (lt == "escapes") logType = ltEscapes; - else if (lt == "flat") logType = ltFlat; - else throw UsageError("unknown log type"); -} - - string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end) { @@ -103,7 +89,18 @@ string getArg(const string & opt, } -void detectStackOverflow(); +/* OpenSSL is not thread-safe by default - it will randomly crash + unless the user supplies a mutex locking function. So let's do + that. */ +static std::vector<std::mutex> opensslLocks; + +static void opensslLockCallback(int mode, int type, const char * file, int line) +{ + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); +} void initNix() @@ -114,7 +111,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - std::ios::sync_with_stdio(false); + logger = makeDefaultLogger(); + + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); settings.processEnvironment(); settings.loadConfFile(); @@ -161,77 +162,77 @@ void initNix() } -void parseCmdLine(int argc, char * * argv, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) +struct LegacyArgs : public MixCommonArgs { - /* Put the arguments in a vector. */ - Strings args; - argc--; argv++; - while (argc--) args.push_back(*argv++); - - /* Process default options. */ - for (Strings::iterator i = args.begin(); i != args.end(); ++i) { - string arg = *i; - - /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'). */ - if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { - *i = (string) "-" + arg[1]; - auto next = i; ++next; - for (unsigned int j = 2; j < arg.length(); j++) - if (isalpha(arg[j])) - args.insert(next, (string) "-" + arg[j]); - else { - args.insert(next, string(arg, j)); - break; - } - arg = *i; - } + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; - if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) (verbosity + 1); - else if (arg == "--quiet") verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; - else if (arg == "--log-type") { - string s = getArg(arg, i, args.end()); - setLogType(s); - } - else if (arg == "--no-build-output" || arg == "-Q") - settings.buildVerbosity = lvlVomit; - else if (arg == "--print-build-trace") - settings.printBuildTrace = true; - else if (arg == "--keep-failed" || arg == "-K") - settings.keepFailed = true; - else if (arg == "--keep-going" || arg == "-k") - settings.keepGoing = true; - else if (arg == "--fallback") + LegacyArgs(const std::string & programName, + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) + : MixCommonArgs(programName), parseArg(parseArg) + { + mkFlag('Q', "no-build-output", "do not show build output", + &settings.verboseBuild, false); + + mkFlag('K', "keep-failed", "keep temporary directories of failed builds", + &settings.keepFailed); + + mkFlag('k', "keep-going", "keep going after a build fails", + &settings.keepGoing); + + mkFlag(0, "fallback", "build from source if substitution fails", []() { settings.set("build-fallback", "true"); - else if (arg == "--max-jobs" || arg == "-j") - settings.set("build-max-jobs", getArg(arg, i, args.end())); - else if (arg == "--cores") - settings.set("build-cores", getArg(arg, i, args.end())); - else if (arg == "--readonly-mode") - settings.readOnlyMode = true; - else if (arg == "--max-silent-time") - settings.set("build-max-silent-time", getArg(arg, i, args.end())); - else if (arg == "--timeout") - settings.set("build-timeout", getArg(arg, i, args.end())); - else if (arg == "--no-build-hook") - settings.useBuildHook = false; - else if (arg == "--show-trace") - settings.showTrace = true; - else if (arg == "--no-gc-warning") - gcWarning = false; - else if (arg == "--option") { - ++i; if (i == args.end()) throw UsageError("‘--option’ requires two arguments"); - string name = *i; - ++i; if (i == args.end()) throw UsageError("‘--option’ requires two arguments"); - string value = *i; - settings.set(name, value); - } - else { - if (!parseArg(i, args.end())) - throw UsageError(format("unrecognised option ‘%1%’") % *i); - } + }); + + auto intSettingAlias = [&](char shortName, const std::string & longName, + const std::string & description, const std::string & dest) { + mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); + }); + }; + + intSettingAlias('j', "max-jobs", "maximum number of parallel builds", "build-max-jobs"); + intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "build-cores"); + intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "build-max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", "build-timeout"); + + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); + + mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", + &settings.useBuildHook, false); + + mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", + &settings.showTrace); + + mkFlag(0, "no-gc-warning", "disable warning about not using ‘--add-root’", + &gcWarning, false); } + bool processFlag(Strings::iterator & pos, Strings::iterator end) override + { + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; + } + + bool processArgs(const Strings & args, bool finish) override + { + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } +}; + + +void parseCmdLine(int argc, char * * argv, + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) +{ + LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv)); settings.update(); } @@ -239,6 +240,20 @@ void parseCmdLine(int argc, char * * argv, void printVersion(const string & programName) { std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; + if (verbosity > lvlInfo) { + Strings cfg; +#if HAVE_BOEHMGC + cfg.push_back("gc"); +#endif +#if HAVE_SODIUM + cfg.push_back("signed-caches"); +#endif + std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; + std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "Store directory: " << settings.nixStore << "\n"; + std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "Database directory: " << settings.nixDBPath << "\n"; + } throw Exit(); } @@ -262,8 +277,7 @@ int handleExceptions(const string & programName, std::function<void()> fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - blockInt = 1; /* ignore further SIGINTs */ - _isInterrupted = 0; + interruptThrown = true; throw; } } catch (Exit & e) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index a7e6ef762120..6d94a22f788e 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,6 +1,7 @@ #pragma once #include "util.hh" +#include "args.hh" #include <signal.h> @@ -9,8 +10,6 @@ namespace nix { -MakeError(UsageError, nix::Error); - class Exit : public std::exception { public: @@ -19,8 +18,6 @@ public: Exit(int status) : status(status) { } }; -class StoreAPI; - int handleExceptions(const string & programName, std::function<void()> fun); void initNix(); @@ -33,9 +30,11 @@ void printVersion(const string & programName); /* Ugh. No better place to put this. */ void printGCWarning(); -void printMissing(StoreAPI & store, const PathSet & paths); +class Store; + +void printMissing(ref<Store> store, const PathSet & paths); -void printMissing(const PathSet & willBuild, +void printMissing(ref<Store> store, const PathSet & willBuild, const PathSet & willSubstitute, const PathSet & unknown, unsigned long long downloadSize, unsigned long long narSize); @@ -66,6 +65,7 @@ template<class N> N getIntArg(const string & opt, return n * multiplier; } + /* Show the manual page for the specified program. */ void showManPage(const string & name); @@ -88,7 +88,7 @@ extern volatile ::sig_atomic_t blockInt; string showBytes(unsigned long long bytes); -class GCResults; +struct GCResults; struct PrintFreed { @@ -100,4 +100,8 @@ struct PrintFreed }; +/* Install a SIGSEGV handler to detect stack overflows. */ +void detectStackOverflow(); + + } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc new file mode 100644 index 000000000000..58cb87a516b9 --- /dev/null +++ b/src/libstore/binary-cache-store.cc @@ -0,0 +1,299 @@ +#include "archive.hh" +#include "binary-cache-store.hh" +#include "compression.hh" +#include "derivations.hh" +#include "fs-accessor.hh" +#include "globals.hh" +#include "nar-info.hh" +#include "sync.hh" +#include "worker-protocol.hh" +#include "nar-accessor.hh" +#include "nar-info-disk-cache.hh" + +#include <chrono> + +namespace nix { + +BinaryCacheStore::BinaryCacheStore(const StoreParams & params) + : compression(get(params, "compression", "xz")) +{ + auto secretKeyFile = get(params, "secret-key", ""); + if (secretKeyFile != "") + secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); + + StringSink sink; + sink << narVersionMagic1; + narMagic = *sink.s; +} + +void BinaryCacheStore::init() +{ + std::string cacheInfoFile = "nix-cache-info"; + + auto cacheInfo = getFile(cacheInfoFile); + if (!cacheInfo) { + upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n"); + } else { + for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + auto name = line.substr(0, colon); + auto value = trim(line.substr(colon + 1, std::string::npos)); + if (name == "StoreDir") { + if (value != settings.nixStore) + throw Error(format("binary cache ‘%s’ is for Nix stores with prefix ‘%s’, not ‘%s’") + % getUri() % value % settings.nixStore); + } else if (name == "WantMassQuery") { + wantMassQuery_ = value == "1"; + } else if (name == "Priority") { + string2Int(value, priority); + } + } + } +} + +void BinaryCacheStore::notImpl() +{ + throw Error("operation not implemented for binary cache stores"); +} + +Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +{ + assertStorePath(storePath); + return storePathToHash(storePath) + ".narinfo"; +} + +void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar, bool repair) +{ + if (!repair && isValidPath(info.path)) return; + + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) + try { + if (ref != info.path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error(format("cannot add ‘%s’ to the binary cache because the reference ‘%s’ is not valid") + % info.path % ref); + } + + auto narInfoFile = narInfoFileFor(info.path); + + assert(nar.compare(0, narMagic.size(), narMagic) == 0); + + auto narInfo = make_ref<NarInfo>(info); + + narInfo->narSize = nar.size(); + narInfo->narHash = hashString(htSHA256, nar); + + if (info.narHash && info.narHash != narInfo->narHash) + throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + + /* Compress the NAR. */ + narInfo->compression = compression; + auto now1 = std::chrono::steady_clock::now(); + auto narCompressed = compress(compression, nar); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") + % narInfo->path % narInfo->narSize + % ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0) + % duration); + + /* Atomically write the NAR file. */ + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + ""); + if (repair || !fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, *narCompressed); + } else + stats.narWriteAverted++; + + stats.narWriteBytes += nar.size(); + stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) narInfo->sign(*secretKey); + + upsertFile(narInfoFile, narInfo->to_string()); + + auto hashPart = storePathToHash(narInfo->path); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); + } + + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); + + stats.narInfoWrite++; +} + +bool BinaryCacheStore::isValidPathUncached(const Path & storePath) +{ + // FIXME: this only checks whether a .narinfo with a matching hash + // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even + // though they shouldn't. Not easily fixed. + return fileExists(narInfoFileFor(storePath)); +} + +void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) +{ + auto info = queryPathInfo(storePath).cast<const NarInfo>(); + + auto nar = getFile(info->url); + + if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % info->url); + + stats.narRead++; + stats.narReadCompressedBytes += nar->size(); + + /* Decompress the NAR. FIXME: would be nice to have the remote + side do this. */ + try { + nar = decompress(info->compression, *nar); + } catch (UnknownCompressionMethod &) { + throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’") + % storePath % info->compression); + } + + stats.narReadBytes += nar->size(); + + printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar->size()); + + assert(nar->size() % 8 == 0); + + sink((unsigned char *) nar->c_str(), nar->size()); +} + +std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Path & storePath) +{ + auto narInfoFile = narInfoFileFor(storePath); + auto data = getFile(narInfoFile); + if (!data) return 0; + + auto narInfo = make_ref<NarInfo>(*data, narInfoFile); + + stats.narInfoRead++; + + return std::shared_ptr<NarInfo>(narInfo); +} + +Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) +{ + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info; + info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); + + addToStore(info, *sink.s, repair); + + return info.path; +} + +Path BinaryCacheStore::addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) +{ + ValidPathInfo info; + info.path = computeStorePathForText(name, s, references); + info.references = references; + + if (repair || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); + addToStore(info, *sink.s, repair); + } + + return info.path; +} + +/* Given requests for a path /nix/store/<x>/<y>, this accessor will + first download the NAR for /nix/store/<x> from the binary cache, + build a NAR accessor for that NAR, and use that to access <y>. */ +struct BinaryCacheStoreAccessor : public FSAccessor +{ + ref<BinaryCacheStore> store; + + std::map<Path, ref<FSAccessor>> nars; + + BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) + : store(store) + { + } + + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) + { + auto path = canonPath(path_); + + auto storePath = toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw Error(format("path ‘%1%’ is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + store->narFromPath(storePath, sink); + + auto accessor = makeNarAccessor(sink.s); + nars.emplace(storePath, accessor); + return {accessor, restPath}; + } + + Stat stat(const Path & path) override + { + auto res = fetch(path); + return res.first->stat(res.second); + } + + StringSet readDirectory(const Path & path) override + { + auto res = fetch(path); + return res.first->readDirectory(res.second); + } + + std::string readFile(const Path & path) override + { + auto res = fetch(path); + return res.first->readFile(res.second); + } + + std::string readLink(const Path & path) override + { + auto res = fetch(path); + return res.first->readLink(res.second); + } +}; + +ref<FSAccessor> BinaryCacheStore::getFSAccessor() +{ + return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>( + std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this()))); +} + +} diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh new file mode 100644 index 000000000000..c14ab8676a9c --- /dev/null +++ b/src/libstore/binary-cache-store.hh @@ -0,0 +1,136 @@ +#pragma once + +#include "crypto.hh" +#include "store-api.hh" + +#include "pool.hh" + +#include <atomic> + +namespace nix { + +struct NarInfo; + +class BinaryCacheStore : public Store +{ +private: + + std::unique_ptr<SecretKey> secretKey; + + std::string compression; + +protected: + + BinaryCacheStore(const StoreParams & params); + + [[noreturn]] void notImpl(); + + virtual bool fileExists(const std::string & path) = 0; + + virtual void upsertFile(const std::string & path, const std::string & data) = 0; + + /* Return the contents of the specified file, or null if it + doesn't exist. */ + virtual std::shared_ptr<std::string> getFile(const std::string & path) = 0; + + bool wantMassQuery_ = false; + int priority = 50; + +public: + + virtual void init(); + +private: + + std::string narMagic; + + std::string narInfoFileFor(const Path & storePath); + +public: + + bool isValidPathUncached(const Path & path) override; + + PathSet queryValidPaths(const PathSet & paths) override + { notImpl(); } + + PathSet queryAllValidPaths() override + { notImpl(); } + + std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override; + + void queryReferrers(const Path & path, + PathSet & referrers) override + { notImpl(); } + + PathSet queryValidDerivers(const Path & path) override + { return {}; } + + PathSet queryDerivationOutputs(const Path & path) override + { notImpl(); } + + StringSet queryDerivationOutputNames(const Path & path) override + { notImpl(); } + + Path queryPathFromHashPart(const string & hashPart) override + { notImpl(); } + + PathSet querySubstitutablePaths(const PathSet & paths) override + { return {}; } + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) + { } + + bool wantMassQuery() { return wantMassQuery_; } + + void addToStore(const ValidPathInfo & info, const std::string & nar, + bool repair = false) override; + + Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, bool repair = false) override; + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair = false) override; + + void narFromPath(const Path & path, Sink & sink) override; + + void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override + { notImpl(); } + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { notImpl(); } + + void ensurePath(const Path & path) override + { notImpl(); } + + void addTempRoot(const Path & path) override + { notImpl(); } + + void addIndirectRoot(const Path & path) override + { notImpl(); } + + void syncWithGC() override + { } + + Roots findRoots() override + { notImpl(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { notImpl(); } + + void optimiseStore() override + { } + + bool verifyStore(bool checkContents, bool repair) override + { return true; } + + ref<FSAccessor> getFSAccessor() override; + + void addSignatures(const Path & storePath, const StringSet & sigs) + { notImpl(); } + +}; + +} diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8c4412f11a8b..cca357dfb31b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2,16 +2,21 @@ #include "references.hh" #include "pathlocks.hh" -#include "misc.hh" #include "globals.hh" #include "local-store.hh" #include "util.hh" #include "archive.hh" #include "affinity.hh" +#include "builtins.hh" +#include "finally.hh" +#include "compression.hh" +#include <algorithm> +#include <iostream> #include <map> #include <sstream> -#include <algorithm> +#include <thread> +#include <future> #include <limits.h> #include <time.h> @@ -21,57 +26,35 @@ #include <sys/stat.h> #include <sys/utsname.h> #include <sys/select.h> +#include <sys/resource.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> -#include <stdio.h> #include <cstring> #include <pwd.h> #include <grp.h> -#include <bzlib.h> - -/* Includes required for chroot support. */ -#if HAVE_SYS_PARAM_H -#include <sys/param.h> -#endif -#if HAVE_SYS_MOUNT_H -#include <sys/mount.h> -#endif -#if HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#if HAVE_SCHED_H -#include <sched.h> -#endif - -/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but - <linux/fs.h> does. */ -#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H -#include <linux/fs.h> -#endif - -#define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root) - /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - #define SANDBOX_ENABLED 1 #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" #else - #define SANDBOX_ENABLED 0 - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/bin" "/usr/bin" + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" #endif -#if CHROOT_ENABLED +/* Includes required for chroot support. */ +#if __linux__ #include <sys/socket.h> #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#endif - -#if __linux__ #include <sys/personality.h> +#include <sys/mman.h> +#include <sched.h> +#include <sys/param.h> +#include <sys/mount.h> +#include <sys/syscall.h> +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif #if HAVE_STATVFS @@ -94,6 +77,7 @@ struct HookInstance; /* A pointer to a goal. */ class Goal; +class DerivationGoal; typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; @@ -184,10 +168,10 @@ public: return exitCode; } - /* Cancel the goal. It should wake up its waiters, get rid of any - running child processes that are being monitored by the worker - (important!), etc. */ - virtual void cancel(bool timeout) = 0; + /* Callback in case of a timeout. It should wake up its waiters, + get rid of any running child processes that are being monitored + by the worker (important!), etc. */ + virtual void timedOut() = 0; virtual string key() = 0; @@ -216,8 +200,6 @@ struct Child time_t timeStarted; }; -typedef map<pid_t, Child> Children; - /* The worker class. */ class Worker @@ -237,7 +219,7 @@ private: WeakGoals wantingToBuild; /* Child processes currently running. */ - Children children; + std::list<Child> children; /* Number of build slots occupied. This includes local builds and substitutions but not remote builds via the build hook. */ @@ -257,6 +239,9 @@ private: /* Last time the goals in `waitingForAWhile' where woken up. */ time_t lastWokenUp; + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + public: /* Set if at least one derivation had a BuildError (i.e. permanent @@ -275,6 +260,8 @@ public: /* Make a goal (with caching). */ GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath, + const BasicDerivation & drv, BuildMode buildMode = bmNormal); GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false); /* Remove a dead goal. */ @@ -290,14 +277,14 @@ public: /* Registers a running child process. `inBuildSlot' means that the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, pid_t pid, - const set<int> & fds, bool inBuildSlot, bool respectTimeouts); + void childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts); /* Unregisters a running child process. `wakeSleepers' should be false if there is no sense in waking up goals that are sleeping because they can't run yet (e.g., there is no free build slot, or the hook would still say `postpone'). */ - void childTerminated(pid_t pid, bool wakeSleepers = true); + void childTerminated(GoalPtr goal, bool wakeSleepers = true); /* Put `goal' to sleep until a build slot becomes available (which might be right away). */ @@ -320,6 +307,12 @@ public: void waitForInput(); unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); }; @@ -330,8 +323,8 @@ void addToWeakGoals(WeakGoals & goals, GoalPtr p) { // FIXME: necessary? // FIXME: O(n) - foreach (WeakGoals::iterator, i, goals) - if (i->lock() == p) return; + for (auto & i : goals) + if (i.lock() == p) return; goals.push_back(p); } @@ -361,11 +354,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) /* If we failed and keepGoing is not set, we remove all remaining waitees. */ - foreach (Goals::iterator, i, waitees) { - GoalPtr goal = *i; + for (auto & goal : waitees) { WeakGoals waiters2; - foreach (WeakGoals::iterator, j, goal->waiters) - if (j->lock() != shared_from_this()) waiters2.push_back(*j); + for (auto & j : goal->waiters) + if (j.lock() != shared_from_this()) waiters2.push_back(j); goal->waiters = waiters2; } waitees.clear(); @@ -381,8 +373,8 @@ void Goal::amDone(ExitCode result) assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; - foreach (WeakGoals::iterator, i, waiters) { - GoalPtr goal = i->lock(); + for (auto & i : waiters) { + GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); } waiters.clear(); @@ -508,13 +500,13 @@ void UserLock::acquire() /* Find a user account that isn't currently in use for another build. */ - foreach (Strings::iterator, i, users) { - debug(format("trying user ‘%1%’") % *i); + for (auto & i : users) { + debug(format("trying user ‘%1%’") % i); - struct passwd * pw = getpwnam(i->c_str()); + struct passwd * pw = getpwnam(i.c_str()); if (!pw) throw Error(format("the user ‘%1%’ in the group ‘%2%’ does not exist") - % *i % settings.buildUsersGroup); + % i % settings.buildUsersGroup); createDirs(settings.nixStateDir + "/userpool"); @@ -532,7 +524,7 @@ void UserLock::acquire() if (lockFile(fd, ltWrite, false)) { fdUserLock = fd.borrow(); lockedPaths.insert(fnUserLock); - user = *i; + user = i; uid = pw->pw_uid; /* Sanity check... */ @@ -540,6 +532,7 @@ void UserLock::acquire() throw Error(format("the Nix user should not be a member of ‘%1%’") % settings.buildUsersGroup); +#if __linux__ /* Get the list of supplementary groups of this build user. This is usually either empty or contains a group such as "kvm". */ supplementaryGIDs.resize(10); @@ -550,6 +543,7 @@ void UserLock::acquire() throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name); supplementaryGIDs.resize(ngroups); +#endif return; } @@ -634,11 +628,14 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide, 4) == -1) throw SysError("dupping builder's stdout/stderr"); - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); + Strings args = { + baseNameOf(buildHook), + settings.thisSystem, + (format("%1%") % settings.maxSilentTime).str(), + (format("%1%") % settings.buildTimeout).str() + }; + + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); throw SysError(format("executing ‘%1%’") % buildHook); }); @@ -668,12 +665,12 @@ typedef map<string, string> HashRewrites; string rewriteHashes(string s, const HashRewrites & rewrites) { - foreach (HashRewrites::const_iterator, i, rewrites) { - assert(i->first.size() == i->second.size()); + for (auto & i : rewrites) { + assert(i.first.size() == i.second.size()); size_t j = 0; - while ((j = s.find(i->first, j)) != string::npos) { + while ((j = s.find(i.first, j)) != string::npos) { debug(format("rewriting @ %1%") % j); - s.replace(j, i->second.size(), i->second); + s.replace(j, i.second.size(), i.second); } } return s; @@ -690,6 +687,9 @@ class SubstitutionGoal; class DerivationGoal : public Goal { private: + /* Whether to use an on-disk .drv file. */ + bool useDerivation; + /* The path of the derivation. */ Path drvPath; @@ -698,14 +698,14 @@ private: StringSet wantedOutputs; /* Whether additional wanted outputs have been added. */ - bool needRestart; + bool needRestart = false; /* Whether to retry substituting the outputs after building the inputs. */ - bool retrySubstitution; + bool retrySubstitution = false; /* The derivation stored at drvPath. */ - Derivation drv; + std::unique_ptr<BasicDerivation> drv; /* The remainder is state held during the build. */ @@ -735,14 +735,22 @@ private: /* The temporary directory. */ Path tmpDir; + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + /* File descriptor for the log file. */ - FILE * fLogFile; - BZFILE * bzLogFile; AutoCloseFD fdLogFile; + std::shared_ptr<BufferedSink> logFileSink, logSink; /* Number of bytes received from the builder's stdout/stderr. */ unsigned long logSize; + /* The most recent log lines. */ + std::list<std::string> logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -750,16 +758,13 @@ private: std::shared_ptr<HookInstance> hook; /* Whether we're currently doing a chroot build. */ - bool useChroot; + bool useChroot = false; Path chrootRootDir; /* RAII object to delete the chroot directory. */ std::shared_ptr<AutoDelete> autoDelChroot; - /* All inputs that are regular files. */ - PathSet regularInputPaths; - /* Whether this is a fixed-output derivation. */ bool fixedOutput; @@ -772,6 +777,12 @@ private: typedef map<string, string> Environment; Environment env; +#if __APPLE__ + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; + AutoDelete autoDelSandbox; +#endif + /* Hash rewriting. */ HashRewrites rewritesToTmp, rewritesFromTmp; typedef map<Path, Path> RedirectedOutputs; @@ -784,18 +795,29 @@ private: temporary paths. */ PathSet redirectedBadOutputs; - /* Set of inodes seen during calls to canonicalisePathMetaData() - for this build's outputs. This needs to be shared between - outputs to allow hard links between outputs. */ - InodesSeen inodesSeen; + BuildResult result; + + /* The current round, if we're building multiple times. */ + unsigned int curRound = 1; + + unsigned int nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + ValidPathInfos prevInfos; public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); - void cancel(bool timeout); + void timedOut() override; - string key() + string key() override { /* Ensure that derivations get built in order of their name, i.e. a derivation named "aardvark" always comes before @@ -804,7 +826,7 @@ public: return "b$" + storePathToName(drvPath) + "$" + drvPath; } - void work(); + void work() override; Path getDrvPath() { @@ -814,9 +836,12 @@ public: /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); + BuildResult getResult() { return result; } + private: /* The states. */ - void init(); + void getDerivation(); + void loadDerivation(); void haveDerivation(); void outputsSubstituted(); void closureRepaired(); @@ -849,8 +874,9 @@ private: void deleteTmpDir(bool force); /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data); - void handleEOF(int fd); + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; + void flushLine(); /* Return the set of (in)valid paths. */ PathSet checkPathValidity(bool returnValid, bool checkHash); @@ -864,26 +890,43 @@ private: Path addHashRewrite(const Path & path); void repairClosure(); + + void done(BuildResult::Status status, const string & msg = ""); }; -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) +DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + Worker & worker, BuildMode buildMode) : Goal(worker) + , useDerivation(true) + , drvPath(drvPath) , wantedOutputs(wantedOutputs) - , needRestart(false) - , retrySubstitution(false) - , fLogFile(0) - , bzLogFile(0) - , useChroot(false) , buildMode(buildMode) { - this->drvPath = drvPath; - state = &DerivationGoal::init; + state = &DerivationGoal::getDerivation; name = (format("building of ‘%1%’") % drvPath).str(); trace("created"); } +DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(false) + , drvPath(drvPath) + , buildMode(buildMode) +{ + this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); + state = &DerivationGoal::haveDerivation; + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + trace("created"); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(drvPath); +} + + DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a @@ -897,7 +940,7 @@ DerivationGoal::~DerivationGoal() void DerivationGoal::killChild() { if (pid != -1) { - worker.childTerminated(pid); + worker.childTerminated(shared_from_this()); if (buildUser.enabled()) { /* If we're using a build user, then there is a tricky @@ -919,12 +962,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::cancel(bool timeout) +void DerivationGoal::timedOut() { - if (settings.printBuildTrace && timeout) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); killChild(); - amDone(ecFailed); + done(BuildResult::TimedOut); } @@ -943,42 +984,39 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs) wantedOutputs.clear(); needRestart = true; } else - foreach (StringSet::const_iterator, i, outputs) - if (wantedOutputs.find(*i) == wantedOutputs.end()) { - wantedOutputs.insert(*i); + for (auto & i : outputs) + if (wantedOutputs.find(i) == wantedOutputs.end()) { + wantedOutputs.insert(i); needRestart = true; } } -void DerivationGoal::init() +void DerivationGoal::getDerivation() { trace("init"); - if (settings.readOnlyMode) - throw Error(format("cannot build derivation ‘%1%’ - no write access to the Nix store") % drvPath); - /* The first thing to do is to make sure that the derivation exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { - haveDerivation(); + loadDerivation(); return; } addWaitee(worker.makeSubstitutionGoal(drvPath)); - state = &DerivationGoal::haveDerivation; + state = &DerivationGoal::loadDerivation; } -void DerivationGoal::haveDerivation() +void DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { printMsg(lvlError, format("cannot build missing derivation ‘%1%’") % drvPath); - amDone(ecFailed); + done(BuildResult::MiscFailure); return; } @@ -990,31 +1028,43 @@ void DerivationGoal::haveDerivation() assert(worker.store.isValidPath(drvPath)); /* Get the derivation. */ - drv = derivationFromPath(worker.store, drvPath); + drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath))); + + haveDerivation(); +} + + +void DerivationGoal::haveDerivation() +{ + trace("have derivation"); - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.addTempRoot(i->second.path); + for (auto & i : drv->outputs) + worker.store.addTempRoot(i.second.path); /* Check what outputs paths are not already valid. */ PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0 && buildMode == bmNormal) { - amDone(ecSuccess); + done(BuildResult::AlreadyValid); return; } - /* Check whether any output previously failed to build. If so, - don't bother. */ - foreach (PathSet::iterator, i, invalidOutputs) - if (pathFailed(*i)) return; + /* Reject doing a hash build of anything other than a fixed-output + derivation. */ + if (buildMode == bmHash) { + if (drv->outputs.size() != 1 || + drv->outputs.find("out") == drv->outputs.end() || + drv->outputs["out"].hashAlgo == "") + throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); + } /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && substitutesAllowed(drv)) - foreach (PathSet::iterator, i, invalidOutputs) - addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair)); + if (settings.useSubstitutes && drv->substitutesAllowed()) + for (auto & i : invalidOutputs) + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); @@ -1045,7 +1095,7 @@ void DerivationGoal::outputsSubstituted() unsigned int nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); if (buildMode == bmNormal && nrInvalid == 0) { - amDone(ecSuccess); + done(BuildResult::Substituted); return; } if (buildMode == bmRepair && nrInvalid == 0) { @@ -1063,11 +1113,17 @@ void DerivationGoal::outputsSubstituted() wantedOutputs = PathSet(); /* The inputs must be built before we can build this goal. */ - foreach (DerivationInputs::iterator, i, drv.inputDrvs) - addWaitee(worker.makeDerivationGoal(i->first, i->second, buildMode == bmRepair ? bmRepair : bmNormal)); + if (useDerivation) + for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); - foreach (PathSet::iterator, i, drv.inputSrcs) - addWaitee(worker.makeSubstitutionGoal(*i)); + for (auto & i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) continue; + if (!settings.useSubstitutes) + throw Error(format("dependency of ‘%1%’ of ‘%2%’ does not exist, and substitution is disabled") + % i % drvPath); + addWaitee(worker.makeSubstitutionGoal(i)); + } if (waitees.empty()) /* to prevent hang (no wake-up event) */ inputsRealised(); @@ -1085,40 +1141,42 @@ void DerivationGoal::repairClosure() /* Get the output closure. */ PathSet outputClosure; - foreach (DerivationOutputs::iterator, i, drv.outputs) - computeFSClosure(worker.store, i->second.path, outputClosure); + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second.path, outputClosure); + } /* Filter out our own outputs (which we have already checked). */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - outputClosure.erase(i->second.path); + for (auto & i : drv->outputs) + outputClosure.erase(i.second.path); /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output closure. */ PathSet inputClosure; - computeFSClosure(worker.store, drvPath, inputClosure); + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); std::map<Path, Path> outputsToDrv; - foreach (PathSet::iterator, i, inputClosure) - if (isDerivation(*i)) { - Derivation drv = derivationFromPath(worker.store, *i); - foreach (DerivationOutputs::iterator, j, drv.outputs) - outputsToDrv[j->second.path] = *i; + for (auto & i : inputClosure) + if (isDerivation(i)) { + Derivation drv = worker.store.derivationFromPath(i); + for (auto & j : drv.outputs) + outputsToDrv[j.second.path] = i; } /* Check each path (slow!). */ PathSet broken; - foreach (PathSet::iterator, i, outputClosure) { - if (worker.store.pathContentsGood(*i)) continue; - printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % *i % drvPath); - Path drvPath2 = outputsToDrv[*i]; + for (auto & i : outputClosure) { + if (worker.pathContentsGood(i)) continue; + printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); + Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") - addWaitee(worker.makeSubstitutionGoal(*i, true)); + addWaitee(worker.makeSubstitutionGoal(i, true)); else addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); } if (waitees.empty()) { - amDone(ecSuccess); + done(BuildResult::AlreadyValid); return; } @@ -1131,7 +1189,7 @@ void DerivationGoal::closureRepaired() trace("closure repaired"); if (nrFailed > 0) throw Error(format("some paths in the output closure of derivation ‘%1%’ could not be repaired") % drvPath); - amDone(ecSuccess); + done(BuildResult::AlreadyValid); } @@ -1140,10 +1198,12 @@ void DerivationGoal::inputsRealised() trace("all inputs realised"); if (nrFailed != 0) { + if (!useDerivation) + throw Error(format("some dependencies of ‘%1%’ are missing") % drvPath); printMsg(lvlError, format("cannot build derivation ‘%1%’: %2% dependencies couldn't be built") % drvPath % nrFailed); - amDone(ecFailed); + done(BuildResult::DependencyFailed); return; } @@ -1156,32 +1216,33 @@ void DerivationGoal::inputsRealised() running the build hook. */ /* The outputs are referenceable paths. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - debug(format("building path ‘%1%’") % i->second.path); - allPaths.insert(i->second.path); + for (auto & i : drv->outputs) { + debug(format("building path ‘%1%’") % i.second.path); + allPaths.insert(i.second.path); } /* Determine the full set of input paths. */ /* First, the input derivations. */ - foreach (DerivationInputs::iterator, i, drv.inputDrvs) { - /* Add the relevant output closures of the input derivation - `*i' as input paths. Only add the closures of output paths - that are specified as inputs. */ - assert(worker.store.isValidPath(i->first)); - Derivation inDrv = derivationFromPath(worker.store, i->first); - foreach (StringSet::iterator, j, i->second) - if (inDrv.outputs.find(*j) != inDrv.outputs.end()) - computeFSClosure(worker.store, inDrv.outputs[*j].path, inputPaths); - else - throw Error( - format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") - % drvPath % *j % i->first); - } + if (useDerivation) + for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i.first)); + Derivation inDrv = worker.store.derivationFromPath(i.first); + for (auto & j : i.second) + if (inDrv.outputs.find(j) != inDrv.outputs.end()) + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + else + throw Error( + format("derivation ‘%1%’ requires non-existent output ‘%2%’ from input derivation ‘%3%’") + % drvPath % j % i.first); + } /* Second, the input sources. */ - foreach (PathSet::iterator, i, drv.inputSrcs) - computeFSClosure(worker.store, *i, inputPaths); + for (auto & i : drv->inputSrcs) + worker.store.computeFSClosure(i, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1189,8 +1250,12 @@ void DerivationGoal::inputsRealised() /* Is this a fixed-output derivation? */ fixedOutput = true; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (i->second.hash == "") fixedOutput = false; + for (auto & i : drv->outputs) + if (i.second.hash == "") fixedOutput = false; + + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1200,35 +1265,6 @@ void DerivationGoal::inputsRealised() } -static bool canBuildLocally(const string & platform) -{ - return platform == settings.thisSystem -#if __linux__ - || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") -#endif - ; -} - - -static string get(const StringPairs & map, const string & key, const string & def = "") -{ - StringPairs::const_iterator i = map.find(key); - return i == map.end() ? def : i->second; -} - - -bool willBuildLocally(const Derivation & drv) -{ - return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform); -} - - -bool substitutesAllowed(const Derivation & drv) -{ - return get(drv.env, "allowSubstitutes", "1") == "1"; -} - - void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1238,10 +1274,10 @@ void DerivationGoal::tryToBuild() (It can't happen between here and the lockPaths() call below because we're not allowing multi-threading.) If so, put this goal to sleep until another goal finishes, then try again. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (pathIsLockedByMe(i->second.path)) { + for (auto & i : drv->outputs) + if (pathIsLockedByMe(i.second.path)) { debug(format("putting derivation ‘%1%’ to sleep because ‘%2%’ is locked by another goal") - % drvPath % i->second.path); + % drvPath % i.second.path); worker.waitForAnyGoal(shared_from_this()); return; } @@ -1251,7 +1287,7 @@ void DerivationGoal::tryToBuild() can't acquire the lock, then continue; hopefully some other goal can start a build, and if not, the main loop will sleep a few seconds and then retry this goal. */ - if (!outputLocks.lockPaths(outputPaths(drv), "", false)) { + if (!outputLocks.lockPaths(drv->outputPaths(), "", false)) { worker.waitForAWhile(shared_from_this()); return; } @@ -1264,38 +1300,30 @@ void DerivationGoal::tryToBuild() now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); - assert(buildMode != bmCheck || validPaths.size() == drv.outputs.size()); - if (buildMode != bmCheck && validPaths.size() == drv.outputs.size()) { + if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { debug(format("skipping build of derivation ‘%1%’, someone beat us to it") % drvPath); outputLocks.setDeletion(true); - amDone(ecSuccess); + done(BuildResult::AlreadyValid); return; } - missingPaths = outputPaths(drv); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) - foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i); + for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; + for (auto & i : drv->outputs) { + Path path = i.second.path; if (worker.store.isValidPath(path)) continue; - if (!pathExists(path)) continue; debug(format("removing invalid path ‘%1%’") % path); deletePath(path); } - /* Check again whether any output previously failed to build, - because some other process may have tried and failed before we - acquired the lock. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (pathFailed(i->second.path)) return; - /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || willBuildLocally(drv); + bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); /* Is the build hook willing to accept this job? */ if (!buildLocally) { @@ -1336,11 +1364,8 @@ void DerivationGoal::tryToBuild() printMsg(lvlError, e.msg()); outputLocks.unlock(); buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 0 % e.msg()); worker.permanentFailure = true; - amDone(ecFailed); + done(BuildResult::InputRejected, e.msg()); return; } @@ -1361,11 +1386,13 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) throw SysError(format("moving ‘%1%’ to ‘%2%’") % tmpPath % storePath); - if (pathExists(oldPath)) - deletePath(oldPath); + deletePath(oldPath); } +MakeError(NotDeterministic, BuildError) + + void DerivationGoal::buildDone() { trace("build done"); @@ -1374,22 +1401,14 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe --- just don't do that :-) */ - int status; - pid_t savedPid; - if (hook) { - savedPid = hook->pid; - status = hook->pid.wait(true); - } else { - /* !!! this could block! security problem! solution: kill the - child */ - savedPid = pid; - status = pid.wait(true); - } + /* !!! this could block! security problem! solution: kill the + child */ + int status = hook ? hook->pid.wait(true) : pid.wait(true); debug(format("builder process for ‘%1%’ finished") % drvPath); /* So the child is gone now. */ - worker.childTerminated(savedPid); + worker.childTerminated(shared_from_this()); /* Close the read side of the logger pipe. */ if (hook) { @@ -1436,15 +1455,23 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - foreach (PathSet::iterator, i, missingPaths) - if (pathExists(chrootRootDir + *i)) - rename((chrootRootDir + *i).c_str(), i->c_str()); + for (auto & i : missingPaths) + if (pathExists(chrootRootDir + i)) + rename((chrootRootDir + i).c_str(), i.c_str()); + + std::string msg = (format("builder for ‘%1%’ %2%") + % drvPath % statusToString(status)).str(); + + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto & line : logTail) + msg += "\n " + line; + } if (diskFull) - printMsg(lvlError, "note: build failure may have been caused by lack of free disk space"); + msg += "\nnote: build failure may have been caused by lack of free disk space"; - throw BuildError(format("builder for ‘%1%’ %2%") - % drvPath % statusToString(status)); + throw BuildError(msg); } /* Compute the FS closure of the outputs and register them as @@ -1452,19 +1479,28 @@ void DerivationGoal::buildDone() registerOutputs(); if (buildMode == bmCheck) { - amDone(ecSuccess); + done(BuildResult::Built); return; } /* Delete unused redirected outputs (when doing hash rewriting). */ - foreach (RedirectedOutputs::iterator, i, redirectedOutputs) - if (pathExists(i->second)) deletePath(i->second); + for (auto & i : redirectedOutputs) + deletePath(i.second); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ deleteTmpDir(true); + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + buildUser.release(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old @@ -1478,67 +1514,50 @@ void DerivationGoal::buildDone() outputLocks.unlock(); buildUser.release(); - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); - worker.timedOut = true; - } + BuildResult::Status st = BuildResult::MiscFailure; + + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) + st = BuildResult::TimedOut; else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") - % drvPath % status % e.msg()); } else { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 1 % e.msg()); - worker.permanentFailure = !fixedOutput && !diskFull; - - /* Register the outputs of this build as "failed" so we - won't try to build them again (negative caching). - However, don't do this for fixed-output derivations, - since they're likely to fail for transient reasons - (e.g., fetchurl not being able to access the network). - Hook errors (like communication problems with the - remote machine) shouldn't be cached either. */ - if (settings.cacheFailure && !fixedOutput && !diskFull) - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.registerFailedPath(i->second.path); + st = + dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + fixedOutput || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; } - amDone(ecFailed); + done(st, e.msg()); return; } /* Release the build user, if applicable. */ buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); - - amDone(ecSuccess); + done(BuildResult::Built); } HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; + if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; if (!worker.hook) - worker.hook = std::shared_ptr<HookInstance>(new HookInstance); + worker.hook = std::make_shared<HookInstance>(); /* Tell the hook about system features (beyond the system type) required from the build machine. (The hook could parse the drv file itself, but this is easier.) */ - Strings features = tokenizeString<Strings>(get(drv.env, "requiredSystemFeatures")); - foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ + Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures")); + for (auto & i : features) checkStoreName(i); /* !!! abuse */ /* Send the request to the hook. */ writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv.platform % drvPath % concatStringsSep(",", features)).str()); + % drv->platform % drvPath % concatStringsSep(",", features)).str()); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ @@ -1572,16 +1591,16 @@ HookReply DerivationGoal::tryBuildHook() list it since the remote system *probably* already has it.) */ PathSet allInputs; allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(worker.store, drvPath, allInputs); + worker.store.computeFSClosure(drvPath, allInputs); string s; - foreach (PathSet::iterator, i, allInputs) { s += *i; s += ' '; } + for (auto & i : allInputs) { s += i; s += ' '; } writeLine(hook->toHook.writeSide, s); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ s = ""; - foreach (PathSet::iterator, i, missingPaths) { s += *i; s += ' '; } + for (auto & i : missingPaths) { s += i; s += ' '; } writeLine(hook->toHook.writeSide, s); hook->toHook.writeSide.close(); @@ -1592,11 +1611,7 @@ HookReply DerivationGoal::tryBuildHook() set<int> fds; fds.insert(hook->fromHook.readSide); fds.insert(hook->builderOut.readSide); - worker.childStarted(shared_from_this(), hook->pid, fds, false, false); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); + worker.childStarted(shared_from_this(), fds, false, false); return rpAccept; } @@ -1618,21 +1633,55 @@ int childEntry(void * arg) void DerivationGoal::startBuilder() { - startNest(nest, lvlInfo, format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - "building path(s) %1%") % showPaths(missingPaths)); + auto f = format( + buildMode == bmRepair ? "repairing path(s) %1%" : + buildMode == bmCheck ? "checking path(s) %1%" : + nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : + "building path(s) %1%"); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + printMsg(lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ - if (!canBuildLocally(drv.platform)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform); + if (!drv->canBuildLocally()) { throw Error( format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’") - % drv.platform % settings.thisSystem % drvPath); + % drv->platform % settings.thisSystem % drvPath); + } + +#if __APPLE__ + additionalSandboxProfile = get(drv->env, "__sandboxProfile"); +#endif + + /* Are we doing a chroot build? Note that fixed-output + derivations are never done in a chroot, mainly so that + functions like fetchurl (which needs a proper /etc/resolv.conf) + work properly. Purity checking for fixed-output derivations + is somewhat pointless anyway. */ + { + string x = settings.get("build-use-sandbox", + /* deprecated alias */ + settings.get("build-use-chroot", string("false"))); + if (x != "true" && x != "false" && x != "relaxed") + throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); + if (x == "true") { + if (get(drv->env, "__noChroot") == "1") + throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " + "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error(format("derivation ‘%1%’ specifies a sandbox profile, " + "but this is only allowed when ‘build-use-sandbox’ is ‘relaxed’") % drvPath); +#endif + useChroot = true; + } + else if (x == "false") + useChroot = false; + else if (x == "relaxed") + useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; } /* Construct the environment passed to the builder. */ + env.clear(); /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when PATH is not set. We don't want this, so we fill it in with some dummy @@ -1659,38 +1708,44 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + storePathToName(drvPath), false, false, 0700); + auto drvName = storePathToName(drvPath); + tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + + /* In a sandbox, for determinism, always use the same temporary + directory. */ + tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; /* Add all bindings specified in the derivation via the environments, except those listed in the passAsFile attribute. Those are passed as file names pointing to temporary files containing the contents. */ PathSet filesToChown; - StringSet passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile")); + StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); int fileNr = 0; - for (auto & i : drv.env) { + for (auto & i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - Path p = tmpDir + "/.attr-" + int2String(fileNr++); + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; writeFile(p, i.second); filesToChown.insert(p); - env[i.first + "Path"] = p; + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } } /* For convenience, set an environment pointing to the top build directory. */ - env["NIX_BUILD_TOP"] = tmpDir; + env["NIX_BUILD_TOP"] = tmpDirInSandbox; /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir; + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; /* Explicitly set PWD to prevent problems with chroot builds. In particular, dietlibc cannot figure out the cwd because the inode of the current directory doesn't appear in .. (because getdents returns the inode of the mount point). */ - env["PWD"] = tmpDir; + env["PWD"] = tmpDirInSandbox; /* Compatibility hack with Nix <= 0.7: if this is a fixed-output derivation, tell the builder, so that for instance `fetchurl' @@ -1708,8 +1763,8 @@ void DerivationGoal::startBuilder() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv.env, "impureEnvVars")); - foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); + Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); + for (auto & i : varNames) env[i] = getEnv(i); } /* The `exportReferencesGraph' feature allows the references graph @@ -1719,7 +1774,7 @@ void DerivationGoal::startBuilder() temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - string s = get(drv.env, "exportReferencesGraph"); + string s = get(drv->env, "exportReferencesGraph"); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s); @@ -1742,14 +1797,14 @@ void DerivationGoal::startBuilder() like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ PathSet paths, paths2; - computeFSClosure(worker.store, storePath, paths); + worker.store.computeFSClosure(storePath, paths); paths2 = paths; - foreach (PathSet::iterator, j, paths2) { - if (isDerivation(*j)) { - Derivation drv = derivationFromPath(worker.store, *j); - foreach (DerivationOutputs::iterator, k, drv.outputs) - computeFSClosure(worker.store, k->second.path, paths); + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputs) + worker.store.computeFSClosure(k.second.path, paths); } } @@ -1779,40 +1834,27 @@ void DerivationGoal::startBuilder() } - /* Are we doing a chroot build? Note that fixed-output - derivations are never done in a chroot, mainly so that - functions like fetchurl (which needs a proper /etc/resolv.conf) - work properly. Purity checking for fixed-output derivations - is somewhat pointless anyway. */ - { - string x = settings.get("build-use-chroot", string("false")); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-chroot’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { - if (get(drv.env, "__noChroot") == "1") - throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, but that's not allowed when ‘build-use-chroot’ is ‘true’") % drvPath); - useChroot = true; - } - else if (x == "false") - useChroot = false; - else if (x == "relaxed") - useChroot = !fixedOutput && get(drv.env, "__noChroot") != "1"; - } - if (useChroot) { string defaultChrootDirs; -#if CHROOT_ENABLED +#if __linux__ if (isInStore(BASH_PATH)) defaultChrootDirs = "/bin/sh=" BASH_PATH; #endif /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", defaultChrootDirs)); - PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string(""))); + PathSet dirs = tokenizeString<StringSet>( + settings.get("build-sandbox-paths", + /* deprecated alias with lower priority */ + settings.get("build-chroot-dirs", defaultChrootDirs))); + PathSet dirs2 = tokenizeString<StringSet>( + settings.get("build-extra-chroot-dirs", + settings.get("build-extra-sandbox-paths", string("")))); dirs.insert(dirs2.begin(), dirs2.end()); + dirsInChroot.clear(); + for (auto & i : dirs) { size_t p = i.find('='); if (p == string::npos) @@ -1820,13 +1862,13 @@ void DerivationGoal::startBuilder() else dirsInChroot[string(i, 0, p)] = string(i, p + 1); } - dirsInChroot[tmpDir] = tmpDir; + dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ PathSet closure; for (auto & i : dirsInChroot) if (isInStore(i.second)) - computeFSClosure(worker.store, toStorePath(i.second), closure); + worker.store.computeFSClosure(toStorePath(i.second), closure); for (auto & i : closure) dirsInChroot[i] = i; @@ -1834,7 +1876,7 @@ void DerivationGoal::startBuilder() PathSet allowedPaths = tokenizeString<StringSet>(allowed); /* This works like the above, except on a per-derivation level */ - Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps")); + Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); for (auto & i : impurePaths) { bool found = false; @@ -1851,21 +1893,21 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); dirsInChroot[i] = i; } -#if CHROOT_ENABLED +#if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ chrootRootDir = drvPath + ".chroot"; - if (pathExists(chrootRootDir)) deletePath(chrootRootDir); + deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ - autoDelChroot = std::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir)); + autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); printMsg(lvlChatty, format("setting up chroot environment in ‘%1%’") % chrootRootDir); @@ -1918,44 +1960,42 @@ void DerivationGoal::startBuilder() if (chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1) throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir); - foreach (PathSet::iterator, i, inputPaths) { + for (auto & i : inputPaths) { struct stat st; - if (lstat(i->c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % *i); + if (lstat(i.c_str(), &st)) + throw SysError(format("getting attributes of path ‘%1%’") % i); if (S_ISDIR(st.st_mode)) - dirsInChroot[*i] = *i; + dirsInChroot[i] = i; else { - Path p = chrootRootDir + *i; - if (link(i->c_str(), p.c_str()) == -1) { + Path p = chrootRootDir + i; + if (link(i.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), which is quite possible after a `nix-store --optimise'. */ if (errno != EMLINK) - throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % *i); + throw SysError(format("linking ‘%1%’ to ‘%2%’") % p % i); StringSink sink; - dumpPath(*i, sink); - StringSource source(sink.s); + dumpPath(i, sink); + StringSource source(*sink.s); restorePath(p, source); } - - regularInputPaths.insert(*i); } } - /* If we're repairing or checking, it's possible that we're + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're rebuilding a path that is in settings.dirsInChroot (typically the dependencies of /bin/sh). Throw them out. */ - if (buildMode != bmNormal) - foreach (DerivationOutputs::iterator, i, drv.outputs) - dirsInChroot.erase(i->second.path); + for (auto & i : drv->outputs) + dirsInChroot.erase(i.second.path); -#elif SANDBOX_ENABLED +#elif __APPLE__ /* We don't really have any parent prep work to do (yet?) All work happens in the child, instead. */ #else - throw Error("chroot builds are not supported on this platform"); + throw Error("sandboxing builds is not supported on this platform"); #endif } @@ -1974,16 +2014,16 @@ void DerivationGoal::startBuilder() contents of the new outputs to replace the dummy strings with the actual hashes. */ if (validPaths.size() > 0) - foreach (PathSet::iterator, i, validPaths) - addHashRewrite(*i); + for (auto & i : validPaths) + addHashRewrite(i); /* If we're repairing, then we don't want to delete the corrupt outputs in advance. So rewrite them as well. */ if (buildMode == bmRepair) - foreach (PathSet::iterator, i, missingPaths) - if (worker.store.isValidPath(*i) && pathExists(*i)) { - addHashRewrite(*i); - redirectedBadOutputs.insert(*i); + for (auto & i : missingPaths) + if (worker.store.isValidPath(i) && pathExists(i)) { + addHashRewrite(i); + redirectedBadOutputs.insert(i); } } @@ -2001,10 +2041,10 @@ void DerivationGoal::startBuilder() auto lastPos = std::string::size_type{0}; for (auto nlPos = lines.find('\n'); nlPos != string::npos; nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos}; + auto line = std::string{lines, lastPos, nlPos - lastPos}; lastPos = nlPos + 1; if (state == stBegin) { - if (line == "extra-chroot-dirs") { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { throw Error(format("unknown pre-build hook command ‘%1%’") @@ -2025,7 +2065,7 @@ void DerivationGoal::startBuilder() } /* Run the builder. */ - printMsg(lvlChatty, format("executing builder ‘%1%’") % drv.builder); + printMsg(lvlChatty, format("executing builder ‘%1%’") % drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2034,7 +2074,7 @@ void DerivationGoal::startBuilder() builderOut.create(); /* Fork a child to build the package. */ -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2072,16 +2112,19 @@ void DerivationGoal::startBuilder() ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { - char stack[32 * 1024]; + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (!fixedOutput) flags |= CLONE_NEWNET; - pid_t child = clone(childEntry, stack + sizeof(stack) - 8, flags, this); + pid_t child = clone(childEntry, stack + stackSize, flags, this); if (child == -1 && errno == EINVAL) /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + sizeof(stack) - 8, flags & ~CLONE_NEWPID, this); + child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); if (child == -1) throw SysError("cloning builder process"); - writeFull(builderOut.writeSide, int2String(child) + "\n"); + writeFull(builderOut.writeSide, std::to_string(child) + "\n"); _exit(0); }, options); if (helper.wait(true) != 0) @@ -2093,7 +2136,7 @@ void DerivationGoal::startBuilder() #endif { ProcessOptions options; - options.allowVfork = !buildUser.enabled(); + options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); }, options); @@ -2102,16 +2145,16 @@ void DerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); builderOut.writeSide.close(); - worker.childStarted(shared_from_this(), pid, - singleton<set<int> >(builderOut.readSide), true, true); + worker.childStarted(shared_from_this(), {builderOut.readSide}, true, true); /* Check if setting up the build environment failed. */ - string msg = readLine(builderOut.readSide); - if (!msg.empty()) throw Error(msg); - - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv.platform % logFile); + while (true) { + string msg = readLine(builderOut.readSide); + if (string(msg, 0, 1) == "\1") { + if (msg.size() == 1) break; + throw Error(string(msg, 1)); + } + printMsg(lvlDebug, msg); } } @@ -2125,7 +2168,7 @@ void DerivationGoal::runChild() commonChildInit(builderOut); -#if CHROOT_ENABLED +#if __linux__ if (useChroot) { /* Initialise the loopback interface. */ @@ -2157,8 +2200,8 @@ void DerivationGoal::runChild() local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); - foreach (Strings::iterator, i, mounts) { - vector<string> fields = tokenizeString<vector<string> >(*i, " "); + for (auto & i : mounts) { + vector<string> fields = tokenizeString<vector<string> >(i, " "); string fs = decodeOctalEscaped(fields.at(4)); if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) throw SysError(format("unable to make filesystem ‘%1%’ private") % fs); @@ -2206,10 +2249,10 @@ void DerivationGoal::runChild() /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - foreach (DirsInChroot::iterator, i, dirsInChroot) { + for (auto & i : dirsInChroot) { struct stat st; - Path source = i->second; - Path target = chrootRootDir + i->first; + Path source = i.second; + Path target = chrootRootDir + i.first; if (source == "/proc") continue; // backwards compatibility debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target); if (stat(source.c_str(), &st) == -1) @@ -2258,10 +2301,8 @@ void DerivationGoal::runChild() if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) if (pivot_root(".", "real-root") == -1) throw SysError(format("cannot pivot old root directory onto ‘%1%’") % (chrootRootDir + "/real-root")); -#undef pivot_root if (chroot(".") == -1) throw SysError(format("cannot change root directory to ‘%1%’") % chrootRootDir); @@ -2274,7 +2315,7 @@ void DerivationGoal::runChild() } #endif - if (chdir(tmpDir.c_str()) == -1) + if (chdir(tmpDirInSandbox.c_str()) == -1) throw SysError(format("changing into ‘%1%’") % tmpDir); /* Close all other file descriptors. */ @@ -2285,7 +2326,7 @@ void DerivationGoal::runChild() i686-linux build on an x86_64-linux machine. */ struct utsname utsbuf; uname(&utsbuf); - if (drv.platform == "i686-linux" && + if (drv->platform == "i686-linux" && (settings.thisSystem == "x86_64-linux" || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { if (personality(PER_LINUX32) == -1) @@ -2294,7 +2335,7 @@ void DerivationGoal::runChild() /* Impersonate a Linux 2.6 machine to get some determinism in builds that depend on the kernel version. */ - if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) { + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { int cur = personality(0xffffffff); if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); } @@ -2305,10 +2346,16 @@ void DerivationGoal::runChild() if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); #endif + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + /* Fill in the environment. */ Strings envStrs; - foreach (Environment::const_iterator, i, env) - envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); + for (auto & i : env) + envStrs.push_back(rewriteHashes(i.first + "=" + i.second, rewritesToTmp)); /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop all root @@ -2319,7 +2366,8 @@ void DerivationGoal::runChild() if (buildUser.enabled()) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (setgroups(buildUser.getSupplementaryGIDs().size(), + if (!buildUser.getSupplementaryGIDs().empty() && + setgroups(buildUser.getSupplementaryGIDs().size(), buildUser.getSupplementaryGIDs().data()) == -1) throw SysError("cannot set supplementary groups of build user"); @@ -2340,7 +2388,10 @@ void DerivationGoal::runChild() const char *builder = "invalid"; string sandboxProfile; - if (useChroot && SANDBOX_ENABLED) { + if (drv->isBuiltin()) { + ; +#if __APPLE__ + } else if (useChroot) { /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ PathSet ancestry; @@ -2367,8 +2418,7 @@ void DerivationGoal::runChild() for (auto & i : inputPaths) dirsInChroot[i] = i; - - /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */ + /* This has to appear before import statements */ sandboxProfile += "(version 1)\n"; /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2378,15 +2428,6 @@ void DerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(allow file-read* file-write-data (literal \"/dev/null\"))\n"; - - sandboxProfile += "(allow file-read-metadata\n" - "\t(literal \"/var\")\n" - "\t(literal \"/tmp\")\n" - "\t(literal \"/etc\")\n" - "\t(literal \"/etc/nix\")\n" - "\t(literal \"/etc/nix/nix.conf\"))\n"; - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); @@ -2394,20 +2435,6 @@ void DerivationGoal::runChild() /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - /* This is where our temp folders are and where most of the building will happen, so we want rwx on it. */ - sandboxProfile += (format("(allow file-read* file-write* process-exec (subpath \"%1%\") (subpath \"/private/tmp\"))\n") % globalTmpDir).str(); - - sandboxProfile += "(allow process-fork)\n"; - sandboxProfile += "(allow sysctl-read)\n"; - sandboxProfile += "(allow signal (target same-sandbox))\n"; - - /* Enables getpwuid (used by git and others) */ - sandboxProfile += "(allow mach-lookup (global-name \"com.apple.system.notification_center\") (global-name \"com.apple.system.opendirectoryd.libinfo\"))\n"; - - /* Allow local networking operations, mostly because lots of test suites use it and it seems mostly harmless */ - sandboxProfile += "(allow network* (local ip) (remote unix-socket))"; - - /* Our rwx outputs */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : missingPaths) { @@ -2416,15 +2443,15 @@ void DerivationGoal::runChild() sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) - Note that the sandbox profile allows file-write* even though it isn't seemingly necessary. First of all, nix's standard user permissioning - mechanism still prevents builders from writing to input directories, so no security/purity is lost. The reason we allow file-write* is that - denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since - it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of - these is the entire GHC Haskell ecosystem. */ + + without file-write* allowed, access() incorrectly returns EPERM + */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second) - throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin")); + throw Error(format( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") + % i.first % i.second); string path = i.first; struct stat st; @@ -2437,46 +2464,66 @@ void DerivationGoal::runChild() } sandboxProfile += ")\n"; - /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that, - you open up the entire filesystem because you end up with (subpath "/") */ - sandboxProfile += "(allow file-read-metadata\n"; + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); } sandboxProfile += ")\n"; + sandboxProfile += additionalSandboxProfile; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + Path sandboxFile = drvPath + ".sb"; + deletePath(sandboxFile); + autoDelSandbox.reset(sandboxFile, false); + + writeFile(sandboxFile, sandboxProfile); + builder = "/usr/bin/sandbox-exec"; args.push_back("sandbox-exec"); - args.push_back("-p"); - args.push_back(sandboxProfile); - args.push_back(drv.builder); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back(drv->builder); +#endif } else { - builder = drv.builder.c_str(); - string builderBasename = baseNameOf(drv.builder); + builder = drv->builder.c_str(); + string builderBasename = baseNameOf(drv->builder); args.push_back(builderBasename); } - foreach (Strings::iterator, i, drv.args) - args.push_back(rewriteHashes(*i, rewritesToTmp)); + for (auto & i : drv->args) + args.push_back(rewriteHashes(i, rewritesToTmp)); restoreSIGPIPE(); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, "\n"); + writeFull(STDERR_FILENO, string("\1\n")); - /* This needs to be after that fateful '\n', and I didn't want to duplicate code */ - if (useChroot && SANDBOX_ENABLED) { - printMsg(lvlDebug, "Generated sandbox profile:"); - printMsg(lvlDebug, sandboxProfile); + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(*drv); + else + throw Error(format("unsupported builtin function ‘%1%’") % string(drv->builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } } - /* Execute the program. This should not return. */ execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing ‘%1%’") % drv.builder); + throw SysError(format("executing ‘%1%’") % drv->builder); } catch (std::exception & e) { - writeFull(STDERR_FILENO, "while setting up the build environment: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); _exit(1); } } @@ -2485,18 +2532,17 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) +PathSet parseReferenceSpecifiers(const BasicDerivation & drv, string attr) { PathSet result; Paths paths = tokenizeString<Paths>(attr); - foreach (Strings::iterator, i, paths) { - if (isStorePath(*i)) - result.insert(*i); - else if (drv.outputs.find(*i) != drv.outputs.end()) - result.insert(drv.outputs.find(*i)->second.path); + for (auto & i : paths) { + if (isStorePath(i)) + result.insert(i); + else if (drv.outputs.find(i) != drv.outputs.end()) + result.insert(drv.outputs.find(i)->second.path); else throw BuildError( - format("derivation contains an illegal reference specifier ‘%1%’") - % *i); + format("derivation contains an illegal reference specifier ‘%1%’") % i); } return result; } @@ -2509,18 +2555,25 @@ void DerivationGoal::registerOutputs() to do anything here. */ if (hook) { bool allValid = true; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!worker.store.isValidPath(i->second.path)) allValid = false; + for (auto & i : drv->outputs) + if (!worker.store.isValidPath(i.second.path)) allValid = false; if (allValid) return; } ValidPathInfos infos; + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = "-check"; + /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) { - Path path = i->second.path; + for (auto & i : drv->outputs) { + Path path = i.second.path; if (missingPaths.find(path) == missingPaths.end()) continue; Path actualPath = path; @@ -2532,7 +2585,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), path.c_str()) == -1) - throw SysError(format("moving build output ‘%1%’ from the chroot to the Nix store") % path); + throw SysError(format("moving build output ‘%1%’ from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = path; } else { @@ -2541,7 +2594,7 @@ void DerivationGoal::registerOutputs() && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && pathExists(redirected)) replaceValidPath(path, redirected); - if (buildMode == bmCheck) + if (buildMode == bmCheck && redirected != "") actualPath = redirected; } @@ -2578,23 +2631,22 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); deletePath(actualPath); - sink.s = rewriteHashes(sink.s, rewritesFromTmp); - StringSource source(sink.s); + sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp)); + StringSource source(*sink.s); restorePath(actualPath, source); rewritten = true; } - startNest(nest, lvlTalkative, - format("scanning for references inside ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - if (i->second.hash != "") { + if (i.second.hash != "") { bool recursive; HashType ht; Hash h; - i->second.parseHashInfo(recursive, ht, h); + i.second.parseHashInfo(recursive, ht, h); if (!recursive) { /* The output path should be a regular file without @@ -2604,12 +2656,28 @@ void DerivationGoal::registerOutputs() format("output path ‘%1%’ should be a non-executable regular file") % path); } - /* Check the hash. */ + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath); - if (h != h2) - throw BuildError( - format("output path ‘%1%’ should have %2% hash ‘%3%’, instead has ‘%4%’") - % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2)); + if (buildMode == bmHash) { + Path dest = makeFixedOutputPath(recursive, ht, h2, drv->env["name"]); + printMsg(lvlError, format("build produced path ‘%1%’ with %2% hash ‘%3%’") + % dest % printHashType(ht) % printHash16or32(h2)); + if (worker.store.isValidPath(dest)) + return; + if (actualPath != dest) { + PathLocks outputLocks({dest}); + deletePath(dest); + if (rename(actualPath.c_str(), dest.c_str()) == -1) + throw SysError(format("moving ‘%1%’ to ‘%2%’") % actualPath % dest); + } + path = actualPath = dest; + } else { + if (h != h2) + throw BuildError( + format("output path ‘%1%’ has %2% hash ‘%3%’ when ‘%4%’ was expected") + % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h)); + } } /* Get rid of all weird permissions. This also checks that @@ -2625,27 +2693,47 @@ void DerivationGoal::registerOutputs() PathSet references = scanForReferences(actualPath, allPaths, hash); if (buildMode == bmCheck) { - ValidPathInfo info = worker.store.queryPathInfo(path); - if (hash.first != info.hash) - throw Error(format("derivation ‘%1%’ may not be deterministic: hash mismatch in output ‘%2%’") % drvPath % path); + if (!worker.store.isValidPath(path)) continue; + auto info = *worker.store.queryPathInfo(path); + if (hash.first != info.narHash) { + if (settings.keepFailed) { + Path dst = path + checkSuffix; + deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % actualPath % dst); + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs from ‘%3%’") + % drvPath % path % dst); + } else + throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") + % drvPath % path); + } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + continue; } /* For debugging, print out the referenced and unreferenced paths. */ - foreach (PathSet::iterator, i, inputPaths) { - PathSet::iterator j = references.find(*i); + for (auto & i : inputPaths) { + PathSet::iterator j = references.find(i); if (j == references.end()) - debug(format("unreferenced input: ‘%1%’") % *i); + debug(format("unreferenced input: ‘%1%’") % i); else - debug(format("referenced input: ‘%1%’") % *i); + debug(format("referenced input: ‘%1%’") % i); } /* Enforce `allowedReferences' and friends. */ auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - if (drv.env.find(attrName) == drv.env.end()) return; + if (drv->env.find(attrName) == drv->env.end()) return; - PathSet spec = parseReferenceSpecifiers(drv, get(drv.env, attrName)); + PathSet spec = parseReferenceSpecifiers(*drv, get(drv->env, attrName)); PathSet used; if (recursive) { @@ -2653,18 +2741,29 @@ void DerivationGoal::registerOutputs() for (auto & i : references) /* Don't call computeFSClosure on ourselves. */ if (actualPath != i) - computeFSClosure(worker.store, i, used); + worker.store.computeFSClosure(i, used); } else used = references; + PathSet badPaths; + for (auto & i : used) if (allowed) { if (spec.find(i) == spec.end()) - throw BuildError(format("output (‘%1%’) is not allowed to refer to path ‘%2%’") % actualPath % i); + badPaths.insert(i); } else { if (spec.find(i) != spec.end()) - throw BuildError(format("output (‘%1%’) is not allowed to refer to path ‘%2%’") % actualPath % i); + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n\t"; + badPathsStr += i; } + throw BuildError(format("output ‘%1%’ is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); + } }; checkRefs("allowedReferences", true, false); @@ -2672,21 +2771,63 @@ void DerivationGoal::registerOutputs() checkRefs("disallowedReferences", false, false); checkRefs("disallowedRequisites", false, true); - worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + if (curRound == nrRounds) { + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.markContentsGood(path); + } ValidPathInfo info; info.path = path; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; info.references = references; info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + infos.push_back(info); } if (buildMode == bmCheck) return; + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + Path prev = i->path + checkSuffix; + if (pathExists(prev)) + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") + % i->path % drvPath % prev); + else + throw NotDeterministic( + format("output ‘%1%’ of ‘%2%’ differs from previous round") + % i->path % drvPath); + } + assert(false); // shouldn't happen + } + + if (settings.keepFailed) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + if (curRound < nrRounds) { + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); + } + } + + } + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -2709,46 +2850,31 @@ Path DerivationGoal::openLogFile() Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str(); createDirs(dir); - if (settings.compressLog) { + Path logFileName = (format("%1%/%2%%3%") + % dir + % string(baseName, 2) + % (settings.compressLog ? ".bz2" : "")).str(); - Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str(); - AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fd == -1) throw SysError(format("creating log file ‘%1%’") % logFileName); - closeOnExec(fd); + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + if (fdLogFile == -1) throw SysError(format("creating log file ‘%1%’") % logFileName); - if (!(fLogFile = fdopen(fd.borrow(), "w"))) - throw SysError(format("opening file ‘%1%’") % logFileName); + logFileSink = std::make_shared<FdSink>(fdLogFile); - int err; - if (!(bzLogFile = BZ2_bzWriteOpen(&err, fLogFile, 9, 0, 0))) - throw Error(format("cannot open compressed log file ‘%1%’") % logFileName); - - return logFileName; + if (settings.compressLog) + logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink)); + else + logSink = logFileSink; - } else { - Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str(); - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666); - if (fdLogFile == -1) throw SysError(format("creating log file ‘%1%’") % logFileName); - closeOnExec(fdLogFile); - return logFileName; - } + return logFileName; } void DerivationGoal::closeLogFile() { - if (bzLogFile) { - int err; - BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0); - bzLogFile = 0; - if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err); - } - - if (fLogFile) { - fclose(fLogFile); - fLogFile = 0; - } - + auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink); + if (logSink2) logSink2->finish(); + if (logFileSink) logFileSink->flush(); + logSink = logFileSink = 0; fdLogFile.close(); } @@ -2779,59 +2905,61 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) printMsg(lvlError, format("%1% killed after writing more than %2% bytes of log output") % getName() % settings.maxLogSize); - cancel(true); // not really a timeout, but close enough + killChild(); + done(BuildResult::LogLimitExceeded); return; } - if (verbosity >= settings.buildVerbosity) - writeToStderr(filterANSIEscapes(data, true)); - if (bzLogFile) { - int err; - BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); - if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); - } else if (fdLogFile != -1) - writeFull(fdLogFile, data); + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + + if (logSink) (*logSink)(data); } if (hook && fd == hook->fromHook.readSide) - writeToStderr(data); + printMsg(lvlError, data); // FIXME? } void DerivationGoal::handleEOF(int fd) { + if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); } -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +void DerivationGoal::flushLine() { - PathSet result; - foreach (DerivationOutputs::iterator, i, drv.outputs) { - if (!wantOutput(i->first, wantedOutputs)) continue; - bool good = - worker.store.isValidPath(i->second.path) && - (!checkHash || worker.store.pathContentsGood(i->second.path)); - if (good == returnValid) result.insert(i->second.path); + if (settings.verboseBuild) + printMsg(lvlInfo, filterANSIEscapes(currentLogLine, true)); + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); } - return result; + currentLogLine = ""; + currentLogLinePos = 0; } -bool DerivationGoal::pathFailed(const Path & path) +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { - if (!settings.cacheFailure) return false; - - if (!worker.store.hasPathFailed(path)) return false; - - printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); - - worker.permanentFailure = true; - amDone(ecFailed); - - return true; + PathSet result; + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + bool good = + worker.store.isValidPath(i.second.path) && + (!checkHash || worker.pathContentsGood(i.second.path)); + if (good == returnValid) result.insert(i.second.path); + } + return result; } @@ -2840,7 +2968,7 @@ Path DerivationGoal::addHashRewrite(const Path & path) string h1 = string(path, settings.nixStore.size() + 1, 32); string h2 = string(printHash32(hashString(htSHA256, "rewrite:" + drvPath + ":" + path)), 0, 32); Path p = settings.nixStore + "/" + h2 + string(path, settings.nixStore.size() + 33); - if (pathExists(p)) deletePath(p); + deletePath(p); assert(path.size() == p.size()); rewritesToTmp[h1] = h2; rewritesFromTmp[h2] = h1; @@ -2849,6 +2977,18 @@ Path DerivationGoal::addHashRewrite(const Path & path) } +void DerivationGoal::done(BuildResult::Status status, const string & msg) +{ + result.status = status; + result.errorMsg = msg; + amDone(result.success() ? ecSuccess : ecFailed); + if (result.status == BuildResult::TimedOut) + worker.timedOut = true; + if (result.status == BuildResult::PermanentFailure) + worker.permanentFailure = true; +} + + ////////////////////////////////////////////////////////////////////// @@ -2861,28 +3001,24 @@ private: Path storePath; /* The remaining substituters. */ - Paths subs; + std::list<ref<Store>> subs; /* The current substituter. */ - Path sub; + std::shared_ptr<Store> sub; - /* Whether any substituter can realise this path */ + /* Whether any substituter can realise this path. */ bool hasSubstitute; /* Path info returned by the substituter's query info operation. */ - SubstitutablePathInfo info; + std::shared_ptr<const ValidPathInfo> info; /* Pipe for the substituter's standard output. */ Pipe outPipe; - /* Pipe for the substituter's standard error. */ - Pipe logPipe; + /* The substituter thread. */ + std::thread thr; - /* The process ID of the builder. */ - Pid pid; - - /* Lock on the store path. */ - std::shared_ptr<PathLocks> outputLock; + std::promise<void> promise; /* Whether to try to repair a valid path. */ bool repair; @@ -2898,7 +3034,7 @@ public: SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); ~SubstitutionGoal(); - void cancel(bool timeout); + void timedOut() { abort(); }; string key() { @@ -2939,20 +3075,14 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool SubstitutionGoal::~SubstitutionGoal() { - if (pid != -1) worker.childTerminated(pid); -} - - -void SubstitutionGoal::cancel(bool timeout) -{ - if (settings.printBuildTrace && timeout) - printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); - if (pid != -1) { - pid_t savedPid = pid; - pid.kill(); - worker.childTerminated(savedPid); + try { + if (thr.joinable()) { + thr.join(); + worker.childTerminated(shared_from_this()); + } + } catch (...) { + ignoreException(); } - amDone(ecFailed); } @@ -2977,7 +3107,7 @@ void SubstitutionGoal::init() if (settings.readOnlyMode) throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath); - subs = settings.substituters; + subs = getDefaultSubstituters(); tryNext(); } @@ -2991,6 +3121,7 @@ void SubstitutionGoal::tryNext() /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path ‘%1%’ is required, but there is no substituter that can build it") % storePath); + /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ @@ -3001,19 +3132,31 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - SubstitutablePathInfos infos; - PathSet dummy(singleton<PathSet>(storePath)); - worker.store.querySubstitutablePathInfos(sub, dummy, infos); - SubstitutablePathInfos::iterator k = infos.find(storePath); - if (k == infos.end()) { tryNext(); return; } - info = k->second; + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath &) { + tryNext(); + return; + } + hasSubstitute = true; + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (worker.store.requireSigs && !info->checkSignatures(worker.store.publicKeys)) { + printMsg(lvlInfo, format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’") + % sub->getUri() % storePath); + tryNext(); + return; + } + /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - foreach (PathSet::iterator, i, info.references) - if (*i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(*i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -3032,9 +3175,9 @@ void SubstitutionGoal::referencesValid() return; } - foreach (PathSet::iterator, i, info.references) - if (*i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(*i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); state = &SubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); @@ -3054,76 +3197,29 @@ void SubstitutionGoal::tryToRun() return; } - /* Maybe a derivation goal has already locked this path - (exceedingly unlikely, since it should have used a substitute - first, but let's be defensive). */ - outputLock.reset(); // make sure this goal's lock is gone - if (pathIsLockedByMe(storePath)) { - debug(format("restarting substitution of ‘%1%’ because it's locked by another goal") - % storePath); - worker.waitForAnyGoal(shared_from_this()); - return; /* restart in the tryToRun() state when another goal finishes */ - } - - /* Acquire a lock on the output path. */ - outputLock = std::shared_ptr<PathLocks>(new PathLocks); - if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Check again whether the path is invalid. */ - if (!repair && worker.store.isValidPath(storePath)) { - debug(format("store path ‘%1%’ has become valid") % storePath); - outputLock->setDeletion(true); - amDone(ecSuccess); - return; - } - printMsg(lvlInfo, format("fetching path ‘%1%’...") % storePath); outPipe.create(); - logPipe.create(); - - destPath = repair ? storePath + ".tmp" : storePath; - - /* Remove the (stale) output path if it exists. */ - if (pathExists(destPath)) - deletePath(destPath); - - worker.store.setSubstituterEnv(); - - /* Fill in the arguments. */ - Strings args; - args.push_back(baseNameOf(sub)); - args.push_back("--substitute"); - args.push_back(storePath); - args.push_back(destPath); - - /* Fork the substitute program. */ - pid = startProcess([&]() { - commonChildInit(logPipe); + promise = std::promise<void>(); - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide.close(); }); - execv(sub.c_str(), stringsToCharPtrs(args).data()); + copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), + storePath, repair); - throw SysError(format("executing ‘%1%’") % sub); + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } }); - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - outPipe.writeSide.close(); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton<set<int> >(logPipe.readSide), true, true); + worker.childStarted(shared_from_this(), {outPipe.readSide}, true, false); state = &SubstitutionGoal::finished; - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); } @@ -3131,110 +3227,40 @@ void SubstitutionGoal::finished() { trace("substitute finished"); - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - pid_t savedPid = pid; - int status = pid.wait(true); + thr.join(); + worker.childTerminated(shared_from_this()); - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); - - /* Get the hash info from stdout. */ - string dummy = readLine(outPipe.readSide); - string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; - outPipe.readSide.close(); - - /* Check the exit status and the build result. */ - HashResult hash; try { - - if (!statusOk(status)) - throw SubstError(format("fetching path ‘%1%’ %2%") - % storePath % statusToString(status)); - - if (!pathExists(destPath)) - throw SubstError(format("substitute did not produce path ‘%1%’") % destPath); - - hash = hashPath(htSHA256, destPath); - - /* Verify the expected hash we got from the substituer. */ - if (expectedHashStr != "") { - size_t n = expectedHashStr.find(':'); - if (n == string::npos) - throw Error(format("bad hash from substituter: %1%") % expectedHashStr); - HashType hashType = parseHashType(string(expectedHashStr, 0, n)); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm in ‘%1%’") % expectedHashStr); - Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); - Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) - throw SubstError(format("hash mismatch in downloaded path ‘%1%’: expected %2%, got %3%") - % storePath % printHash(expectedHash) % printHash(actualHash)); - } - - } catch (SubstError & e) { - + promise.get_future().get(); + } catch (Error & e) { printMsg(lvlInfo, e.msg()); - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") - % storePath % status % e.msg()); - } - /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } - if (repair) replaceValidPath(storePath, destPath); - - canonicalisePathMetaData(storePath, -1); - - worker.store.optimisePath(storePath); // FIXME: combine with hashPath() - - ValidPathInfo info2; - info2.path = storePath; - info2.hash = hash.first; - info2.narSize = hash.second; - info2.references = info.references; - info2.deriver = info.deriver; - worker.store.registerValidPath(info2); - - outputLock->setDeletion(true); - outputLock.reset(); - - worker.store.markContentsGood(storePath); + worker.markContentsGood(storePath); printMsg(lvlChatty, format("substitution of path ‘%1%’ succeeded") % storePath); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); - amDone(ecSuccess); } void SubstitutionGoal::handleChildOutput(int fd, const string & data) { - assert(fd == logPipe.readSide); - if (verbosity >= settings.buildVerbosity) writeToStderr(data); - /* Don't write substitution output to a log file for now. We - probably should, though. */ } void SubstitutionGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + if (fd == outPipe.readSide) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -3266,11 +3292,12 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, BuildMode buildMode) +GoalPtr Worker::makeDerivationGoal(const Path & path, + const StringSet & wantedOutputs, BuildMode buildMode) { GoalPtr goal = derivationGoals[path].lock(); if (!goal) { - goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, buildMode)); + goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); derivationGoals[path] = goal; wakeUp(goal); } else @@ -3279,11 +3306,20 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOu } +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drvPath, + const BasicDerivation & drv, BuildMode buildMode) +{ + auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); + wakeUp(goal); + return goal; +} + + GoalPtr Worker::makeSubstitutionGoal(const Path & path, bool repair) { GoalPtr goal = substitutionGoals[path].lock(); if (!goal) { - goal = GoalPtr(new SubstitutionGoal(path, *this, repair)); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair); substitutionGoals[path] = goal; wakeUp(goal); } @@ -3318,8 +3354,8 @@ void Worker::removeGoal(GoalPtr goal) } /* Wake up goals waiting for any goal to finish. */ - foreach (WeakGoals::iterator, i, waitingForAnyGoal) { - GoalPtr goal = i->lock(); + for (auto & i : waitingForAnyGoal) { + GoalPtr goal = i.lock(); if (goal) wakeUp(goal); } @@ -3340,9 +3376,8 @@ unsigned Worker::getNrLocalBuilds() } -void Worker::childStarted(GoalPtr goal, - pid_t pid, const set<int> & fds, bool inBuildSlot, - bool respectTimeouts) +void Worker::childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; @@ -3350,30 +3385,29 @@ void Worker::childStarted(GoalPtr goal, child.timeStarted = child.lastOutput = time(0); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; - children[pid] = child; + children.emplace_back(child); if (inBuildSlot) nrLocalBuilds++; } -void Worker::childTerminated(pid_t pid, bool wakeSleepers) +void Worker::childTerminated(GoalPtr goal, bool wakeSleepers) { - assert(pid != -1); /* common mistake */ - - Children::iterator i = children.find(pid); + auto i = std::find_if(children.begin(), children.end(), + [&](const Child & child) { return child.goal.lock() == goal; }); assert(i != children.end()); - if (i->second.inBuildSlot) { + if (i->inBuildSlot) { assert(nrLocalBuilds > 0); nrLocalBuilds--; } - children.erase(pid); + children.erase(i); if (wakeSleepers) { /* Wake up goals waiting for a build slot. */ - foreach (WeakGoals::iterator, i, wantingToBuild) { - GoalPtr goal = i->lock(); + for (auto & j : wantingToBuild) { + GoalPtr goal = j.lock(); if (goal) wakeUp(goal); } @@ -3408,9 +3442,9 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - foreach (Goals::iterator, i, _topGoals) topGoals.insert(*i); + for (auto & i : _topGoals) topGoals.insert(i); - startNest(nest, lvlDebug, format("entered goal loop")); + Activity act(*logger, lvlDebug, "entered goal loop"); while (1) { @@ -3474,12 +3508,12 @@ void Worker::waitForInput() deadline for any child. */ assert(sizeof(time_t) >= sizeof(long)); time_t nearest = LONG_MAX; // nearest deadline - foreach (Children::iterator, i, children) { - if (!i->second.respectTimeouts) continue; + for (auto & i : children) { + if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i->second.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i->second.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); } if (nearest != LONG_MAX) { timeout.tv_sec = std::max((time_t) 1, nearest - before); @@ -3497,17 +3531,16 @@ void Worker::waitForInput() timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); } else lastWokenUp = 0; - using namespace std; /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ fd_set fds; FD_ZERO(&fds); int fdMax = 0; - foreach (Children::iterator, i, children) { - foreach (set<int>::iterator, j, i->second.fds) { - FD_SET(*j, &fds); - if (*j >= fdMax) fdMax = *j + 1; + for (auto & i : children) { + for (auto & j : i.fds) { + FD_SET(j, &fds); + if (j >= fdMax) fdMax = j + 1; } } @@ -3519,73 +3552,65 @@ void Worker::waitForInput() time_t after = time(0); /* Process all available file descriptors. */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); - /* Since goals may be canceled from inside the loop below (causing - them go be erased from the `children' map), we have to be - careful that we don't keep iterators alive across calls to - cancel(). */ - set<pid_t> pids; - foreach (Children::iterator, i, children) pids.insert(i->first); - - foreach (set<pid_t>::iterator, i, pids) { checkInterrupt(); - Children::iterator j = children.find(*i); - if (j == children.end()) continue; // child destroyed - GoalPtr goal = j->second.goal.lock(); + + GoalPtr goal = j->goal.lock(); assert(goal); - set<int> fds2(j->second.fds); - foreach (set<int>::iterator, k, fds2) { - if (FD_ISSET(*k, &fds)) { + set<int> fds2(j->fds); + for (auto & k : fds2) { + if (FD_ISSET(k, &fds)) { unsigned char buffer[4096]; - ssize_t rd = read(*k, buffer, sizeof(buffer)); + ssize_t rd = read(k, buffer, sizeof(buffer)); if (rd == -1) { if (errno != EINTR) throw SysError(format("reading from %1%") % goal->getName()); } else if (rd == 0) { debug(format("%1%: got EOF") % goal->getName()); - goal->handleEOF(*k); - j->second.fds.erase(*k); + goal->handleEOF(k); + j->fds.erase(k); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); string data((char *) buffer, rd); - j->second.lastOutput = after; - goal->handleChildOutput(*k, data); + j->lastOutput = after; + goal->handleChildOutput(k, data); } } } if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && - j->second.respectTimeouts && - after - j->second.lastOutput >= (time_t) settings.maxSilentTime) + j->respectTimeouts && + after - j->lastOutput >= (time_t) settings.maxSilentTime) { printMsg(lvlError, format("%1% timed out after %2% seconds of silence") % goal->getName() % settings.maxSilentTime); - goal->cancel(true); - timedOut = true; + goal->timedOut(); } else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && - j->second.respectTimeouts && - after - j->second.timeStarted >= (time_t) settings.buildTimeout) + j->respectTimeouts && + after - j->timeStarted >= (time_t) settings.buildTimeout) { printMsg(lvlError, format("%1% timed out after %2% seconds") % goal->getName() % settings.buildTimeout); - goal->cancel(true); - timedOut = true; + goal->timedOut(); } } - if (!waitingForAWhile.empty() && lastWokenUp + settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { lastWokenUp = after; - foreach (WeakGoals::iterator, i, waitingForAWhile) { - GoalPtr goal = i->lock(); + for (auto & i : waitingForAWhile) { + GoalPtr goal = i.lock(); if (goal) wakeUp(goal); } waitingForAWhile.clear(); @@ -3599,33 +3624,56 @@ unsigned int Worker::exitStatus() } +bool Worker::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path ‘%1%’...") % path); + auto info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info->narHash.type, path); + Hash nullHash(htSHA256); + res = info->narHash == nullHash || info->narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); + return res; +} + + +void Worker::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + ////////////////////////////////////////////////////////////////////// void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - startNest(nest, lvlDebug, - format("building %1%") % showPaths(drvPaths)); - Worker worker(*this); Goals goals; - foreach (PathSet::const_iterator, i, drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + for (auto & i : drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); if (isDerivation(i2.first)) goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(*i, buildMode)); + goals.insert(worker.makeSubstitutionGoal(i, buildMode)); } worker.run(goals); PathSet failed; - foreach (Goals::iterator, i, goals) - if ((*i)->getExitCode() == Goal::ecFailed) { - DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get()); + for (auto & i : goals) + if (i->getExitCode() == Goal::ecFailed) { + DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath()); + else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } if (!failed.empty()) @@ -3633,6 +3681,26 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) } +BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + Worker worker(*this); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + + BuildResult result; + + try { + worker.run(Goals{goal}); + result = goal->getResult(); + } catch (Error & e) { + result.status = BuildResult::MiscFailure; + result.errorMsg = e.msg(); + } + + return result; +} + + void LocalStore::ensurePath(const Path & path) { /* If the path is already valid, we're done. */ @@ -3640,7 +3708,7 @@ void LocalStore::ensurePath(const Path & path) Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = singleton<Goals>(goal); + Goals goals = {goal}; worker.run(goals); @@ -3653,12 +3721,21 @@ void LocalStore::repairPath(const Path & path) { Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path, true); - Goals goals = singleton<Goals>(goal); + Goals goals = {goal}; worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + auto deriver = queryPathInfo(path)->deriver; + if (deriver != "" && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(format("cannot repair path ‘%1%’") % path, worker.exitStatus()); + } } diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc new file mode 100644 index 000000000000..e1ce36e922e1 --- /dev/null +++ b/src/libstore/builtins.cc @@ -0,0 +1,47 @@ +#include "builtins.hh" +#include "download.hh" +#include "store-api.hh" +#include "archive.hh" +#include "compression.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation & drv) +{ + auto url = drv.env.find("url"); + if (url == drv.env.end()) throw Error("attribute ‘url’ missing"); + + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadOptions options; + options.verifyTLS = false; + + /* Show a progress indicator, even though stderr is not a tty. */ + options.showProgress = DownloadOptions::yes; + + auto data = makeDownloader()->download(url->second, options); + assert(data.data); + + auto out = drv.env.find("out"); + if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); + + Path storePath = out->second; + assertStorePath(storePath); + + auto unpack = drv.env.find("unpack"); + if (unpack != drv.env.end() && unpack->second == "1") { + if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) + data.data = decompress("xz", *data.data); + StringSource source(*data.data); + restorePath(storePath, source); + } else + writeFile(storePath, *data.data); + + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) + throw SysError(format("making ‘%1%’ executable") % storePath); + } +} + +} diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh new file mode 100644 index 000000000000..4b2431aa08cf --- /dev/null +++ b/src/libstore/builtins.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "derivations.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation & drv); + +} diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc new file mode 100644 index 000000000000..747483afb30b --- /dev/null +++ b/src/libstore/crypto.cc @@ -0,0 +1,126 @@ +#include "crypto.hh" +#include "util.hh" +#include "globals.hh" + +#if HAVE_SODIUM +#include <sodium.h> +#endif + +namespace nix { + +static std::pair<std::string, std::string> split(const string & s) +{ + size_t colon = s.find(':'); + if (colon == std::string::npos || colon == 0) + return {"", ""}; + return {std::string(s, 0, colon), std::string(s, colon + 1)}; +} + +Key::Key(const string & s) +{ + auto ss = split(s); + + name = ss.first; + key = ss.second; + + if (name == "" || key == "") + throw Error("secret key is corrupt"); + + key = base64Decode(key); +} + +SecretKey::SecretKey(const string & s) + : Key(s) +{ +#if HAVE_SODIUM + if (key.size() != crypto_sign_SECRETKEYBYTES) + throw Error("secret key is not valid"); +#endif +} + +#if !HAVE_SODIUM +[[noreturn]] static void noSodium() +{ + throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +} +#endif + +std::string SecretKey::signDetached(const std::string & data) const +{ +#if HAVE_SODIUM + unsigned char sig[crypto_sign_BYTES]; + unsigned long long sigLen; + crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), + (unsigned char *) key.data()); + return name + ":" + base64Encode(std::string((char *) sig, sigLen)); +#else + noSodium(); +#endif +} + +PublicKey SecretKey::toPublicKey() const +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); + return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(const string & s) + : Key(s) +{ +#if HAVE_SODIUM + if (key.size() != crypto_sign_PUBLICKEYBYTES) + throw Error("public key is not valid"); +#endif +} + +bool verifyDetached(const std::string & data, const std::string & sig, + const PublicKeys & publicKeys) +{ +#if HAVE_SODIUM + auto ss = split(sig); + + auto key = publicKeys.find(ss.first); + if (key == publicKeys.end()) return false; + + auto sig2 = base64Decode(ss.second); + if (sig2.size() != crypto_sign_BYTES) + throw Error("signature is not valid"); + + return crypto_sign_verify_detached((unsigned char *) sig2.data(), + (unsigned char *) data.data(), data.size(), + (unsigned char *) key->second.key.data()) == 0; +#else + noSodium(); +#endif +} + +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.get("binary-cache-public-keys", Strings())) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + +} diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh new file mode 100644 index 000000000000..9110af3aa9e5 --- /dev/null +++ b/src/libstore/crypto.hh @@ -0,0 +1,54 @@ +#pragma once + +#include "types.hh" + +#include <map> + +namespace nix { + +struct Key +{ + std::string name; + std::string key; + + /* Construct Key from a string in the format + ‘<name>:<key-in-base64>’. */ + Key(const std::string & s); + +protected: + Key(const std::string & name, const std::string & key) + : name(name), key(key) { } +}; + +struct PublicKey; + +struct SecretKey : Key +{ + SecretKey(const std::string & s); + + /* Return a detached signature of the given string. */ + std::string signDetached(const std::string & s) const; + + PublicKey toPublicKey() const; +}; + +struct PublicKey : Key +{ + PublicKey(const std::string & data); + +private: + PublicKey(const std::string & name, const std::string & key) + : Key(name, key) { } + friend struct SecretKey; +}; + +typedef std::map<std::string, PublicKey> PublicKeys; + +/* Return true iff ‘sig’ is a correct signature over ‘data’ using one + of the given public keys. */ +bool verifyDetached(const std::string & data, const std::string & sig, + const PublicKeys & publicKeys); + +PublicKeys getDefaultPublicKeys(); + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fbc1d99f3d6f..becf8524546c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -2,7 +2,7 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" -#include "misc.hh" +#include "worker-protocol.hh" namespace nix { @@ -26,21 +26,63 @@ void DerivationOutput::parseHashInfo(bool & recursive, HashType & hashType, Hash } -Path writeDerivation(StoreAPI & store, +Path BasicDerivation::findOutput(const string & id) const +{ + auto i = outputs.find(id); + if (i == outputs.end()) + throw Error(format("derivation has no output ‘%1%’") % id); + return i->second.path; +} + + +bool BasicDerivation::willBuildLocally() const +{ + return get(env, "preferLocalBuild") == "1" && canBuildLocally(); +} + + +bool BasicDerivation::substitutesAllowed() const +{ + return get(env, "allowSubstitutes", "1") == "1"; +} + + +bool BasicDerivation::isBuiltin() const +{ + return string(builder, 0, 8) == "builtin:"; +} + + +bool BasicDerivation::canBuildLocally() const +{ + return platform == settings.thisSystem + || isBuiltin() +#if __linux__ + || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux") + || (platform == "armv6l-linux" && settings.thisSystem == "armv7l-linux") +#elif __FreeBSD__ + || (platform == "i686-linux" && settings.thisSystem == "x86_64-freebsd") + || (platform == "i686-linux" && settings.thisSystem == "i686-freebsd") +#endif + ; +} + + +Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, bool repair) { PathSet references; references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) - references.insert(i->first); + for (auto & i : drv.inputDrvs) + references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ string suffix = name + drvExtension; - string contents = unparseDerivation(drv); + string contents = drv.unparse(); return settings.readOnlyMode ? computeStorePathForText(suffix, contents, references) - : store.addTextToStore(suffix, contents, references, repair); + : store->addTextToStore(suffix, contents, references, repair); } @@ -148,44 +190,44 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j) } -string unparseDerivation(const Derivation & drv) +string Derivation::unparse() const { string s; s.reserve(65536); s += "Derive(["; bool first = true; - foreach (DerivationOutputs::const_iterator, i, drv.outputs) { + for (auto & i : outputs) { if (first) first = false; else s += ','; - s += '('; printString(s, i->first); - s += ','; printString(s, i->second.path); - s += ','; printString(s, i->second.hashAlgo); - s += ','; printString(s, i->second.hash); + s += '('; printString(s, i.first); + s += ','; printString(s, i.second.path); + s += ','; printString(s, i.second.hashAlgo); + s += ','; printString(s, i.second.hash); s += ')'; } s += "],["; first = true; - foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { + for (auto & i : inputDrvs) { if (first) first = false; else s += ','; - s += '('; printString(s, i->first); - s += ','; printStrings(s, i->second.begin(), i->second.end()); + s += '('; printString(s, i.first); + s += ','; printStrings(s, i.second.begin(), i.second.end()); s += ')'; } s += "],"; - printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end()); + printStrings(s, inputSrcs.begin(), inputSrcs.end()); - s += ','; printString(s, drv.platform); - s += ','; printString(s, drv.builder); - s += ','; printStrings(s, drv.args.begin(), drv.args.end()); + s += ','; printString(s, platform); + s += ','; printString(s, builder); + s += ','; printStrings(s, args.begin(), args.end()); s += ",["; first = true; - foreach (StringPairs::const_iterator, i, drv.env) { + for (auto & i : env) { if (first) first = false; else s += ','; - s += '('; printString(s, i->first); - s += ','; printString(s, i->second); + s += '('; printString(s, i.first); + s += ','; printString(s, i.second); s += ')'; } @@ -201,11 +243,11 @@ bool isDerivation(const string & fileName) } -bool isFixedOutputDrv(const Derivation & drv) +bool BasicDerivation::isFixedOutput() const { - return drv.outputs.size() == 1 && - drv.outputs.begin()->first == "out" && - drv.outputs.begin()->second.hash != ""; + return outputs.size() == 1 && + outputs.begin()->first == "out" && + outputs.begin()->second.hash != ""; } @@ -232,10 +274,10 @@ DrvHashes drvHashes; paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -Hash hashDerivationModulo(StoreAPI & store, Derivation drv) +Hash hashDerivationModulo(Store & store, Derivation drv) { /* Return a fixed hash for fixed-output derivations. */ - if (isFixedOutputDrv(drv)) { + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" @@ -246,19 +288,19 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv) /* For other derivations, replace the inputs paths with recursive calls to this function.*/ DerivationInputs inputs2; - foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) { - Hash h = drvHashes[i->first]; - if (h.type == htUnknown) { - assert(store.isValidPath(i->first)); - Derivation drv2 = readDerivation(i->first); + for (auto & i : drv.inputDrvs) { + Hash h = drvHashes[i.first]; + if (!h) { + assert(store.isValidPath(i.first)); + Derivation drv2 = readDerivation(i.first); h = hashDerivationModulo(store, drv2); - drvHashes[i->first] = h; + drvHashes[i.first] = h; } - inputs2[printHash(h)] = i->second; + inputs2[printHash(h)] = i.second; } drv.inputDrvs = inputs2; - return hashString(htSHA256, unparseDerivation(drv)); + return hashString(htSHA256, drv.unparse()); } @@ -285,13 +327,53 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -PathSet outputPaths(const Derivation & drv) +PathSet BasicDerivation::outputPaths() const { PathSet paths; - for (auto & i : drv.outputs) + for (auto & i : outputs) paths.insert(i.second.path); return paths; } +Source & operator >> (Source & in, BasicDerivation & drv) +{ + drv.outputs.clear(); + auto nr = readInt(in); + for (unsigned int n = 0; n < nr; n++) { + auto name = readString(in); + DerivationOutput o; + in >> o.path >> o.hashAlgo >> o.hash; + assertStorePath(o.path); + drv.outputs[name] = o; + } + + drv.inputSrcs = readStorePaths<PathSet>(in); + in >> drv.platform >> drv.builder; + drv.args = readStrings<Strings>(in); + + nr = readInt(in); + for (unsigned int n = 0; n < nr; n++) { + auto key = readString(in); + auto value = readString(in); + drv.env[key] = value; + } + + return in; +} + + +Sink & operator << (Sink & out, const BasicDerivation & drv) +{ + out << drv.outputs.size(); + for (auto & i : drv.outputs) + out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; + out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << drv.env.size(); + for (auto & i : drv.env) + out << i.first << i.second; + return out; +} + + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8d5e4d05d469..6f98869b0fe0 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -40,44 +40,66 @@ typedef std::map<Path, StringSet> DerivationInputs; typedef std::map<string, string> StringPairs; -struct Derivation +struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ PathSet inputSrcs; /* inputs that are sources */ string platform; Path builder; Strings args; StringPairs env; + + virtual ~BasicDerivation() { }; + + /* Return the path corresponding to the output identifier `id' in + the given derivation. */ + Path findOutput(const string & id) const; + + bool willBuildLocally() const; + + bool substitutesAllowed() const; + + bool isBuiltin() const; + + bool canBuildLocally() const; + + /* Return true iff this is a fixed-output derivation. */ + bool isFixedOutput() const; + + /* Return the output paths of a derivation. */ + PathSet outputPaths() const; + }; +struct Derivation : BasicDerivation +{ + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + + /* Print a derivation. */ + std::string unparse() const; +}; -class StoreAPI; + +class Store; /* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(StoreAPI & store, +Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, bool repair = false); /* Read a derivation from a file. */ Derivation readDerivation(const Path & drvPath); -/* Print a derivation. */ -string unparseDerivation(const Derivation & drv); - -/* Check whether a file name ends with the extensions for +/* Check whether a file name ends with the extension for derivations. */ bool isDerivation(const string & fileName); -/* Return true iff this is a fixed-output derivation. */ -bool isFixedOutputDrv(const Derivation & drv); - -Hash hashDerivationModulo(StoreAPI & store, Derivation drv); +Hash hashDerivationModulo(Store & store, Derivation drv); /* Memoisation of hashDerivationModulo(). */ typedef std::map<Path, Hash> DrvHashes; -extern DrvHashes drvHashes; +extern DrvHashes drvHashes; // FIXME: global, not thread-safe /* Split a string specifying a derivation and a set of outputs (/nix/store/hash-foo!out1,out2,...) into the derivation path and @@ -89,6 +111,10 @@ Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outpu bool wantOutput(const string & output, const std::set<string> & wanted); -PathSet outputPaths(const Derivation & drv); +struct Source; +struct Sink; + +Source & operator >> (Source & in, BasicDerivation & drv); +Sink & operator << (Sink & out, const BasicDerivation & drv); } diff --git a/src/libexpr/download.cc b/src/libstore/download.cc index 9bf3e13aa9da..6e39330e40d9 100644 --- a/src/libexpr/download.cc +++ b/src/libstore/download.cc @@ -6,47 +6,72 @@ #include <curl/curl.h> +#include <iostream> + + namespace nix { -struct Curl +double getTime() +{ + struct timeval tv; + gettimeofday(&tv, 0); + return tv.tv_sec + (tv.tv_usec / 1000000.0); +} + +std::string resolveUri(const std::string & uri) +{ + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + else + return uri; +} + +struct CurlDownloader : public Downloader { CURL * curl; - string data; + ref<std::string> data; string etag, status, expectedETag; struct curl_slist * requestHeaders; - static size_t writeCallback(void * contents, size_t size, size_t nmemb, void * userp) + bool showProgress; + double prevProgressTime{0}, startTime{0}; + unsigned int moveBack{1}; + + size_t writeCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; - c.data.append((char *) contents, realSize); + data->append((char *) contents, realSize); return realSize; } - static size_t headerCallback(void * contents, size_t size, size_t nmemb, void * userp) + static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((CurlDownloader *) userp)->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void * contents, size_t size, size_t nmemb) { - Curl & c(* (Curl *) userp); size_t realSize = size * nmemb; string line = string((char *) contents, realSize); printMsg(lvlVomit, format("got header: %1%") % trim(line)); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts - c.etag = ""; + etag = ""; auto ss = tokenizeString<vector<string>>(line, " "); - c.status = ss.size() >= 2 ? ss[1] : ""; + status = ss.size() >= 2 ? ss[1] : ""; } else { auto i = line.find(':'); if (i != string::npos) { string name = trim(string(line, 0, i)); if (name == "ETag") { // FIXME: case - c.etag = trim(string(line, i + 1)); + etag = trim(string(line, i + 1)); /* Hack to work around a GitHub bug: it sends ETags, but ignores If-None-Match. So if we get the expected ETag on a 200 response, then shut down the connection because we already have the data. */ - printMsg(lvlDebug, format("got ETag: %1%") % c.etag); - if (c.etag == c.expectedETag && c.status == "200") { + printMsg(lvlDebug, format("got ETag: %1%") % etag); + if (etag == expectedETag && status == "200") { printMsg(lvlDebug, format("shutting down on 200 HTTP response with expected ETag")); return 0; } @@ -56,90 +81,151 @@ struct Curl return realSize; } - static int progressCallback(void * clientp, double dltotal, double dlnow, double ultotal, double ulnow) + static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { + return ((CurlDownloader *) userp)->headerCallback(contents, size, nmemb); + } + + int progressCallback(double dltotal, double dlnow) + { + if (showProgress) { + double now = getTime(); + if (prevProgressTime <= now - 1) { + string s = (format(" [%1$.0f/%2$.0f KiB, %3$.1f KiB/s]") + % (dlnow / 1024.0) + % (dltotal / 1024.0) + % (now == startTime ? 0 : dlnow / 1024.0 / (now - startTime))).str(); + std::cerr << "\e[" << moveBack << "D" << s; + moveBack = s.size(); + std::cerr.flush(); + prevProgressTime = now; + } + } return _isInterrupted; } - Curl() + static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) + { + return ((CurlDownloader *) userp)->progressCallback(dltotal, dlnow); + } + + CurlDownloader() + : data(make_ref<std::string>()) { requestHeaders = 0; curl = curl_easy_init(); - if (!curl) throw Error("unable to initialize curl"); + if (!curl) throw nix::Error("unable to initialize curl"); + } + + ~CurlDownloader() + { + if (curl) curl_easy_cleanup(curl); + if (requestHeaders) curl_slist_free_all(requestHeaders); + } + + bool fetch(const string & url, const DownloadOptions & options) + { + showProgress = + options.showProgress == DownloadOptions::yes || + (options.showProgress == DownloadOptions::automatic && isatty(STDERR_FILENO)); + + curl_easy_reset(curl); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, ("Nix/" + nixVersion).c_str()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) &curl); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, headerCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *) this); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *) this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); - } - ~Curl() - { - if (curl) curl_easy_cleanup(curl); - if (requestHeaders) curl_slist_free_all(requestHeaders); - } + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - bool fetch(const string & url, const string & expectedETag = "") - { curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - data.clear(); + if (options.verifyTLS) + curl_easy_setopt(curl, CURLOPT_CAINFO, getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt").c_str()); + else { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + } + + data->clear(); if (requestHeaders) { curl_slist_free_all(requestHeaders); requestHeaders = 0; } - if (!expectedETag.empty()) { - this->expectedETag = expectedETag; - requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + expectedETag).c_str()); + if (!options.expectedETag.empty()) { + this->expectedETag = options.expectedETag; + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + options.expectedETag).c_str()); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders); + if (options.head) + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + + if (showProgress) { + std::cerr << (format("downloading ‘%1%’... ") % url); + std::cerr.flush(); + startTime = getTime(); + } + CURLcode res = curl_easy_perform(curl); + if (showProgress) + //std::cerr << "\e[" << moveBack << "D\e[K\n"; + std::cerr << "\n"; checkInterrupt(); - if (res == CURLE_WRITE_ERROR && etag == expectedETag) return false; - if (res != CURLE_OK) - throw DownloadError(format("unable to download ‘%1%’: %2% (%3%)") - % url % curl_easy_strerror(res) % res); + if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; - long httpStatus = 0; + long httpStatus = -1; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + + if (res != CURLE_OK) { + Error err = + httpStatus == 404 ? NotFound : + httpStatus == 403 ? Forbidden : Misc; + throw DownloadError(err, format("unable to download ‘%1%’: %2% (%3%)") + % url % curl_easy_strerror(res) % res); + } + if (httpStatus == 304) return false; return true; } -}; + DownloadResult download(string url, const DownloadOptions & options) override + { + DownloadResult res; + if (fetch(resolveUri(url), options)) { + res.cached = false; + res.data = data; + } else + res.cached = true; + res.etag = etag; + return res; + } +}; -DownloadResult downloadFile(string url, string expectedETag) +ref<Downloader> makeDownloader() { - DownloadResult res; - Curl curl; - if (curl.fetch(url, expectedETag)) { - res.cached = false; - res.data = curl.data; - } else - res.cached = true; - res.etag = curl.etag; - return res; + return make_ref<CurlDownloader>(); } - -Path downloadFileCached(const string & url, bool unpack) +Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack) { - Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; + auto url = resolveUri(url_); + + Path cacheDir = getCacheDir() + "/nix/tarballs"; createDirs(cacheDir); string urlHash = printHash32(hashString(htSHA256, url)); @@ -178,21 +264,18 @@ Path downloadFileCached(const string & url, bool unpack) if (!skip) { - if (storePath.empty()) - printMsg(lvlInfo, format("downloading ‘%1%’...") % url); - else - printMsg(lvlInfo, format("checking ‘%1%’...") % url); - try { - auto res = downloadFile(url, expectedETag); + DownloadOptions options; + options.expectedETag = expectedETag; + auto res = download(url, options); if (!res.cached) - storePath = store->addTextToStore(name, res.data, PathSet(), false); + storePath = store->addTextToStore(name, *res.data, PathSet(), false); assert(!storePath.empty()); replaceSymlink(storePath, fileLink); - writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n"); + writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; printMsg(lvlError, format("warning: %1%; using cached result") % e.msg()); @@ -226,10 +309,11 @@ Path downloadFileCached(const string & url, bool unpack) bool isUri(const string & s) { + if (s.compare(0, 8, "channel:") == 0) return true; size_t pos = s.find("://"); if (pos == string::npos) return false; string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file"; + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git"; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh new file mode 100644 index 000000000000..eb2b76678ac7 --- /dev/null +++ b/src/libstore/download.hh @@ -0,0 +1,48 @@ +#pragma once + +#include "types.hh" + +#include <string> + +namespace nix { + +struct DownloadOptions +{ + string expectedETag; + bool verifyTLS{true}; + enum { yes, no, automatic } showProgress{yes}; + bool head{false}; +}; + +struct DownloadResult +{ + bool cached; + string etag; + std::shared_ptr<std::string> data; +}; + +class Store; + +struct Downloader +{ + virtual DownloadResult download(string url, const DownloadOptions & options) = 0; + + Path downloadCached(ref<Store> store, const string & url, bool unpack); + + enum Error { NotFound, Forbidden, Misc }; +}; + +ref<Downloader> makeDownloader(); + +class DownloadError : public Error +{ +public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString & fs) + : Error(fs), error(error) + { } +}; + +bool isUri(const string & s); + +} diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc new file mode 100644 index 000000000000..4ec01add3026 --- /dev/null +++ b/src/libstore/export-import.cc @@ -0,0 +1,136 @@ +#include "store-api.hh" +#include "archive.hh" +#include "worker-protocol.hh" + +#include <algorithm> + +namespace nix { + +struct HashAndWriteSink : Sink +{ + Sink & writeSink; + HashSink hashSink; + HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) + { + } + virtual void operator () (const unsigned char * data, size_t len) + { + writeSink(data, len); + hashSink(data, len); + } + Hash currentHash() + { + return hashSink.currentHash().first; + } +}; + +void Store::exportPaths(const Paths & paths, Sink & sink) +{ + Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + std::reverse(sorted.begin(), sorted.end()); + + std::string doneLabel("paths exported"); + logger->incExpected(doneLabel, sorted.size()); + + for (auto & path : sorted) { + Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path); + sink << 1; + exportPath(path, sink); + logger->incProgress(doneLabel); + } + + sink << 0; +} + +void Store::exportPath(const Path & path, Sink & sink) +{ + auto info = queryPathInfo(path); + + HashAndWriteSink hashAndWriteSink(sink); + + narFromPath(path, hashAndWriteSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashAndWriteSink.currentHash(); + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path + % printHash(info->narHash) % printHash(hash)); + + hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; +} + +struct TeeSource : Source +{ + Source & readSource; + ref<std::string> data; + TeeSource(Source & readSource) + : readSource(readSource) + , data(make_ref<std::string>()) + { + } + size_t read(unsigned char * data, size_t len) + { + size_t n = readSource.read(data, len); + this->data->append((char *) data, n); + return n; + } +}; + +struct NopSink : ParseSink +{ +}; + +Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); + + /* Extract the NAR from the source. */ + TeeSource tee(source); + NopSink sink; + parseDump(sink, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + + info.path = readStorePath(source); + + Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path); + + info.references = readStorePaths<PathSet>(source); + + info.deriver = readString(source); + if (info.deriver != "") assertStorePath(info.deriver); + + info.narHash = hashString(htSHA256, *tee.data); + info.narSize = tee.data->size(); + + // Ignore optional legacy signature. + if (readInt(source) == 1) + readString(source); + + addToStore(info, *tee.data); + + // FIXME: implement accessors? + assert(!accessor); +#if 0 + auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); + if (accessor_) + accessor_->nars.emplace(info.path, makeNarAccessor(tee.data)); +#endif + + res.push_back(info.path); + } + + return res; +} + +} diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh new file mode 100644 index 000000000000..a67e0775b978 --- /dev/null +++ b/src/libstore/fs-accessor.hh @@ -0,0 +1,30 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +/* An abstract class for accessing a filesystem-like structure, such + as a (possibly remote) Nix store or the contents of a NAR file. */ +class FSAccessor +{ +public: + enum Type { tMissing, tRegular, tSymlink, tDirectory }; + + struct Stat + { + Type type; + uint64_t fileSize; // regular files only + bool isExecutable; // regular files only + }; + + virtual Stat stat(const Path & path) = 0; + + virtual StringSet readDirectory(const Path & path) = 0; + + virtual std::string readFile(const Path & path) = 0; + + virtual std::string readLink(const Path & path) = 0; +}; + +} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8d7da67f5204..f4cb672cdeb9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,5 +1,5 @@ +#include "derivations.hh" #include "globals.hh" -#include "misc.hh" #include "local-store.hh" #include <functional> @@ -83,7 +83,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path addPermRoot(StoreAPI & store, const Path & _storePath, +Path Store::addPermRoot(const Path & _storePath, const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) { Path storePath(canonPath(_storePath)); @@ -101,7 +101,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) throw Error(format("cannot create symlink ‘%1%’; already exists") % gcRoot); makeSymlink(gcRoot, storePath); - store.addIndirectRoot(gcRoot); + addIndirectRoot(gcRoot); } else { @@ -127,7 +127,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = store.findRoots(); + Roots roots = findRoots(); if (roots.find(gcRoot) == roots.end()) printMsg(lvlError, format( @@ -139,7 +139,7 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, /* Grab the global GC root, causing us to block while a GC is in progress. This prevents the set of permanent roots from increasing while a GC is in progress. */ - store.syncWithGC(); + syncWithGC(); return gcRoot; } @@ -147,35 +147,36 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, void LocalStore::addTempRoot(const Path & path) { + auto state(_state.lock()); + /* Create the temporary roots file for this process. */ - if (fdTempRoots == -1) { + if (state->fdTempRoots == -1) { while (1) { Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); createDirs(dir); - fnTempRoots = (format("%1%/%2%") - % dir % getpid()).str(); + state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str(); AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) + if (pathExists(state->fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ - unlink(fnTempRoots.c_str()); + unlink(state->fnTempRoots.c_str()); - fdTempRoots = openLockFile(fnTempRoots, true); + state->fdTempRoots = openLockFile(state->fnTempRoots, true); fdGCLock.close(); - debug(format("acquiring read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); /* Check whether the garbage collector didn't get in our way. */ struct stat st; - if (fstat(fdTempRoots, &st) == -1) - throw SysError(format("statting ‘%1%’") % fnTempRoots); + if (fstat(state->fdTempRoots, &st) == -1) + throw SysError(format("statting ‘%1%’") % state->fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -187,15 +188,15 @@ void LocalStore::addTempRoot(const Path & path) /* Upgrade the lock to a write lock. This will cause us to block if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltWrite, true); + debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltWrite, true); string s = path + '\0'; - writeFull(fdTempRoots, s); + writeFull(state->fdTempRoots, s); /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); } @@ -260,19 +261,16 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds) } -static void foundRoot(StoreAPI & store, - const Path & path, const Path & target, Roots & roots) +void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { - Path storePath = toStorePath(target); - if (store.isValidPath(storePath)) - roots[path] = storePath; - else - printMsg(lvlInfo, format("skipping invalid root from ‘%1%’ to ‘%2%’") % path % storePath); -} - + auto foundRoot = [&](const Path & path, const Path & target) { + Path storePath = toStorePath(target); + if (isStorePath(storePath) && isValidPath(storePath)) + roots[path] = storePath; + else + printMsg(lvlInfo, format("skipping invalid root from ‘%1%’ to ‘%2%’") % path % storePath); + }; -static void findRoots(StoreAPI & store, const Path & path, unsigned char type, Roots & roots) -{ try { if (type == DT_UNKNOWN) @@ -280,13 +278,13 @@ static void findRoots(StoreAPI & store, const Path & path, unsigned char type, R if (type == DT_DIR) { for (auto & i : readDirectory(path)) - findRoots(store, path + "/" + i.name, i.type, roots); + findRoots(path + "/" + i.name, i.type, roots); } else if (type == DT_LNK) { Path target = readLink(path); if (isInStore(target)) - foundRoot(store, path, target, roots); + foundRoot(path, target); /* Handle indirect roots. */ else { @@ -300,14 +298,14 @@ static void findRoots(StoreAPI & store, const Path & path, unsigned char type, R struct stat st2 = lstat(target); if (!S_ISLNK(st2.st_mode)) return; Path target2 = readLink(target); - if (isInStore(target2)) foundRoot(store, target, target2, roots); + if (isInStore(target2)) foundRoot(target, target2); } } } else if (type == DT_REG) { Path storePath = settings.nixStore + "/" + baseNameOf(path); - if (store.isValidPath(storePath)) + if (isStorePath(storePath) && isValidPath(storePath)) roots[path] = storePath; } @@ -328,16 +326,16 @@ Roots LocalStore::findRoots() Roots roots; /* Process direct roots in {gcroots,manifests,profiles}. */ - nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); + findRoots(settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); if (pathExists(settings.nixStateDir + "/manifests")) - nix::findRoots(*this, settings.nixStateDir + "/manifests", DT_UNKNOWN, roots); - nix::findRoots(*this, settings.nixStateDir + "/profiles", DT_UNKNOWN, roots); + findRoots(settings.nixStateDir + "/manifests", DT_UNKNOWN, roots); + findRoots(settings.nixStateDir + "/profiles", DT_UNKNOWN, roots); return roots; } -static void addAdditionalRoots(StoreAPI & store, PathSet & roots) +void LocalStore::findRuntimeRoots(PathSet & roots) { Path rootFinder = getEnv("NIX_ROOT_FINDER", settings.nixLibexecDir + "/nix/find-runtime-roots.pl"); @@ -350,15 +348,14 @@ static void addAdditionalRoots(StoreAPI & store, PathSet & roots) StringSet paths = tokenizeString<StringSet>(result, "\n"); - foreach (StringSet::iterator, i, paths) { - if (isInStore(*i)) { - Path path = toStorePath(*i); - if (roots.find(path) == roots.end() && store.isValidPath(path)) { + for (auto & i : paths) + if (isInStore(i)) { + Path path = toStorePath(i); + if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { debug(format("got additional root ‘%1%’") % path); roots.insert(path); } } - } } @@ -405,12 +402,12 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) unsigned long long size = 0; - if (isValidPath(path)) { + if (isStorePath(path) && isValidPath(path)) { PathSet referrers; queryReferrers(path, referrers); - foreach (PathSet::iterator, i, referrers) - if (*i != path) deletePathRecursive(state, *i); - size = queryPathInfo(path).narSize; + for (auto & i : referrers) + if (i != path) deletePathRecursive(state, i); + size = queryPathInfo(path)->narSize; invalidatePathChecked(path); } @@ -476,7 +473,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p visited.insert(path); - if (!isValidPath(path)) return false; + if (!isStorePath(path) || !isValidPath(path)) return false; PathSet incoming; @@ -487,22 +484,22 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p don't delete the derivation if any of the outputs are alive. */ if (state.gcKeepDerivations && isDerivation(path)) { PathSet outputs = queryDerivationOutputs(path); - foreach (PathSet::iterator, i, outputs) - if (isValidPath(*i) && queryDeriver(*i) == path) - incoming.insert(*i); + for (auto & i : outputs) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + incoming.insert(i); } /* If gc-keep-outputs is set, then don't delete this path if there are derivers of this path that are not garbage. */ if (state.gcKeepOutputs) { PathSet derivers = queryValidDerivers(path); - foreach (PathSet::iterator, i, derivers) - incoming.insert(*i); + for (auto & i : derivers) + incoming.insert(i); } - foreach (PathSet::iterator, i, incoming) - if (*i != path) - if (canReachRoot(state, visited, *i)) { + for (auto & i : incoming) + if (i != path) + if (canReachRoot(state, visited, i)) { state.alive.insert(path); return true; } @@ -517,9 +514,9 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) if (path == linksDir || path == state.trashDir) return; - startNest(nest, lvlDebug, format("considering whether to delete ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); - if (!isValidPath(path)) { + if (!isStorePath(path) || !isValidPath(path)) { /* A lock file belonging to a path that we're building right now isn't garbage. */ if (isActiveTempFile(state, path, ".lock")) return; @@ -612,6 +609,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + if (state.shouldDelete) + deletePath(reservedPath); + /* Acquire the global GC root. This prevents a) New roots from being added. b) Processes from creating new temporary root files. */ @@ -622,14 +622,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printMsg(lvlError, format("finding garbage collector roots...")); Roots rootMap = options.ignoreLiveness ? Roots() : findRoots(); - foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second); + for (auto & i : rootMap) state.roots.insert(i.second); /* Add additional roots returned by the program specified by the NIX_ROOT_FINDER environment variable. This is typically used to add running programs to the set of roots (to prevent them from being garbage collected). */ if (!options.ignoreLiveness) - addAdditionalRoots(*this, state.roots); + findRuntimeRoots(state.roots); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -659,11 +659,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteSpecific) { - foreach (PathSet::iterator, i, options.pathsToDelete) { - assertStorePath(*i); - tryToDelete(state, *i); - if (state.dead.find(*i) == state.dead.end()) - throw Error(format("cannot delete path ‘%1%’ since it is still alive") % *i); + for (auto & i : options.pathsToDelete) { + assertStorePath(i); + tryToDelete(state, i); + if (state.dead.find(i) == state.dead.end()) + throw Error(format("cannot delete path ‘%1%’ since it is still alive") % i); } } else if (options.maxFreed > 0) { @@ -691,7 +691,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) string name = dirent->d_name; if (name == "." || name == "..") continue; Path path = settings.nixStore + "/" + name; - if (isValidPath(path)) + if (isStorePath(path) && isValidPath(path)) entries.push_back(path); else tryToDelete(state, path); @@ -707,8 +707,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) vector<Path> entries_(entries.begin(), entries.end()); random_shuffle(entries_.begin(), entries_.end()); - foreach (vector<Path>::iterator, i, entries_) - tryToDelete(state, *i); + for (auto & i : entries_) + tryToDelete(state, i); } catch (GCLimitReached & e) { } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 50374f782dee..c12178e4028a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -28,7 +28,6 @@ Settings::Settings() keepFailed = false; keepGoing = false; tryFallback = false; - buildVerbosity = lvlError; maxBuildJobs = 1; buildCores = 1; #ifdef _SC_NPROCESSORS_ONLN @@ -40,7 +39,6 @@ Settings::Settings() maxSilentTime = 0; buildTimeout = 0; useBuildHook = true; - printBuildTrace = false; reservedSize = 8 * 1024 * 1024; fsyncMetadata = true; useSQLiteWAL = true; @@ -52,7 +50,6 @@ Settings::Settings() keepLog = true; compressLog = true; maxLogSize = 0; - cacheFailure = false; pollInterval = 5; checkRootReachability = false; gcKeepOutputs = false; @@ -77,6 +74,11 @@ void Settings::processEnvironment() nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); + + // should be set with the other config options, but depends on nixLibexecDir +#ifdef __APPLE__ + preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies.pl"; +#endif } @@ -170,7 +172,6 @@ void Settings::update() _get(keepLog, "build-keep-log"); _get(compressLog, "build-compress-log"); _get(maxLogSize, "build-max-log-size"); - _get(cacheFailure, "build-cache-failure"); _get(pollInterval, "build-poll-interval"); _get(checkRootReachability, "gc-check-reachability"); _get(gcKeepOutputs, "gc-keep-outputs"); @@ -183,20 +184,6 @@ void Settings::update() _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); _get(useCaseHack, "use-case-hack"); _get(preBuildHook, "pre-build-hook"); - - string subs = getEnv("NIX_SUBSTITUTERS", "default"); - if (subs == "default") { - substituters.clear(); -#if 0 - if (getEnv("NIX_OTHER_STORES") != "") - substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); -#endif - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); - substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); - if (useSshSubstituter && !sshSubstituterHosts.empty()) - substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); - } else - substituters = tokenizeString<Strings>(subs, ":"); } @@ -248,12 +235,12 @@ template<class N> void Settings::_get(N & res, const string & name) string Settings::pack() { string s; - foreach (SettingsMap::iterator, i, settings) { - if (i->first.find('\n') != string::npos || - i->first.find('=') != string::npos || - i->second.find('\n') != string::npos) + for (auto & i : settings) { + if (i.first.find('\n') != string::npos || + i.first.find('=') != string::npos || + i.second.find('\n') != string::npos) throw Error("illegal option name/value"); - s += i->first; s += '='; s += i->second; s += '\n'; + s += i.first; s += '='; s += i.second; s += '\n'; } return s; } @@ -261,11 +248,11 @@ string Settings::pack() void Settings::unpack(const string & pack) { Strings lines = tokenizeString<Strings>(pack, "\n"); - foreach (Strings::iterator, i, lines) { - string::size_type eq = i->find('='); + for (auto & i : lines) { + string::size_type eq = i.find('='); if (eq == string::npos) throw Error("illegal option name/value"); - set(i->substr(0, eq), i->substr(eq + 1)); + set(i.substr(0, eq), i.substr(eq + 1)); } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 60b11afe6088..65f763ace3c7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <map> #include <sys/types.h> @@ -77,8 +78,12 @@ struct Settings { instead. */ bool tryFallback; - /* Verbosity level for build output. */ - Verbosity buildVerbosity; + /* Whether to show build log output in real time. */ + bool verboseBuild = true; + + /* If verboseBuild is false, the number of lines of the tail of + the log to show if a build fails. */ + size_t logLines = 10; /* Maximum number of parallel build jobs. 0 means unlimited. */ unsigned int maxBuildJobs; @@ -105,31 +110,10 @@ struct Settings { means infinity. */ time_t buildTimeout; - /* The substituters. There are programs that can somehow realise - a store path without building, e.g., by downloading it or - copying it from a CD. */ - Paths substituters; - /* Whether to use build hooks (for distributed builds). Sometimes users want to disable this from the command-line. */ bool useBuildHook; - /* Whether buildDerivations() should print out lines on stderr in - a fixed format to allow its progress to be monitored. Each - line starts with a "@". The following are defined: - - @ build-started <drvpath> <outpath> <system> <logfile> - @ build-failed <drvpath> <outpath> <exitcode> <error text> - @ build-succeeded <drvpath> <outpath> - @ substituter-started <outpath> <substituter> - @ substituter-failed <outpath> <exitcode> <error text> - @ substituter-succeeded <outpath> - - Best combined with --no-build-output, otherwise stderr might - conceivably contain lines in this format printed by the - builders. */ - bool printBuildTrace; - /* Amount of reserved space for the garbage collector (/nix/var/nix/db/reserved). */ off_t reservedSize; @@ -168,9 +152,6 @@ struct Settings { before being killed (0 means no limit). */ unsigned long maxLogSize; - /* Whether to cache build failures. */ - bool cacheFailure; - /* How often (in seconds) to poll for locks. */ unsigned int pollInterval; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000000..8c8d545c6d85 --- /dev/null +++ b/src/libstore/http-binary-cache-store.cc @@ -0,0 +1,105 @@ +#include "binary-cache-store.hh" +#include "download.hh" +#include "globals.hh" +#include "nar-info-disk-cache.hh" + +namespace nix { + +MakeError(UploadToHTTP, Error); + +class HttpBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path cacheUri; + + Pool<Downloader> downloaders; + +public: + + HttpBinaryCacheStore( + const StoreParams & params, const Path & _cacheUri) + : BinaryCacheStore(params) + , cacheUri(_cacheUri) + , downloaders( + std::numeric_limits<size_t>::max(), + []() { return makeDownloader(); }) + { + if (cacheUri.back() == '/') + cacheUri.pop_back(); + + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override + { + return cacheUri; + } + + void init() override + { + // FIXME: do this lazily? + if (!diskCache->cacheExists(cacheUri)) { + try { + BinaryCacheStore::init(); + } catch (UploadToHTTP &) { + throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + } + diskCache->createCache(cacheUri, wantMassQuery_, priority); + } + } + +protected: + + bool fileExists(const std::string & path) override + { + try { + auto downloader(downloaders.get()); + DownloadOptions options; + options.showProgress = DownloadOptions::no; + options.head = true; + downloader->download(cacheUri + "/" + path, options); + return true; + } catch (DownloadError & e) { + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return false; + throw; + } + } + + void upsertFile(const std::string & path, const std::string & data) override + { + throw UploadToHTTP("uploading to an HTTP binary cache is not supported"); + } + + std::shared_ptr<std::string> getFile(const std::string & path) override + { + auto downloader(downloaders.get()); + DownloadOptions options; + options.showProgress = DownloadOptions::no; + try { + return downloader->download(cacheUri + "/" + path, options).data; + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return 0; + throw; + } + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://") return 0; + auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); + store->init(); + return store; +}); + +} + diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000000..b418c9c04148 --- /dev/null +++ b/src/libstore/local-binary-cache-store.cc @@ -0,0 +1,105 @@ +#include "binary-cache-store.hh" +#include "globals.hh" +#include "nar-info-disk-cache.hh" + +namespace nix { + +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore( + const StoreParams & params, const Path & binaryCacheDir) + : BinaryCacheStore(params) + , binaryCacheDir(binaryCacheDir) + { + /* For testing the NAR info cache. */ + if (getEnv("_NIX_CACHE_FILE_URLS") == "1") + diskCache = getNarInfoDiskCache(); + } + + void init() override; + + std::string getUri() override + { + return "file://" + binaryCacheDir; + } + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, const std::string & data) override; + + std::shared_ptr<std::string> getFile(const std::string & path) override; + + PathSet queryAllValidPaths() override + { + PathSet paths; + + for (auto & entry : readDirectory(binaryCacheDir)) { + if (entry.name.size() != 40 || + !hasSuffix(entry.name, ".narinfo")) + continue; + paths.insert(settings.nixStore + "/" + entry.name.substr(0, entry.name.size() - 8)); + } + + return paths; + } + +}; + +void LocalBinaryCacheStore::init() +{ + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); + + if (diskCache && !diskCache->cacheExists(getUri())) + diskCache->createCache(getUri(), wantMassQuery_, priority); +} + +static void atomicWrite(const Path & path, const std::string & s) +{ + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path); + del.cancel(); +} + +bool LocalBinaryCacheStore::fileExists(const std::string & path) +{ + return pathExists(binaryCacheDir + "/" + path); +} + +void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data) +{ + atomicWrite(binaryCacheDir + "/" + path, data); +} + +std::shared_ptr<std::string> LocalBinaryCacheStore::getFile(const std::string & path) +{ + try { + return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)); + } catch (SysError & e) { + if (e.errNo == ENOENT) return 0; + throw; + } +} + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, 7) != "file://") return 0; + auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); + store->init(); + return store; +}); + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc new file mode 100644 index 000000000000..303c3af27b8d --- /dev/null +++ b/src/libstore/local-fs-store.cc @@ -0,0 +1,79 @@ +#include "archive.hh" +#include "fs-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct LocalStoreAccessor : public FSAccessor +{ + ref<Store> store; + + LocalStoreAccessor(ref<Store> store) : store(store) { } + + void assertStore(const Path & path) + { + Path storePath = toStorePath(path); + if (!store->isValidPath(storePath)) + throw Error(format("path ‘%1%’ is not a valid store path") % storePath); + } + + FSAccessor::Stat stat(const Path & path) override + { + assertStore(path); + + struct stat st; + if (lstat(path.c_str(), &st)) { + if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; + throw SysError(format("getting status of ‘%1%’") % path); + } + + if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + throw Error(format("file ‘%1%’ has unsupported type") % path); + + return { + S_ISREG(st.st_mode) ? Type::tRegular : + S_ISLNK(st.st_mode) ? Type::tSymlink : + Type::tDirectory, + S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + } + + StringSet readDirectory(const Path & path) override + { + assertStore(path); + + auto entries = nix::readDirectory(path); + + StringSet res; + for (auto & entry : entries) + res.insert(entry.name); + + return res; + } + + std::string readFile(const Path & path) override + { + assertStore(path); + return nix::readFile(path); + } + + std::string readLink(const Path & path) override + { + assertStore(path); + return nix::readLink(path); + } +}; + +ref<FSAccessor> LocalFSStore::getFSAccessor() +{ + return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this())); +} + +void LocalFSStore::narFromPath(const Path & path, Sink & sink) +{ + if (!isValidPath(path)) + throw Error(format("path ‘%s’ is not valid") % path); + dumpPath(path, sink); +} + +} diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 074c3394ffb1..b44384957ca6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,7 +5,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" -#include "affinity.hh" +#include "nar-info.hh" #include <iostream> #include <algorithm> @@ -23,16 +23,11 @@ #include <time.h> #include <grp.h> -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H +#if __linux__ #include <sched.h> #include <sys/statvfs.h> #include <sys/mount.h> -#endif - -#if HAVE_LINUX_FS_H -#include <linux/fs.h> #include <sys/ioctl.h> -#include <errno.h> #endif #include <sqlite3.h> @@ -41,171 +36,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -static void throwSQLiteError(sqlite3 * db, const format & f) - __attribute__ ((noreturn)); - -static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -224,20 +54,24 @@ void checkStoreNotSymlink() } -LocalStore::LocalStore(bool reserveSpace) - : didSetSubstituterEnv(false) +LocalStore::LocalStore() + : linksDir(settings.nixStore + "/.links") + , reservedPath(settings.nixDBPath + "/reserved") + , schemaPath(settings.nixDBPath + "/schema") + , requireSigs(settings.get("signed-binary-caches", std::string("")) != "") // FIXME: rename option + , publicKeys(getDefaultPublicKeys()) { - schemaPath = settings.nixDBPath + "/schema"; + auto state(_state.lock()); if (settings.readOnlyMode) { - openDB(false); + openDB(*state, false); return; } /* Create missing state directories if they don't already exist. */ createDirs(settings.nixStore); makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); + createDirs(linksDir); Path profilesDir = settings.nixStateDir + "/profiles"; createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); @@ -284,25 +118,20 @@ LocalStore::LocalStore(bool reserveSpace) needed, we reserve some dummy space that we can free just before doing a garbage collection. */ try { - Path reservedPath = settings.nixDBPath + "/reserved"; - if (reserveSpace) { - struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || - st.st_size != settings.reservedSize) - { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600); - int res = -1; + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) + { + AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600); + int res = -1; #if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd, 0, settings.reservedSize); + res = posix_fallocate(fd, 0, settings.reservedSize); #endif - if (res == -1) { - writeFull(fd, string(settings.reservedSize, 'X')); - ftruncate(fd, settings.reservedSize); - } + if (res == -1) { + writeFull(fd, string(settings.reservedSize, 'X')); + ftruncate(fd, settings.reservedSize); } } - else - deletePath(reservedPath); } catch (SysError & e) { /* don't care about errors */ } @@ -314,7 +143,7 @@ LocalStore::LocalStore(bool reserveSpace) } catch (SysError & e) { if (e.errNo != EACCES) throw; settings.readOnlyMode = true; - openDB(false); + openDB(*state, false); return; } @@ -332,7 +161,7 @@ LocalStore::LocalStore(bool reserveSpace) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - openDB(true); + openDB(*state, true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } @@ -343,6 +172,12 @@ LocalStore::LocalStore(bool reserveSpace) "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); + if (curSchema < 6) + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + if (!lockFile(globalLock, ltWrite, false)) { printMsg(lvlError, "waiting for exclusive access to the Nix store..."); lockFile(globalLock, ltWrite, true); @@ -352,37 +187,43 @@ LocalStore::LocalStore(bool reserveSpace) have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 6) upgradeStore6(); - else if (curSchema < 7) { upgradeStore7(); openDB(true); } + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); } - else openDB(false); + else openDB(*state, false); } LocalStore::~LocalStore() { - try { - foreach (RunningSubstituters::iterator, i, runningSubstituters) { - if (i->second.disabled) continue; - i->second.to.close(); - i->second.from.close(); - i->second.error.close(); - if (i->second.pid != -1) - i->second.pid.wait(true); - } - } catch (...) { - ignoreException(); - } + auto state(_state.lock()); try { - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); + if (state->fdTempRoots != -1) { + state->fdTempRoots.close(); + unlink(state->fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -390,6 +231,12 @@ LocalStore::~LocalStore() } +std::string LocalStore::getUri() +{ + return "local"; +} + + int LocalStore::getSchema() { int curSchema = 0; @@ -402,13 +249,20 @@ int LocalStore::getSchema() } -void LocalStore::openDB(bool create) +bool LocalStore::haveWriteAccess() +{ + return access(settings.nixDBPath.c_str(), R_OK | W_OK) == 0; +} + + +void LocalStore::openDB(State & state, bool create) { - if (access(settings.nixDBPath.c_str(), R_OK | W_OK)) + if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ string dbPath = settings.nixDBPath + "/db.sqlite"; + auto & db(state.db); if (sqlite3_open_v2(dbPath.c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); @@ -461,40 +315,31 @@ void LocalStore::openDB(bool create) } /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, + state.stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);"); + state.stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;"); + state.stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, + state.stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;"); + state.stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, + state.stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, + state.stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, + state.stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, + state.stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, + state.stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, + state.stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + state.stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -502,7 +347,7 @@ void LocalStore::openDB(bool create) bind mount. So make the Nix store writable for this process. */ void LocalStore::makeStoreWritable() { -#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT) +#if __linux__ if (getuid() != 0) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; @@ -607,10 +452,10 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe users group); we check for this case below. */ if (st.st_uid != geteuid()) { #if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1) + if (lchown(path.c_str(), geteuid(), getegid()) == -1) #else if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), (gid_t) -1) == -1) + chown(path.c_str(), geteuid(), getegid()) == -1) #endif throw SysError(format("changing owner of ‘%1%’ to %2%") % path % geteuid()); @@ -654,7 +499,7 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & assert(isDerivation(drvName)); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); - if (isFixedOutputDrv(drv)) { + if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error(format("derivation ‘%1%’ does not have an output named ‘out’") % drvPath); @@ -671,41 +516,37 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & else { Derivation drvCopy(drv); - foreach (DerivationOutputs::iterator, i, drvCopy.outputs) { - i->second.path = ""; - drvCopy.env[i->first] = ""; + for (auto & i : drvCopy.outputs) { + i.second.path = ""; + drvCopy.env[i.first] = ""; } Hash h = hashDerivationModulo(*this, drvCopy); - foreach (DerivationOutputs::const_iterator, i, drv.outputs) { - Path outPath = makeOutputPath(i->first, h, drvName); - StringPairs::const_iterator j = drv.env.find(i->first); - if (i->second.path != outPath || j == drv.env.end() || j->second != outPath) + for (auto & i : drv.outputs) { + Path outPath = makeOutputPath(i.first, h, drvName); + StringPairs::const_iterator j = drv.env.find(i.first); + if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) throw Error(format("derivation ‘%1%’ has incorrect output ‘%2%’, should be ‘%3%’") - % drvPath % i->second.path % outPath); + % drvPath % i.second.path % outPath); } } } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path ‘%1%’ in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + state.stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.narHash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -721,90 +562,21 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i->first); - stmtAddDerivationOutput.bind(i->second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for ‘%1%’ in database") % info.path); + for (auto & i : drv.outputs) { + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } - return id; -} - - -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path ‘%1%’") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - foreach (PathSet::const_iterator, i, paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(*i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path ‘%1%’ in database") % *i); - } + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info)); + } - txn.commit(); - } end_retry_sqlite; + return id; } @@ -822,174 +594,124 @@ Hash parseHashField(const Path & path, const string & s) } -ValidPathInfo LocalStore::queryPathInfo(const Path & path) +std::shared_ptr<ValidPathInfo> LocalStore::queryPathInfoUncached(const Path & path) { - ValidPathInfo info; - info.path = path; + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; assertStorePath(path); - retry_sqlite { + return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + auto state(_state.lock()); /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); - - stmtQueryPathInfo.bind(path); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + if (!useQueryPathInfo.next()) + return std::shared_ptr<ValidPathInfo>(); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info->id = useQueryPathInfo.getInt(0); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.hash = parseHashField(path, s); + info->narHash = parseHashField(path, useQueryPathInfo.getStr(1)); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info->registrationTime = useQueryPathInfo.getInt(2); - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); - if (s) info.deriver = s; + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + if (s) info->deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info->narSize = useQueryPathInfo.getInt(4); - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); + info->ultimate = useQueryPathInfo.getInt(5) == 1; - stmtQueryReferences.bind(info.id); + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info->sigs = tokenizeString<StringSet>(s, " "); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferences.next()) + info->references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.narHash)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } -bool LocalStore::isValidPath_(const Path & path) +bool LocalStore::isValidPath_(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return state.stmtQueryPathInfo.use()(path).next(); } -bool LocalStore::isValidPath(const Path & path) +bool LocalStore::isValidPathUncached(const Path & path) { - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { - PathSet res; - foreach (PathSet::const_iterator, i, paths) - if (isValidPath_(*i)) res.insert(*i); - return res; - } end_retry_sqlite; + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; -} - - -void LocalStore::queryReferences(const Path & path, - PathSet & references) -{ - ValidPathInfo info = queryPathInfo(path); - references.insert(info.references.begin(), info.references.end()); + }); } -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; -} - - -Path LocalStore::queryDeriver(const Path & path) -{ - return queryPathInfo(path).deriver; + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } @@ -997,294 +719,114 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of ‘%1%’") % path); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of ‘%1%’") % path); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of ‘%1%’") % path); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } Path LocalStore::queryPathFromHashPart(const string & hashPart) { - if (hashPart.size() != 32) throw Error("invalid hash part"); + if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); - - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); - - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; -} - - -void LocalStore::setSubstituterEnv() -{ - if (didSetSubstituterEnv) return; - - /* Pass configuration options (including those overridden with - --option) to substituters. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; -} - + return retrySQLite<Path>([&]() { + auto state(_state.lock()); -void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) -{ - if (run.disabled || run.pid != -1) return; - - debug(format("starting substituter program ‘%1%’") % substituter); - - Pipe toPipe, fromPipe, errorPipe; - - toPipe.create(); - fromPipe.create(); - errorPipe.create(); + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - setSubstituterEnv(); + if (!useQueryPathFromHashPart.next()) return ""; - run.pid = startProcess([&]() { - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing ‘%1%’") % substituter); + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; }); - - run.program = baseNameOf(substituter); - run.to = toPipe.writeSide.borrow(); - run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); - run.error = errorPipe.readSide.borrow(); - - toPipe.readSide.close(); - fromPipe.writeSide.close(); - errorPipe.writeSide.close(); - - /* The substituter may exit right away if it's disabled in any way - (e.g. copy-from-other-stores.pl will exit if no other stores - are configured). */ - try { - getLineFromSubstituter(run); - } catch (EndOfFile & e) { - run.to.close(); - run.from.close(); - run.error.close(); - run.disabled = true; - if (run.pid.wait(true) != 0) throw; - } -} - - -/* Read a line from the substituter's stdout, while also processing - its stderr. */ -string LocalStore::getLineFromSubstituter(RunningSubstituter & run) -{ - string res, err; - - /* We might have stdout data left over from the last time. */ - if (run.fromBuf.hasData()) goto haveData; - - while (1) { - checkInterrupt(); - - fd_set fds; - FD_ZERO(&fds); - FD_SET(run.from, &fds); - FD_SET(run.error, &fds); - - /* Wait for data to appear on the substituter's stdout or - stderr. */ - if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { - if (errno == EINTR) continue; - throw SysError("waiting for input from the substituter"); - } - - /* Completely drain stderr before dealing with stdout. */ - if (FD_ISSET(run.error, &fds)) { - char buf[4096]; - ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from substituter's stderr"); - } - if (n == 0) throw EndOfFile(format("substituter ‘%1%’ died unexpectedly") % run.program); - err.append(buf, n); - string::size_type p; - while ((p = err.find('\n')) != string::npos) { - printMsg(lvlError, run.program + ": " + string(err, 0, p)); - err = string(err, p + 1); - } - } - - /* Read from stdout until we get a newline or the buffer is empty. */ - else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { - haveData: - do { - unsigned char c; - run.fromBuf(&c, 1); - if (c == '\n') { - if (!err.empty()) printMsg(lvlError, run.program + ": " + err); - return res; - } - res += c; - } while (run.fromBuf.hasData()); - } - } -} - - -template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) -{ - string s = getLineFromSubstituter(run); - T res; - if (!string2Int(s, res)) throw Error("integer expected from stream"); - return res; } PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { PathSet res; - foreach (Paths::iterator, i, settings.substituters) { - if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[*i]); - startSubstituter(*i, run); - if (run.disabled) continue; - string s = "have "; - foreach (PathSet::const_iterator, j, paths) - if (res.find(*j) == res.end()) { s += *j; s += " "; } - writeLine(run.to, s); - while (true) { - /* FIXME: we only read stderr when an error occurs, so - substituters should only write (short) messages to - stderr when they fail. I.e. they shouldn't write debug - output. */ - Path path = getLineFromSubstituter(run); - if (path == "") break; - res.insert(path); + for (auto & sub : getDefaultSubstituters()) { + if (!sub->wantMassQuery()) continue; + for (auto & path : paths) { + if (res.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + if (sub->isValidPath(path)) + res.insert(path); } } return res; } -void LocalStore::querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos) -{ - RunningSubstituter & run(runningSubstituters[substituter]); - startSubstituter(substituter, run); - if (run.disabled) return; - - string s = "info "; - foreach (PathSet::const_iterator, i, paths) - if (infos.find(*i) == infos.end()) { s += *i; s += " "; } - writeLine(run.to, s); - - while (true) { - Path path = getLineFromSubstituter(run); - if (path == "") break; - if (paths.find(path) == paths.end()) - throw Error(format("got unexpected path ‘%1%’ from substituter") % path); - paths.erase(path); - SubstitutablePathInfo & info(infos[path]); - info.deriver = getLineFromSubstituter(run); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLineFromSubstituter<int>(run); - while (nrRefs--) { - Path p = getLineFromSubstituter(run); - assertStorePath(p); - info.references.insert(p); - } - info.downloadSize = getIntLineFromSubstituter<long long>(run); - info.narSize = getIntLineFromSubstituter<long long>(run); - } -} - - void LocalStore::querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) { - PathSet todo = paths; - foreach (Paths::iterator, i, settings.substituters) { - if (todo.empty()) break; - querySubstitutablePathInfos(*i, todo, infos); + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath) { + } + } } } -Hash LocalStore::queryPathHash(const Path & path) -{ - return queryPathInfo(path).hash; -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { ValidPathInfos infos; @@ -1295,69 +837,112 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { - SQLiteTxn txn(db); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); PathSet paths; - foreach (ValidPathInfos::const_iterator, i, infos) { - assert(i->hash.type == htSHA256); - if (isValidPath_(i->path)) - updatePathInfo(*i); + for (auto & i : infos) { + assert(i.narHash.type == htSHA256); + if (isValidPath_(*state, i.path)) + updatePathInfo(*state, i); else - addValidPath(*i, false); - paths.insert(i->path); + addValidPath(*state, i, false); + paths.insert(i.path); } - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); + for (auto & i : infos) { + auto referrer = queryValidPathId(*state, i.path); + for (auto & j : i.references) + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do this in addValidPath() above, because the references might not be valid yet. */ - foreach (ValidPathInfos::const_iterator, i, infos) - if (isDerivation(i->path)) { + for (auto & i : infos) + if (isDerivation(i.path)) { // FIXME: inefficient; we already loaded the // derivation in addValidPath(). - Derivation drv = readDerivation(i->path); - checkDerivationOutputs(i->path, drv); + Derivation drv = readDerivation(i.path); + checkDerivationOutputs(i.path, drv); } /* Do a topological sort of the paths. This will throw an error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation has multiple outputs. */ - topoSortPaths(*this, paths); + topoSortPaths(paths); txn.commit(); - } end_retry_sqlite; + }); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) +void LocalStore::invalidatePath(State & state, const Path & path) { debug(format("invalidating path ‘%1%’") % path); - drvHashes.erase(path); + state.stmtInvalidatePath.use()(path).exec(); - SQLiteStmtUse use(stmtInvalidatePath); + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ - stmtInvalidatePath.bind(path); + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } +} - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path ‘%1%’ in database") % path); - /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ +void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar, bool repair) +{ + Hash h = hashString(htSHA256, nar); + if (h != info.narHash) + throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % + info.path % info.narHash.to_string() % h.to_string()); + + if (requireSigs && !info.checkSignatures(publicKeys)) + throw Error(format("cannot import path ‘%s’ because it lacks a valid signature") % info.path); + + addTempRoot(info.path); + + if (repair || !isValidPath(info.path)) { + + PathLocks outputLock; + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); + if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end()) + outputLock.lockPaths({info.path}); + + if (repair || !isValidPath(info.path)) { + + deletePath(info.path); + + StringSource source(nar); + restorePath(info.path, source); + + canonicalisePathMetaData(info.path, -1); + + optimisePath(info.path); // FIXME: combine with hashPath() + + registerValidPath(info); + } + + outputLock.setDeletion(true); + } } @@ -1375,11 +960,11 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + PathLocks outputLock({dstPath}); if (repair || !isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); if (recursive) { StringSource source(dump); @@ -1404,8 +989,9 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; + info.ultimate = true; registerValidPath(info); } @@ -1420,7 +1006,6 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); - debug(format("adding ‘%1%’ to the store") % srcPath); /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for @@ -1429,9 +1014,9 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, if (recursive) dumpPath(srcPath, sink, filter); else - sink.s = readFile(srcPath); + sink.s = make_ref<std::string>(readFile(srcPath)); - return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); } @@ -1444,25 +1029,28 @@ Path LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - PathLocks outputLock(singleton<PathSet, Path>(dstPath)); + PathLocks outputLock({dstPath}); if (repair || !isValidPath(dstPath)) { - if (pathExists(dstPath)) deletePath(dstPath); + deletePath(dstPath); writeFile(dstPath, s); canonicalisePathMetaData(dstPath, -1); - HashResult hash = hashPath(htSHA256, dstPath); + StringSink sink; + dumpString(s, sink); + auto hash = hashString(htSHA256, *sink.s); optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; + info.narHash = hash; + info.narSize = sink.s->size(); info.references = references; + info.ultimate = true; registerValidPath(info); } @@ -1473,119 +1061,6 @@ Path LocalStore::addTextToStore(const string & name, const string & s, } -struct HashAndWriteSink : Sink -{ - Sink & writeSink; - HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) - { - } - virtual void operator () (const unsigned char * data, size_t len) - { - writeSink(data, len); - hashSink(data, len); - } - Hash currentHash() - { - return hashSink.currentHash().first; - } -}; - - -#define EXPORT_MAGIC 0x4558494e - - -static void checkSecrecy(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError(format("getting status of ‘%1%’") % path); - if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0) - throw Error(format("file ‘%1%’ should be secret (inaccessible to everybody else)!") % path); -} - - -void LocalStore::exportPath(const Path & path, bool sign, - Sink & sink) -{ - assertStorePath(path); - - printMsg(lvlInfo, format("exporting path ‘%1%’") % path); - - if (!isValidPath(path)) - throw Error(format("path ‘%1%’ is not valid") % path); - - HashAndWriteSink hashAndWriteSink(sink); - - dumpPath(path, hashAndWriteSink); - - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashAndWriteSink.currentHash(); - Hash storedHash = queryPathHash(path); - if (hash != storedHash && storedHash != Hash(storedHash.type)) - throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path - % printHash(storedHash) % printHash(hash)); - - writeInt(EXPORT_MAGIC, hashAndWriteSink); - - writeString(path, hashAndWriteSink); - - PathSet references; - queryReferences(path, references); - writeStrings(references, hashAndWriteSink); - - Path deriver = queryDeriver(path); - writeString(deriver, hashAndWriteSink); - - if (sign) { - Hash hash = hashAndWriteSink.currentHash(); - - writeInt(1, hashAndWriteSink); - - Path tmpDir = createTempDir(); - AutoDelete delTmp(tmpDir); - Path hashFile = tmpDir + "/hash"; - writeFile(hashFile, printHash(hash)); - - Path secretKey = settings.nixConfDir + "/signing-key.sec"; - checkSecrecy(secretKey); - - Strings args; - args.push_back("rsautl"); - args.push_back("-sign"); - args.push_back("-inkey"); - args.push_back(secretKey); - args.push_back("-in"); - args.push_back(hashFile); - string signature = runProgram(OPENSSL_PATH, true, args); - - writeString(signature, hashAndWriteSink); - - } else - writeInt(0, hashAndWriteSink); -} - - -struct HashAndReadSource : Source -{ - Source & readSource; - HashSink hashSink; - bool hashing; - HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256) - { - hashing = true; - } - size_t read(unsigned char * data, size_t len) - { - size_t n = readSource.read(data, len); - if (hashing) hashSink(data, n); - return n; - } -}; - - /* Create a temporary directory in the store that won't be garbage-collected. */ Path LocalStore::createTempDirInStore() @@ -1602,145 +1077,26 @@ Path LocalStore::createTempDirInStore() } -Path LocalStore::importPath(bool requireSignature, Source & source) -{ - HashAndReadSource hashAndReadSource(source); - - /* We don't yet know what store path this archive contains (the - store path follows the archive data proper), and besides, we - don't know yet whether the signature is valid. */ - Path tmpDir = createTempDirInStore(); - AutoDelete delTmp(tmpDir); - Path unpacked = tmpDir + "/unpacked"; - - restorePath(unpacked, hashAndReadSource); - - unsigned int magic = readInt(hashAndReadSource); - if (magic != EXPORT_MAGIC) - throw Error("Nix archive cannot be imported; wrong format"); - - Path dstPath = readStorePath(hashAndReadSource); - - PathSet references = readStorePaths<PathSet>(hashAndReadSource); - - Path deriver = readString(hashAndReadSource); - if (deriver != "") assertStorePath(deriver); - - Hash hash = hashAndReadSource.hashSink.finish().first; - hashAndReadSource.hashing = false; - - bool haveSignature = readInt(hashAndReadSource) == 1; - - if (requireSignature && !haveSignature) - throw Error(format("imported archive of ‘%1%’ lacks a signature") % dstPath); - - if (haveSignature) { - string signature = readString(hashAndReadSource); - - if (requireSignature) { - Path sigFile = tmpDir + "/sig"; - writeFile(sigFile, signature); - - Strings args; - args.push_back("rsautl"); - args.push_back("-verify"); - args.push_back("-inkey"); - args.push_back(settings.nixConfDir + "/signing-key.pub"); - args.push_back("-pubin"); - args.push_back("-in"); - args.push_back(sigFile); - string hash2 = runProgram(OPENSSL_PATH, true, args); - - /* Note: runProgram() throws an exception if the signature - is invalid. */ - - if (printHash(hash) != hash2) - throw Error( - "signed hash doesn't match actual contents of imported " - "archive; archive could be corrupt, or someone is trying " - "to import a Trojan horse"); - } - } - - /* Do the actual import. */ - - /* !!! way too much code duplication with addTextToStore() etc. */ - addTempRoot(dstPath); - - if (!isValidPath(dstPath)) { - - PathLocks outputLock; - - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); - if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end()) - outputLock.lockPaths(singleton<PathSet, Path>(dstPath)); - - if (!isValidPath(dstPath)) { - - if (pathExists(dstPath)) deletePath(dstPath); - - if (rename(unpacked.c_str(), dstPath.c_str()) == -1) - throw SysError(format("cannot move ‘%1%’ to ‘%2%’") - % unpacked % dstPath); - - canonicalisePathMetaData(dstPath, -1); - - /* !!! if we were clever, we could prevent the hashPath() - here. */ - HashResult hash = hashPath(htSHA256, dstPath); - - optimisePath(dstPath); // FIXME: combine with hashPath() - - ValidPathInfo info; - info.path = dstPath; - info.hash = hash.first; - info.narSize = hash.second; - info.references = references; - info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -Paths LocalStore::importPaths(bool requireSignature, Source & source) -{ - Paths res; - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); - res.push_back(importPath(requireSignature, source)); - } - return res; -} - - void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteTxn txn(db); + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); + if (isValidPath_(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") % path % showPaths(referrers)); - invalidatePath(path); + invalidatePath(*state, path); } txn.commit(); - } end_retry_sqlite; + }); } @@ -1761,8 +1117,8 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; - foreach (PathSet::iterator, i, validPaths2) - verifyPath(*i, store, done, validPaths, repair, errors); + for (auto & i : validPaths2) + verifyPath(i, store, done, validPaths, repair, errors); /* Release the GC lock so that checking content hashes (which can take ages) doesn't block the GC or builds. */ @@ -1774,45 +1130,48 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) Hash nullHash(htSHA256); - foreach (PathSet::iterator, i, validPaths) { + for (auto & i : validPaths) { try { - ValidPathInfo info = queryPathInfo(*i); + auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of ‘%1%’") % *i); - HashResult current = hashPath(info.hash.type, *i); + printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i); + HashResult current = hashPath(info->narHash.type, i); - if (info.hash != nullHash && info.hash != current.first) { + if (info->narHash != nullHash && info->narHash != current.first) { printMsg(lvlError, format("path ‘%1%’ was modified! " "expected hash ‘%2%’, got ‘%3%’") - % *i % printHash(info.hash) % printHash(current.first)); - if (repair) repairPath(*i); else errors = true; + % i % printHash(info->narHash) % printHash(current.first)); + if (repair) repairPath(i); else errors = true; } else { bool update = false; /* Fill in missing hashes. */ - if (info.hash == nullHash) { - printMsg(lvlError, format("fixing missing hash on ‘%1%’") % *i); - info.hash = current.first; + if (info->narHash == nullHash) { + printMsg(lvlError, format("fixing missing hash on ‘%1%’") % i); + info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ - if (info.narSize == 0) { - printMsg(lvlError, format("updating size field on ‘%1%’ to %2%") % *i % current.second); - info.narSize = current.second; + if (info->narSize == 0) { + printMsg(lvlError, format("updating size field on ‘%1%’ to %2%") % i % current.second); + info->narSize = current.second; update = true; } - if (update) updatePathInfo(info); + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } } } catch (Error & e) { /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ - if (isValidPath(*i)) + if (isValidPath(i)) printMsg(lvlError, format("error: %1%") % e.msg()); else printMsg(lvlError, format("warning: %1%") % e.msg()); @@ -1835,7 +1194,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (!isStorePath(path)) { printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); return; } @@ -1844,16 +1204,17 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, first, then we can invalidate this path as well. */ bool canInvalidate = true; PathSet referrers; queryReferrers(path, referrers); - foreach (PathSet::iterator, i, referrers) - if (*i != path) { - verifyPath(*i, store, done, validPaths, repair, errors); - if (validPaths.find(*i) != validPaths.end()) + for (auto & i : referrers) + if (i != path) { + verifyPath(i, store, done, validPaths, repair, errors); + if (validPaths.find(i) != validPaths.end()) canInvalidate = false; } if (canInvalidate) { printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); } else { printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); if (repair) @@ -1873,114 +1234,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.hash.type, path); - Hash nullHash(htSHA256); - res = info.hash == nullHash || info.hash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - -/* Functions for upgrading from the pre-SQLite database. */ - -PathSet LocalStore::queryValidPathsOld() -{ - PathSet paths; - for (auto & i : readDirectory(settings.nixDBPath + "/info")) - if (i.name.at(0) != '.') paths.insert(settings.nixStore + "/" + i.name); - return paths; -} - - -ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) -{ - ValidPathInfo res; - res.path = path; - - /* Read the info file. */ - string baseName = baseNameOf(path); - Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); - if (!pathExists(infoFile)) - throw Error(format("path ‘%1%’ is not valid") % path); - string info = readFile(infoFile); - - /* Parse it. */ - Strings lines = tokenizeString<Strings>(info, "\n"); - - foreach (Strings::iterator, i, lines) { - string::size_type p = i->find(':'); - if (p == string::npos) - throw Error(format("corrupt line in ‘%1%’: %2%") % infoFile % *i); - string name(*i, 0, p); - string value(*i, p + 2); - if (name == "References") { - Strings refs = tokenizeString<Strings>(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - res.hash = parseHashField(path, value); - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } - } - - return res; -} - - -/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ -void LocalStore::upgradeStore6() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - openDB(true); - - PathSet validPaths = queryValidPathsOld(); - - SQLiteTxn txn(db); - - foreach (PathSet::iterator, i, validPaths) { - addValidPath(queryPathInfoOld(*i), false); - std::cerr << "."; - } - - std::cerr << "|"; - - foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfoOld(*i); - unsigned long long referrer = queryValidPathId(*i); - foreach (PathSet::iterator, j, info.references) - addReference(referrer, queryValidPathId(*j)); - std::cerr << "."; - } - - std::cerr << "\n"; - - txn.commit(); -} - - #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) @@ -2035,8 +1288,41 @@ void LocalStore::upgradeStore7() void LocalStore::vacuumDB() { - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); + auto state(_state.lock()); + + if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "vacuuming SQLite database"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + + info->sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, *info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.get("secret-key-files", Strings()); + + for (auto & secretKeyFile : secretKeyFiles) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 819f59327a23..2a3f452bc5c7 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,15 +1,14 @@ #pragma once -#include <string> -#include <unordered_set> +#include "sqlite.hh" +#include "pathlocks.hh" #include "store-api.hh" +#include "sync.hh" #include "util.hh" -#include "pathlocks.hh" - -class sqlite3; -class sqlite3_stmt; +#include <string> +#include <unordered_set> namespace nix { @@ -18,8 +17,8 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. */ -const int nixSchemaVersion = 7; + Nix 1.0. Version 7 is Nix 1.3. Version 9 is 1.12. */ +const int nixSchemaVersion = 9; extern string drvsLogDir; @@ -41,98 +40,88 @@ struct OptimiseStats }; -struct RunningSubstituter -{ - Path program; - Pid pid; - AutoCloseFD to, from, error; - FdSource fromBuf; - bool disabled; - RunningSubstituter() : disabled(false) { }; -}; - - -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - -class LocalStore : public StoreAPI +class LocalStore : public LocalFSStore { private: - typedef std::map<Path, RunningSubstituter> RunningSubstituters; - RunningSubstituters runningSubstituters; - Path linksDir; + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; + + struct State + { + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; + + /* The file to which we write our temporary roots. */ + Path fnTempRoots; + AutoCloseFD fdTempRoots; + }; + + Sync<State, std::recursive_mutex> _state; + + const Path linksDir; + const Path reservedPath; + const Path schemaPath; + + bool requireSigs; + + PublicKeys publicKeys; public: /* Initialise the local store, upgrading the schema if necessary. */ - LocalStore(bool reserveSpace = true); + LocalStore(); ~LocalStore(); /* Implementations of abstract store API methods. */ - bool isValidPath(const Path & path); - - PathSet queryValidPaths(const PathSet & paths); + std::string getUri() override; - PathSet queryAllValidPaths(); + bool isValidPathUncached(const Path & path) override; - ValidPathInfo queryPathInfo(const Path & path); + PathSet queryValidPaths(const PathSet & paths) override; - Hash queryPathHash(const Path & path); + PathSet queryAllValidPaths() override; - void queryReferences(const Path & path, PathSet & references); + std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override; - void queryReferrers(const Path & path, PathSet & referrers); + void queryReferrers(const Path & path, PathSet & referrers) override; - Path queryDeriver(const Path & path); + PathSet queryValidDerivers(const Path & path) override; - PathSet queryValidDerivers(const Path & path); + PathSet queryDerivationOutputs(const Path & path) override; - PathSet queryDerivationOutputs(const Path & path); + StringSet queryDerivationOutputNames(const Path & path) override; - StringSet queryDerivationOutputNames(const Path & path); + Path queryPathFromHashPart(const string & hashPart) override; - Path queryPathFromHashPart(const string & hashPart); - - PathSet querySubstitutablePaths(const PathSet & paths); - - void querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos); + PathSet querySubstitutablePaths(const PathSet & paths) override; void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos); + SubstitutablePathInfos & infos) override; + + void addToStore(const ValidPathInfo & info, const std::string & nar, + bool repair) override; Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false); + PathFilter & filter = defaultPathFilter, bool repair = false) override; /* Like addToStore(), but the contents of the path are contained in `dump', which is either a NAR serialisation (if recursive == @@ -142,40 +131,35 @@ public: bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false); + const PathSet & references, bool repair = false) override; - void exportPath(const Path & path, bool sign, - Sink & sink); + void buildPaths(const PathSet & paths, BuildMode buildMode) override; - Paths importPaths(bool requireSignature, Source & source); + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; - void buildPaths(const PathSet & paths, BuildMode buildMode); + void ensurePath(const Path & path) override; - void ensurePath(const Path & path); + void addTempRoot(const Path & path) override; - void addTempRoot(const Path & path); + void addIndirectRoot(const Path & path) override; - void addIndirectRoot(const Path & path); + void syncWithGC() override; - void syncWithGC(); + Roots findRoots() override; - Roots findRoots(); - - void collectGarbage(const GCOptions & options, GCResults & results); + void collectGarbage(const GCOptions & options, GCResults & results) override; /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ void optimiseStore(OptimiseStats & stats); - /* Generic variant of the above method. */ - void optimiseStore(); + void optimiseStore() override; /* Optimise a single store path. */ void optimisePath(const Path & path); - /* Check the integrity of the Nix store. Returns true if errors - remain. */ - bool verifyStore(bool checkContents, bool repair); + bool verifyStore(bool checkContents, bool repair) override; /* Register the validity of a path, i.e., that `path' exists, that the paths referenced by it exists, and in the case of an output @@ -187,84 +171,29 @@ public: void registerValidPaths(const ValidPathInfos & infos); - /* Register that the build of a derivation with output `path' has - failed. */ - void registerFailedPath(const Path & path); - - /* Query whether `path' previously failed to build. */ - bool hasPathFailed(const Path & path); - - PathSet queryFailedPaths(); - - void clearFailedPaths(const PathSet & paths); - void vacuumDB(); /* Repair the contents of the given path by redownloading it using a substituter (if available). */ void repairPath(const Path & path); - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const Path & path); - - void markContentsGood(const Path & path); + void addSignatures(const Path & storePath, const StringSet & sigs) override; - void setSubstituterEnv(); + static bool haveWriteAccess(); private: - Path schemaPath; - - /* Lock file used for upgrading. */ - AutoCloseFD globalLock; - - /* The SQLite database object. */ - SQLite db; - - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtRegisterFailedPath; - SQLiteStmt stmtHasPathFailed; - SQLiteStmt stmtQueryFailedPaths; - SQLiteStmt stmtClearFailedPath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - - /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; - - bool didSetSubstituterEnv; - - /* The file to which we write our temporary roots. */ - Path fnTempRoots; - AutoCloseFD fdTempRoots; - int getSchema(); - void openDB(bool create); + void openDB(State & state, bool create); void makeStoreWritable(); - unsigned long long queryValidPathId(const Path & path); - - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + uint64_t queryValidPathId(State & state, const Path & path); - void addReference(unsigned long long referrer, unsigned long long reference); + uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void appendReferrer(const Path & from, const Path & to, bool lock); - - void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - - void invalidatePath(const Path & path); + void invalidatePath(State & state, const Path & path); /* Delete a path from the Nix store. */ void invalidatePathChecked(const Path & path); @@ -272,7 +201,7 @@ private: void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths, bool repair, bool & errors); - void updatePathInfo(const ValidPathInfo & info); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); void upgradeStore7(); @@ -294,19 +223,14 @@ private: int openGCLock(LockType lockType); - void removeUnusedLinks(const GCState & state); + void findRoots(const Path & path, unsigned char type, Roots & roots); - void startSubstituter(const Path & substituter, - RunningSubstituter & runningSubstituter); + void findRuntimeRoots(PathSet & roots); - string getLineFromSubstituter(RunningSubstituter & run); - - template<class T> T getIntLineFromSubstituter(RunningSubstituter & run); + void removeUnusedLinks(const GCState & state); Path createTempDirInStore(); - Path importPath(bool requireSignature, Source & source); - void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); typedef std::unordered_set<ino_t> InodeHash; @@ -316,8 +240,15 @@ private: void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(const Path & path); - void queryReferrers_(const Path & path, PathSet & referrers); + bool isValidPath_(State & state, const Path & path); + void queryReferrers(State & state, const Path & path, PathSet & referrers); + + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + void signPathInfo(ValidPathInfo & info); + + friend class DerivationGoal; + friend class SubstitutionGoal; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 771c06753a65..9d5c04dca0c5 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,11 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat -libstore_LDFLAGS = -lsqlite3 -lbz2 +libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread + +ifeq ($(ENABLE_S3), 1) + libstore_LDFLAGS += -laws-cpp-sdk-s3 -laws-cpp-sdk-core +endif ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket @@ -33,3 +37,4 @@ $(d)/local-store.cc: $(d)/schema.sql.hh clean-files += $(d)/schema.sql.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) +$(eval $(call install-file-in, $(d)/sandbox-defaults.sb, $(datadir)/nix, 0644)) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 736434ca4895..5c284d1b9ab2 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,21 +1,21 @@ -#include "misc.hh" -#include "store-api.hh" -#include "local-store.hh" +#include "derivations.hh" #include "globals.hh" +#include "local-store.hh" +#include "store-api.hh" namespace nix { -Derivation derivationFromPath(StoreAPI & store, const Path & drvPath) +Derivation Store::derivationFromPath(const Path & drvPath) { assertStorePath(drvPath); - store.ensurePath(drvPath); + ensurePath(drvPath); return readDerivation(drvPath); } -void computeFSClosure(StoreAPI & store, const Path & path, +void Store::computeFSClosure(const Path & path, PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers) { if (paths.find(path) != paths.end()) return; @@ -24,50 +24,41 @@ void computeFSClosure(StoreAPI & store, const Path & path, PathSet edges; if (flipDirection) { - store.queryReferrers(path, edges); + queryReferrers(path, edges); if (includeOutputs) { - PathSet derivers = store.queryValidDerivers(path); - foreach (PathSet::iterator, i, derivers) - edges.insert(*i); + PathSet derivers = queryValidDerivers(path); + for (auto & i : derivers) + edges.insert(i); } if (includeDerivers && isDerivation(path)) { - PathSet outputs = store.queryDerivationOutputs(path); - foreach (PathSet::iterator, i, outputs) - if (store.isValidPath(*i) && store.queryDeriver(*i) == path) - edges.insert(*i); + PathSet outputs = queryDerivationOutputs(path); + for (auto & i : outputs) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + edges.insert(i); } } else { - store.queryReferences(path, edges); + auto info = queryPathInfo(path); + edges = info->references; if (includeOutputs && isDerivation(path)) { - PathSet outputs = store.queryDerivationOutputs(path); - foreach (PathSet::iterator, i, outputs) - if (store.isValidPath(*i)) edges.insert(*i); + PathSet outputs = queryDerivationOutputs(path); + for (auto & i : outputs) + if (isValidPath(i)) edges.insert(i); } - if (includeDerivers) { - Path deriver = store.queryDeriver(path); - if (store.isValidPath(deriver)) edges.insert(deriver); - } + if (includeDerivers && isValidPath(info->deriver)) + edges.insert(info->deriver); } - foreach (PathSet::iterator, i, edges) - computeFSClosure(store, *i, paths, flipDirection, includeOutputs, includeDerivers); -} - - -Path findOutput(const Derivation & drv, string id) -{ - foreach (DerivationOutputs::const_iterator, i, drv.outputs) - if (i->first == id) return i->second.path; - throw Error(format("derivation has no output ‘%1%’") % id); + for (auto & i : edges) + computeFSClosure(i, paths, flipDirection, includeOutputs, includeDerivers); } -void queryMissing(StoreAPI & store, const PathSet & targets, +void Store::queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) { @@ -98,60 +89,60 @@ void queryMissing(StoreAPI & store, const PathSet & targets, PathSet query, todoDrv, todoNonDrv; - foreach (PathSet::iterator, i, todo) { - if (done.find(*i) != done.end()) continue; - done.insert(*i); + for (auto & i : todo) { + if (done.find(i) != done.end()) continue; + done.insert(i); - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); if (isDerivation(i2.first)) { - if (!store.isValidPath(i2.first)) { + if (!isValidPath(i2.first)) { // FIXME: we could try to substitute p. - unknown.insert(*i); + unknown.insert(i); continue; } - Derivation drv = derivationFromPath(store, i2.first); + Derivation drv = derivationFromPath(i2.first); PathSet invalid; - foreach (DerivationOutputs::iterator, j, drv.outputs) - if (wantOutput(j->first, i2.second) - && !store.isValidPath(j->second.path)) - invalid.insert(j->second.path); + for (auto & j : drv.outputs) + if (wantOutput(j.first, i2.second) + && !isValidPath(j.second.path)) + invalid.insert(j.second.path); if (invalid.empty()) continue; - todoDrv.insert(*i); - if (settings.useSubstitutes && substitutesAllowed(drv)) + todoDrv.insert(i); + if (settings.useSubstitutes && drv.substitutesAllowed()) query.insert(invalid.begin(), invalid.end()); } else { - if (store.isValidPath(*i)) continue; - query.insert(*i); - todoNonDrv.insert(*i); + if (isValidPath(i)) continue; + query.insert(i); + todoNonDrv.insert(i); } } todo.clear(); SubstitutablePathInfos infos; - store.querySubstitutablePathInfos(query, infos); + querySubstitutablePathInfos(query, infos); - foreach (PathSet::iterator, i, todoDrv) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i); + for (auto & i : todoDrv) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); // FIXME: cache this - Derivation drv = derivationFromPath(store, i2.first); + Derivation drv = derivationFromPath(i2.first); PathSet outputs; bool mustBuild = false; - if (settings.useSubstitutes && substitutesAllowed(drv)) { - foreach (DerivationOutputs::iterator, j, drv.outputs) { - if (!wantOutput(j->first, i2.second)) continue; - if (!store.isValidPath(j->second.path)) { - if (infos.find(j->second.path) == infos.end()) + if (settings.useSubstitutes && drv.substitutesAllowed()) { + for (auto & j : drv.outputs) { + if (!wantOutput(j.first, i2.second)) continue; + if (!isValidPath(j.second.path)) { + if (infos.find(j.second.path) == infos.end()) mustBuild = true; else - outputs.insert(j->second.path); + outputs.insert(j.second.path); } } } else @@ -160,59 +151,61 @@ void queryMissing(StoreAPI & store, const PathSet & targets, if (mustBuild) { willBuild.insert(i2.first); todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - foreach (DerivationInputs::iterator, j, drv.inputDrvs) - todo.insert(makeDrvPathWithOutputs(j->first, j->second)); + for (auto & j : drv.inputDrvs) + todo.insert(makeDrvPathWithOutputs(j.first, j.second)); } else todoNonDrv.insert(outputs.begin(), outputs.end()); } - foreach (PathSet::iterator, i, todoNonDrv) { - done.insert(*i); - SubstitutablePathInfos::iterator info = infos.find(*i); + for (auto & i : todoNonDrv) { + done.insert(i); + SubstitutablePathInfos::iterator info = infos.find(i); if (info != infos.end()) { - willSubstitute.insert(*i); + willSubstitute.insert(i); downloadSize += info->second.downloadSize; narSize += info->second.narSize; todo.insert(info->second.references.begin(), info->second.references.end()); } else - unknown.insert(*i); + unknown.insert(i); } } } -static void dfsVisit(StoreAPI & store, const PathSet & paths, - const Path & path, PathSet & visited, Paths & sorted, - PathSet & parents) +Paths Store::topoSortPaths(const PathSet & paths) { - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of ‘%1%’") % path); + Paths sorted; + PathSet visited, parents; - if (visited.find(path) != visited.end()) return; - visited.insert(path); - parents.insert(path); + std::function<void(const Path & path)> dfsVisit; - PathSet references; - if (store.isValidPath(path)) - store.queryReferences(path, references); + dfsVisit = [&](const Path & path) { + if (parents.find(path) != parents.end()) + throw BuildError(format("cycle detected in the references of ‘%1%’") % path); - foreach (PathSet::iterator, i, references) - /* Don't traverse into paths that don't exist. That can - happen due to substitutes for non-existent paths. */ - if (*i != path && paths.find(*i) != paths.end()) - dfsVisit(store, paths, *i, visited, sorted, parents); + if (visited.find(path) != visited.end()) return; + visited.insert(path); + parents.insert(path); - sorted.push_front(path); - parents.erase(path); -} + PathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath &) { + } + for (auto & i : references) + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (i != path && paths.find(i) != paths.end()) + dfsVisit(i); + + sorted.push_front(path); + parents.erase(path); + }; + + for (auto & i : paths) + dfsVisit(i); -Paths topoSortPaths(StoreAPI & store, const PathSet & paths) -{ - Paths sorted; - PathSet visited, parents; - foreach (PathSet::const_iterator, i, paths) - dfsVisit(store, paths, *i, visited, sorted, parents); return sorted; } diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh deleted file mode 100644 index d3e31d51f72c..000000000000 --- a/src/libstore/misc.hh +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "derivations.hh" - - -namespace nix { - - -/* Read a derivation, after ensuring its existence through - ensurePath(). */ -Derivation derivationFromPath(StoreAPI & store, const Path & drvPath); - -/* Place in `paths' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly or - indirectly reached from it. `paths' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ -void computeFSClosure(StoreAPI & store, const Path & path, - PathSet & paths, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false); - -/* Return the path corresponding to the output identifier `id' in the - given derivation. */ -Path findOutput(const Derivation & drv, string id); - -/* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths that - will be substituted. */ -void queryMissing(StoreAPI & store, const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); - -bool willBuildLocally(const Derivation & drv); - -bool substitutesAllowed(const Derivation & drv); - - -} diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc new file mode 100644 index 000000000000..8896862be149 --- /dev/null +++ b/src/libstore/nar-accessor.cc @@ -0,0 +1,141 @@ +#include "nar-accessor.hh" +#include "archive.hh" + +#include <map> + +namespace nix { + +struct NarMember +{ + FSAccessor::Type type; + + bool isExecutable; + + /* If this is a regular file, position of the contents of this + file in the NAR. */ + size_t start, size; + + std::string target; +}; + +struct NarIndexer : ParseSink, StringSource +{ + // FIXME: should store this as a tree. Now we're vulnerable to + // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}). + typedef std::map<Path, NarMember> Members; + Members members; + + Path currentPath; + std::string currentStart; + bool isExec; + + NarIndexer(const std::string & nar) : StringSource(nar) + { + } + + void createDirectory(const Path & path) override + { + members.emplace(path, + NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); + } + + void createRegularFile(const Path & path) override + { + currentPath = path; + } + + void isExecutable() override + { + isExec = true; + } + + void preallocateContents(unsigned long long size) override + { + currentStart = string(s, pos, 16); + members.emplace(currentPath, + NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); + } + + void receiveContents(unsigned char * data, unsigned int len) override + { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == string((char *) data, 16)); + currentStart.clear(); + } + } + + void createSymlink(const Path & path, const string & target) override + { + members.emplace(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + } + + Members::iterator find(const Path & path) + { + auto i = members.find(path); + if (i == members.end()) + throw Error(format("NAR file does not contain path ‘%1%’") % path); + return i; + } +}; + +struct NarAccessor : public FSAccessor +{ + ref<const std::string> nar; + NarIndexer indexer; + + NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar) + { + parseDump(indexer, indexer); + } + + Stat stat(const Path & path) override + { + auto i = indexer.members.find(path); + if (i == indexer.members.end()) + return {FSAccessor::Type::tMissing, 0, false}; + return {i->second.type, i->second.size, i->second.isExecutable}; + } + + StringSet readDirectory(const Path & path) override + { + auto i = indexer.find(path); + + if (i->second.type != FSAccessor::Type::tDirectory) + throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); + + ++i; + StringSet res; + while (i != indexer.members.end() && isInDir(i->first, path)) { + // FIXME: really bad performance. + if (i->first.find('/', path.size() + 1) == std::string::npos) + res.insert(std::string(i->first, path.size() + 1)); + ++i; + } + return res; + } + + std::string readFile(const Path & path) override + { + auto i = indexer.find(path); + if (i->second.type != FSAccessor::Type::tRegular) + throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); + return std::string(*nar, i->second.start, i->second.size); + } + + std::string readLink(const Path & path) override + { + auto i = indexer.find(path); + if (i->second.type != FSAccessor::Type::tSymlink) + throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); + return i->second.target; + } +}; + +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) +{ + return make_ref<NarAccessor>(nar); +} + +} diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh new file mode 100644 index 000000000000..83c570be4c7b --- /dev/null +++ b/src/libstore/nar-accessor.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "fs-accessor.hh" + +namespace nix { + +/* Return an object that provides access to the contents of a NAR + file. */ +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); + +} diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc new file mode 100644 index 000000000000..ae368e152866 --- /dev/null +++ b/src/libstore/nar-info-disk-cache.cc @@ -0,0 +1,224 @@ +#include "nar-info-disk-cache.hh" +#include "sync.hh" +#include "sqlite.hh" +#include "globals.hh" + +#include <sqlite3.h> + +namespace nix { + +static const char * schema = R"sql( + +create table if not exists BinaryCaches ( + id integer primary key autoincrement not null, + url text unique not null, + timestamp integer not null, + storeDir text not null, + wantMassQuery integer not null, + priority integer not null +); + +create table if not exists NARs ( + cache integer not null, + hashPart text not null, + namePart text not null, + url text, + compression text, + fileHash text, + fileSize integer, + narHash text, + narSize integer, + refs text, + deriver text, + sigs text, + timestamp integer not null, + primary key (cache, hashPart), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +create table if not exists NARExistence ( + cache integer not null, + storePath text not null, + exist integer not null, + timestamp integer not null, + primary key (cache, storePath), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +)sql"; + +class NarInfoDiskCacheImpl : public NarInfoDiskCache +{ +public: + + /* How long negative lookups are valid. */ + const int ttlNegative = 3600; + + struct State + { + SQLite db; + SQLiteStmt insertCache, queryCache, insertNAR, queryNAR, insertNARExistence, queryNARExistence; + std::map<std::string, int> caches; + }; + + Sync<State> _state; + + NarInfoDiskCacheImpl() + { + auto state(_state.lock()); + + Path dbPath = getCacheDir() + "/nix/binary-cache-v4.sqlite"; + createDirs(dirOf(dbPath)); + + if (sqlite3_open_v2(dbPath.c_str(), &state->db.db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error(format("cannot open store cache ‘%s’") % dbPath); + + if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(state->db, "setting timeout"); + + // We can always reproduce the cache. + if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "making database asynchronous"); + if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "setting journal mode"); + + if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "initialising database schema"); + + state->insertCache.create(state->db, + "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + + state->queryCache.create(state->db, + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + + state->insertNAR.create(state->db, + "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " + "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + state->queryNAR.create(state->db, + "select * from NARs where cache = ? and hashPart = ?"); + + state->insertNARExistence.create(state->db, + "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)"); + + state->queryNARExistence.create(state->db, + "select exist, timestamp from NARExistence where cache = ? and storePath = ?"); + } + + int uriToInt(State & state, const std::string & uri) + { + auto i = state.caches.find(uri); + if (i == state.caches.end()) abort(); + return i->second; + } + + void createCache(const std::string & uri, bool wantMassQuery, int priority) override + { + auto state(_state.lock()); + + // FIXME: race + + state->insertCache.use()(uri)(time(0))(settings.nixStore)(wantMassQuery)(priority).exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = sqlite3_last_insert_rowid(state->db); + } + + bool cacheExists(const std::string & uri) override + { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i != state->caches.end()) return true; + + auto queryCache(state->queryCache.use()(uri)); + + if (queryCache.next()) { + state->caches[uri] = queryCache.getInt(0); + return true; + } + + return false; + } + + std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string & uri, const std::string & hashPart) override + { + auto state(_state.lock()); + + auto queryNAR(state->queryNAR.use() + (uriToInt(*state, uri)) + (hashPart)); + + if (!queryNAR.next()) + // FIXME: check NARExistence + return {oUnknown, 0}; + + auto narInfo = make_ref<NarInfo>(); + + // FIXME: implement TTL. + + auto namePart = queryNAR.getStr(2); + narInfo->path = settings.nixStore + "/" + + hashPart + (namePart.empty() ? "" : "-" + namePart); + narInfo->url = queryNAR.getStr(3); + narInfo->compression = queryNAR.getStr(4); + if (!queryNAR.isNull(5)) + narInfo->fileHash = parseHash(queryNAR.getStr(5)); + narInfo->fileSize = queryNAR.getInt(6); + narInfo->narHash = parseHash(queryNAR.getStr(7)); + narInfo->narSize = queryNAR.getInt(8); + for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " ")) + narInfo->references.insert(settings.nixStore + "/" + r); + if (!queryNAR.isNull(10)) + narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(10); + for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " ")) + narInfo->sigs.insert(sig); + + return {oValid, narInfo}; + } + + void upsertNarInfo( + const std::string & uri, const std::string & hashPart, + std::shared_ptr<ValidPathInfo> info) override + { + auto state(_state.lock()); + + if (info) { + + auto narInfo = std::dynamic_pointer_cast<NarInfo>(info); + + assert(hashPart == storePathToHash(info->path)); + + state->insertNAR.use() + (uriToInt(*state, uri)) + (hashPart) + (storePathToName(info->path)) + (narInfo ? narInfo->url : "", narInfo != 0) + (narInfo ? narInfo->compression : "", narInfo != 0) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) + (info->narHash.to_string()) + (info->narSize) + (concatStringsSep(" ", info->shortRefs())) + (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") + (concatStringsSep(" ", info->sigs)) + (time(0)).exec(); + + } else { + // not implemented + abort(); + } + } +}; + +ref<NarInfoDiskCache> getNarInfoDiskCache() +{ + static Sync<std::shared_ptr<NarInfoDiskCache>> cache; + + auto cache_(cache.lock()); + if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>(); + return ref<NarInfoDiskCache>(*cache_); +} + +} diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh new file mode 100644 index 000000000000..ce5da062c5e3 --- /dev/null +++ b/src/libstore/nar-info-disk-cache.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "ref.hh" +#include "nar-info.hh" + +namespace nix { + +class NarInfoDiskCache +{ +public: + typedef enum { oValid, oInvalid, oUnknown } Outcome; + + virtual void createCache(const std::string & uri, bool wantMassQuery, int priority) = 0; + + virtual bool cacheExists(const std::string & uri) = 0; + + virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string & uri, const std::string & hashPart) = 0; + + virtual void upsertNarInfo( + const std::string & uri, const std::string & hashPart, + std::shared_ptr<ValidPathInfo> info) = 0; +}; + +/* Return a singleton cache object that can be used concurrently by + multiple threads. */ +ref<NarInfoDiskCache> getNarInfoDiskCache(); + +} diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc new file mode 100644 index 000000000000..c0c5cecd1730 --- /dev/null +++ b/src/libstore/nar-info.cc @@ -0,0 +1,106 @@ +#include "globals.hh" +#include "nar-info.hh" + +namespace nix { + +NarInfo::NarInfo(const std::string & s, const std::string & whence) +{ + auto corrupt = [&]() [[noreturn]] { + throw Error("NAR info file ‘%1%’ is corrupt"); + }; + + auto parseHashField = [&](const string & s) { + try { + return parseHash(s); + } catch (BadHash &) { + corrupt(); + } + }; + + size_t pos = 0; + while (pos < s.size()) { + + size_t colon = s.find(':', pos); + if (colon == std::string::npos) corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "StorePath") { + if (!isStorePath(value)) corrupt(); + path = value; + } + else if (name == "URL") + url = value; + else if (name == "Compression") + compression = value; + else if (name == "FileHash") + fileHash = parseHashField(value); + else if (name == "FileSize") { + if (!string2Int(value, fileSize)) corrupt(); + } + else if (name == "NarHash") + narHash = parseHashField(value); + else if (name == "NarSize") { + if (!string2Int(value, narSize)) corrupt(); + } + else if (name == "References") { + auto refs = tokenizeString<Strings>(value, " "); + if (!references.empty()) corrupt(); + for (auto & r : refs) { + auto r2 = settings.nixStore + "/" + r; + if (!isStorePath(r2)) corrupt(); + references.insert(r2); + } + } + else if (name == "Deriver") { + auto p = settings.nixStore + "/" + value; + if (!isStorePath(p)) corrupt(); + deriver = p; + } + else if (name == "System") + system = value; + else if (name == "Sig") + sigs.insert(value); + + pos = eol + 1; + } + + if (compression == "") compression = "bzip2"; + + if (path.empty() || url.empty()) corrupt(); +} + +std::string NarInfo::to_string() const +{ + std::string res; + res += "StorePath: " + path + "\n"; + res += "URL: " + url + "\n"; + assert(compression != ""); + res += "Compression: " + compression + "\n"; + assert(fileHash.type == htSHA256); + res += "FileHash: sha256:" + printHash32(fileHash) + "\n"; + res += "FileSize: " + std::to_string(fileSize) + "\n"; + assert(narHash.type == htSHA256); + res += "NarHash: sha256:" + printHash32(narHash) + "\n"; + res += "NarSize: " + std::to_string(narSize) + "\n"; + + res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + + if (!deriver.empty()) + res += "Deriver: " + baseNameOf(deriver) + "\n"; + + if (!system.empty()) + res += "System: " + system + "\n"; + + for (auto sig : sigs) + res += "Sig: " + sig + "\n"; + + return res; +} + +} diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh new file mode 100644 index 000000000000..6bc2f03b139b --- /dev/null +++ b/src/libstore/nar-info.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "store-api.hh" + +namespace nix { + +struct NarInfo : ValidPathInfo +{ + std::string url; + std::string compression; + Hash fileHash; + uint64_t fileSize = 0; + std::string system; + + NarInfo() { } + NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } + NarInfo(const std::string & s, const std::string & whence); + + std::string to_string() const; +}; + +} diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 55c252b9b2e3..ad7fe0e8bebf 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -99,8 +99,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa if (S_ISDIR(st.st_mode)) { Strings names = readDirectoryIgnoringInodes(path, inodeHash); - foreach (Strings::iterator, i, names) - optimisePath_(stats, path + "/" + *i, inodeHash); + for (auto & i : names) + optimisePath_(stats, path + "/" + i, inodeHash); return; } @@ -120,9 +120,9 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa return; } - /* This can still happen on top-level files */ + /* This can still happen on top-level files. */ if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - printMsg(lvlDebug, format("‘%1%’ is already linked, with %2% other file(s).") % path % (st.st_nlink - 2)); + printMsg(lvlDebug, format("‘%1%’ is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); return; } @@ -141,6 +141,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + printHash32(hash); + retry: if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ if (link(path.c_str(), linkPath.c_str()) == 0) { @@ -164,6 +165,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa return; } + if (st.st_size != stLink.st_size) { + printMsg(lvlError, format("removing corrupted link ‘%1%’") % linkPath); + unlink(linkPath.c_str()); + goto retry; + } + printMsg(lvlTalkative, format("linking ‘%1%’ to ‘%2%’") % path % linkPath); /* Make the containing directory writable, but only if it's not @@ -218,11 +225,11 @@ void LocalStore::optimiseStore(OptimiseStats & stats) PathSet paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); - foreach (PathSet::iterator, i, paths) { - addTempRoot(*i); - if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ - startNest(nest, lvlChatty, format("hashing files in ‘%1%’") % *i); - optimisePath_(stats, *i, inodeHash); + for (auto & i : paths) { + addTempRoot(i); + if (!isValidPath(i)) continue; /* path was GC'ed, probably */ + Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); + optimisePath_(stats, i, inodeHash); } } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 9db37e8f9aaa..eddf5bcbda65 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait) while (fcntl(fd, F_SETLK, &lock) != 0) { checkInterrupt(); if (errno == EACCES || errno == EAGAIN) return false; - if (errno != EINTR) + if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } } @@ -94,7 +94,7 @@ bool PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg, bool wait) { assert(fds.empty()); - + /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ @@ -102,11 +102,10 @@ bool PathLocks::lockPaths(const PathSet & _paths, the same order, thus preventing deadlocks. */ Paths paths(_paths.begin(), _paths.end()); paths.sort(); - + /* Acquire the lock for each path. */ - foreach (Paths::iterator, i, paths) { + for (auto & path : paths) { checkInterrupt(); - Path path = *i; Path lockPath = path + ".lock"; debug(format("locking path ‘%1%’") % path); @@ -115,11 +114,11 @@ bool PathLocks::lockPaths(const PathSet & _paths, throw Error("deadlock: trying to re-acquire self-held lock"); AutoCloseFD fd; - + while (1) { /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + fd = openLockFile(lockPath, true); /* Acquire an exclusive lock. */ if (!lockFile(fd, ltWrite, false)) { @@ -162,21 +161,25 @@ bool PathLocks::lockPaths(const PathSet & _paths, PathLocks::~PathLocks() { - unlock(); + try { + unlock(); + } catch (...) { + ignoreException(); + } } void PathLocks::unlock() { - foreach (list<FDPair>::iterator, i, fds) { - if (deletePaths) deleteLockFile(i->second, i->first); + for (auto & i : fds) { + if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths.erase(i->second); - if (close(i->first) == -1) + lockedPaths.erase(i.second); + if (close(i.first) == -1) printMsg(lvlError, - format("error (ignored): cannot close lock file on ‘%1%’") % i->second); + format("error (ignored): cannot close lock file on ‘%1%’") % i.second); - debug(format("lock released on ‘%1%’") % i->second); + debug(format("lock released on ‘%1%’") % i.second); } fds.clear(); @@ -195,5 +198,5 @@ bool pathIsLockedByMe(const Path & path) return lockedPaths.find(lockPath) != lockedPaths.end(); } - + } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index da3f7da9d19d..18e3bcbec4a3 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -74,7 +74,7 @@ static void makeName(const Path & profile, unsigned int num, } -Path createGeneration(Path profile, Path outPath) +Path createGeneration(ref<Store> store, Path profile, Path outPath) { /* The new generation number should be higher than old the previous ones. */ @@ -108,7 +108,7 @@ Path createGeneration(Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - addPermRoot(*store, outPath, generation, false, true); + store->addPermRoot(outPath, generation, false, true); return generation; } @@ -222,8 +222,7 @@ void switchLink(Path link, Path target) void lockProfile(PathLocks & lock, const Path & profile) { - lock.lockPaths(singleton<PathSet>(profile), - (format("waiting for lock on profile ‘%1%’") % profile).str()); + lock.lockPaths({profile}, (format("waiting for lock on profile ‘%1%’") % profile).str()); lock.setDeletion(true); } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index e99bbf398a86..d758d94b603c 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -31,7 +31,9 @@ typedef list<Generation> Generations; profile, sorted by generation number. */ Generations findGenerations(Path profile, int & curGen); -Path createGeneration(Path profile, Path outPath); +class Store; + +Path createGeneration(ref<Store> store, Path profile, Path outPath); void deleteGeneration(const Path & profile, unsigned int gen); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 521244a31377..33eab5a240b5 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -13,7 +13,7 @@ namespace nix { static unsigned int refLength = 32; /* characters */ -static void search(const unsigned char * s, unsigned int len, +static void search(const unsigned char * s, unsigned int len, StringSet & hashes, StringSet & seen) { static bool initialised = false; @@ -24,7 +24,7 @@ static void search(const unsigned char * s, unsigned int len, isBase32[(unsigned char) base32Chars[i]] = true; initialised = true; } - + for (unsigned int i = 0; i + refLength <= len; ) { int j; bool match = true; @@ -56,7 +56,7 @@ struct RefScanSink : Sink string tail; RefScanSink() : hashSink(htSHA256) { } - + void operator () (const unsigned char * data, size_t len); }; @@ -89,17 +89,17 @@ PathSet scanForReferences(const string & path, /* For efficiency (and a higher hit rate), just search for the hash part of the file name. (This assumes that all references have the form `HASH-bla'). */ - foreach (PathSet::const_iterator, i, refs) { - string baseName = baseNameOf(*i); + for (auto & i : refs) { + string baseName = baseNameOf(i); string::size_type pos = baseName.find('-'); if (pos == string::npos) - throw Error(format("bad reference ‘%1%’") % *i); + throw Error(format("bad reference ‘%1%’") % i); string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); // parseHash(htSHA256, s); sink.hashes.insert(s); - backMap[s] = *i; + backMap[s] = i; } /* Look for the hashes in the NAR dump of the path. */ @@ -107,14 +107,14 @@ PathSet scanForReferences(const string & path, /* Map the hashes found back to their store paths. */ PathSet found; - foreach (StringSet::iterator, i, sink.seen) { + for (auto & i : sink.seen) { std::map<string, Path>::iterator j; - if ((j = backMap.find(*i)) == backMap.end()) abort(); + if ((j = backMap.find(i)) == backMap.end()) abort(); found.insert(j->second); } hash = sink.hashSink.finish(); - + return found; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ab87d9d8b16f..9a00a6ed9910 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -5,6 +5,8 @@ #include "archive.hh" #include "affinity.hh" #include "globals.hh" +#include "derivations.hh" +#include "pool.hh" #include <sys/types.h> #include <sys/stat.h> @@ -12,9 +14,8 @@ #include <sys/un.h> #include <errno.h> #include <fcntl.h> - -#include <iostream> #include <unistd.h> + #include <cstring> namespace nix { @@ -31,205 +32,171 @@ Path readStorePath(Source & from) template<class T> T readStorePaths(Source & from) { T paths = readStrings<T>(from); - foreach (typename T::iterator, i, paths) assertStorePath(*i); + for (auto & i : paths) assertStorePath(i); return paths; } template PathSet readStorePaths(Source & from); -RemoteStore::RemoteStore() +RemoteStore::RemoteStore(size_t maxConnections) + : connections(make_ref<Pool<Connection>>( + maxConnections, + [this]() { return openConnection(); }, + [](const ref<Connection> & r) { return r->to.good() && r->from.good(); } + )) +{ +} + + +std::string RemoteStore::getUri() { - initialised = false; + return "daemon"; } -void RemoteStore::openConnection(bool reserveSpace) +ref<RemoteStore::Connection> RemoteStore::openConnection() { - if (initialised) return; - initialised = true; + auto conn = make_ref<Connection>(); - string remoteMode = getEnv("NIX_REMOTE"); + /* Connect to a daemon that does the privileged work for us. */ + conn->fd = socket(PF_UNIX, SOCK_STREAM, 0); + if (conn->fd == -1) + throw SysError("cannot create Unix domain socket"); + closeOnExec(conn->fd); + + string socketPath = settings.nixDaemonSocketFile; - if (remoteMode == "daemon") - /* Connect to a daemon that does the privileged work for - us. */ - connectToDaemon(); - else - throw Error(format("invalid setting for NIX_REMOTE, ‘%1%’") % remoteMode); + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket path ‘%1%’ is too long") % socketPath); + strcpy(addr.sun_path, socketPath.c_str()); - from.fd = fdSocket; - to.fd = fdSocket; + if (connect(conn->fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); + + conn->from.fd = conn->fd; + conn->to.fd = conn->fd; /* Send the magic greeting, check for the reply. */ try { - writeInt(WORKER_MAGIC_1, to); - to.flush(); - unsigned int magic = readInt(from); + conn->to << WORKER_MAGIC_1; + conn->to.flush(); + unsigned int magic = readInt(conn->from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - daemonVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + conn->daemonVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); - writeInt(PROTOCOL_VERSION, to); + conn->to << PROTOCOL_VERSION; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 14) { int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) { - writeInt(1, to); - writeInt(cpu, to); - } else - writeInt(0, to); + if (cpu != -1) + conn->to << 1 << cpu; + else + conn->to << 0; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) - writeInt(reserveSpace, to); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 11) + conn->to << false; - processStderr(); + conn->processStderr(); } catch (Error & e) { throw Error(format("cannot start daemon worker: %1%") % e.msg()); } - setOptions(); -} - - -void RemoteStore::connectToDaemon() -{ - fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); - if (fdSocket == -1) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket); - - string socketPath = settings.nixDaemonSocketFile; - - /* Urgh, sockaddr_un allows path names of only 108 characters. So - chdir to the socket directory so that we can pass a relative - path name. !!! this is probably a bad idea in multi-threaded - applications... */ - AutoCloseFD fdPrevDir = open(".", O_RDONLY); - if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - if (chdir(dirOf(socketPath).c_str()) == -1) throw SysError(format("couldn't change to directory of ‘%1%’") % socketPath); - Path socketPathRel = "./" + baseNameOf(socketPath); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path ‘%1%’ is too long") % socketPathRel); - using namespace std; - strcpy(addr.sun_path, socketPathRel.c_str()); - - if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); + setOptions(conn); - if (fchdir(fdPrevDir) == -1) - throw SysError("couldn't change back to previous directory"); + return conn; } -RemoteStore::~RemoteStore() +void RemoteStore::setOptions(ref<Connection> conn) { - try { - to.flush(); - fdSocket.close(); - } catch (...) { - ignoreException(); - } -} - + conn->to << wopSetOptions + << settings.keepFailed + << settings.keepGoing + << settings.tryFallback + << verbosity + << settings.maxBuildJobs + << settings.maxSilentTime; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 2) + conn->to << settings.useBuildHook; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 4) + conn->to << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 6) + conn->to << settings.buildCores; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 10) + conn->to << settings.useSubstitutes; -void RemoteStore::setOptions() -{ - writeInt(wopSetOptions, to); - - writeInt(settings.keepFailed, to); - writeInt(settings.keepGoing, to); - writeInt(settings.tryFallback, to); - writeInt(verbosity, to); - writeInt(settings.maxBuildJobs, to); - writeInt(settings.maxSilentTime, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 2) - writeInt(settings.useBuildHook, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) { - writeInt(settings.buildVerbosity, to); - writeInt(logType, to); - writeInt(settings.printBuildTrace, to); - } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) - writeInt(settings.buildCores, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 10) - writeInt(settings.useSubstitutes, to); - - if (GET_PROTOCOL_MINOR(daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 12) { Settings::SettingsMap overrides = settings.getOverrides(); if (overrides["ssh-auth-sock"] == "") overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK"); - writeInt(overrides.size(), to); - foreach (Settings::SettingsMap::iterator, i, overrides) { - writeString(i->first, to); - writeString(i->second, to); - } + conn->to << overrides.size(); + for (auto & i : overrides) + conn->to << i.first << i.second; } - processStderr(); + conn->processStderr(); } -bool RemoteStore::isValidPath(const Path & path) +bool RemoteStore::isValidPathUncached(const Path & path) { - openConnection(); - writeInt(wopIsValidPath, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); + auto conn(connections->get()); + conn->to << wopIsValidPath << path; + conn->processStderr(); + unsigned int reply = readInt(conn->from); return reply != 0; } PathSet RemoteStore::queryValidPaths(const PathSet & paths) { - openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + auto conn(connections->get()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; - foreach (PathSet::const_iterator, i, paths) - if (isValidPath(*i)) res.insert(*i); + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); return res; } else { - writeInt(wopQueryValidPaths, to); - writeStrings(paths, to); - processStderr(); - return readStorePaths<PathSet>(from); + conn->to << wopQueryValidPaths << paths; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } } PathSet RemoteStore::queryAllValidPaths() { - openConnection(); - writeInt(wopQueryAllValidPaths, to); - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryAllValidPaths; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { - openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + auto conn(connections->get()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; - foreach (PathSet::const_iterator, i, paths) { - writeInt(wopHasSubstitutes, to); - writeString(*i, to); - processStderr(); - if (readInt(from)) res.insert(*i); + for (auto & i : paths) { + conn->to << wopHasSubstitutes << i; + conn->processStderr(); + if (readInt(conn->from)) res.insert(i); } return res; } else { - writeInt(wopQuerySubstitutablePaths, to); - writeStrings(paths, to); - processStderr(); - return readStorePaths<PathSet>(from); + conn->to << wopQuerySubstitutablePaths << paths; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } } @@ -239,187 +206,163 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, { if (paths.empty()) return; - openConnection(); + auto conn(connections->get()); - if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 3) return; - if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - foreach (PathSet::const_iterator, i, paths) { + for (auto & i : paths) { SubstitutablePathInfo info; - writeInt(wopQuerySubstitutablePathInfo, to); - writeString(*i, to); - processStderr(); - unsigned int reply = readInt(from); + conn->to << wopQuerySubstitutablePathInfo << i; + conn->processStderr(); + unsigned int reply = readInt(conn->from); if (reply == 0) continue; - info.deriver = readString(from); + info.deriver = readString(conn->from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; - infos[*i] = info; + info.references = readStorePaths<PathSet>(conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 7 ? readLongLong(conn->from) : 0; + infos[i] = info; } } else { - writeInt(wopQuerySubstitutablePathInfos, to); - writeStrings(paths, to); - processStderr(); - unsigned int count = readInt(from); + conn->to << wopQuerySubstitutablePathInfos << paths; + conn->processStderr(); + unsigned int count = readInt(conn->from); for (unsigned int n = 0; n < count; n++) { - Path path = readStorePath(from); + Path path = readStorePath(conn->from); SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(from); + info.deriver = readString(conn->from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = readLongLong(from); + info.references = readStorePaths<PathSet>(conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); } } } -ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +std::shared_ptr<ValidPathInfo> RemoteStore::queryPathInfoUncached(const Path & path) { - openConnection(); - writeInt(wopQueryPathInfo, to); - writeString(path, to); - processStderr(); - ValidPathInfo info; - info.path = path; - info.deriver = readString(from); - if (info.deriver != "") assertStorePath(info.deriver); - info.hash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths<PathSet>(from); - info.registrationTime = readInt(from); - info.narSize = readLongLong(from); + auto conn(connections->get()); + conn->to << wopQueryPathInfo << path; + try { + conn->processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.what()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid = readInt(conn->from) != 0; + if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path); + } + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = readString(conn->from); + if (info->deriver != "") assertStorePath(info->deriver); + info->narHash = parseHash(htSHA256, readString(conn->from)); + info->references = readStorePaths<PathSet>(conn->from); + info->registrationTime = readInt(conn->from); + info->narSize = readLongLong(conn->from); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + info->ultimate = readInt(conn->from) != 0; + info->sigs = readStrings<StringSet>(conn->from); + } return info; } -Hash RemoteStore::queryPathHash(const Path & path) -{ - openConnection(); - writeInt(wopQueryPathHash, to); - writeString(path, to); - processStderr(); - string hash = readString(from); - return parseHash(htSHA256, hash); -} - - -void RemoteStore::queryReferences(const Path & path, - PathSet & references) -{ - openConnection(); - writeInt(wopQueryReferences, to); - writeString(path, to); - processStderr(); - PathSet references2 = readStorePaths<PathSet>(from); - references.insert(references2.begin(), references2.end()); -} - - void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) { - openConnection(); - writeInt(wopQueryReferrers, to); - writeString(path, to); - processStderr(); - PathSet referrers2 = readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryReferrers << path; + conn->processStderr(); + PathSet referrers2 = readStorePaths<PathSet>(conn->from); referrers.insert(referrers2.begin(), referrers2.end()); } -Path RemoteStore::queryDeriver(const Path & path) -{ - openConnection(); - writeInt(wopQueryDeriver, to); - writeString(path, to); - processStderr(); - Path drvPath = readString(from); - if (drvPath != "") assertStorePath(drvPath); - return drvPath; -} - - PathSet RemoteStore::queryValidDerivers(const Path & path) { - openConnection(); - writeInt(wopQueryValidDerivers, to); - writeString(path, to); - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryValidDerivers << path; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - openConnection(); - writeInt(wopQueryDerivationOutputs, to); - writeString(path, to); - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryDerivationOutputs << path; + conn->processStderr(); + return readStorePaths<PathSet>(conn->from); } PathSet RemoteStore::queryDerivationOutputNames(const Path & path) { - openConnection(); - writeInt(wopQueryDerivationOutputNames, to); - writeString(path, to); - processStderr(); - return readStrings<PathSet>(from); + auto conn(connections->get()); + conn->to << wopQueryDerivationOutputNames << path; + conn->processStderr(); + return readStrings<PathSet>(conn->from); } Path RemoteStore::queryPathFromHashPart(const string & hashPart) { - openConnection(); - writeInt(wopQueryPathFromHashPart, to); - writeString(hashPart, to); - processStderr(); - Path path = readString(from); + auto conn(connections->get()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn->processStderr(); + Path path = readString(conn->from); if (!path.empty()) assertStorePath(path); return path; } +void RemoteStore::addToStore(const ValidPathInfo & info, const std::string & nar, bool repair) +{ + throw Error("RemoteStore::addToStore() not implemented"); +} + + Path RemoteStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - openConnection(); + auto conn(connections->get()); Path srcPath(absPath(_srcPath)); - writeInt(wopAddToStore, to); - writeString(name, to); - /* backwards compatibility hack */ - writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); - writeInt(recursive ? 1 : 0, to); - writeString(printHashType(hashAlgo), to); + conn->to << wopAddToStore << name + << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ + << (recursive ? 1 : 0) + << printHashType(hashAlgo); try { - to.written = 0; - to.warn = true; - dumpPath(srcPath, to, filter); - to.warn = false; - processStderr(); + conn->to.written = 0; + conn->to.warn = true; + dumpPath(srcPath, conn->to, filter); + conn->to.warn = false; + conn->processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ if (e.errNo == EPIPE) try { - processStderr(); + conn->processStderr(); } catch (EndOfFile & e) { } throw; } - return readStorePath(from); + return readStorePath(conn->from); } @@ -428,109 +371,100 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - openConnection(); - writeInt(wopAddTextToStore, to); - writeString(name, to); - writeString(s, to); - writeStrings(references, to); - - processStderr(); - return readStorePath(from); -} - + auto conn(connections->get()); + conn->to << wopAddTextToStore << name << s << references; -void RemoteStore::exportPath(const Path & path, bool sign, - Sink & sink) -{ - openConnection(); - writeInt(wopExportPath, to); - writeString(path, to); - writeInt(sign ? 1 : 0, to); - processStderr(&sink); /* sink receives the actual data */ - readInt(from); -} - - -Paths RemoteStore::importPaths(bool requireSignature, Source & source) -{ - openConnection(); - writeInt(wopImportPaths, to); - /* We ignore requireSignature, since the worker forces it to true - anyway. */ - processStderr(0, &source); - return readStorePaths<Paths>(from); + conn->processStderr(); + return readStorePath(conn->from); } void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - if (buildMode != bmNormal) throw Error("repairing or checking is not supported when building through the Nix daemon"); - openConnection(); - writeInt(wopBuildPaths, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 13) - writeStrings(drvPaths, to); - else { + auto conn(connections->get()); + conn->to << wopBuildPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { + conn->to << drvPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); + } else { /* For backwards compatibility with old daemons, strip output identifiers. */ PathSet drvPaths2; - foreach (PathSet::const_iterator, i, drvPaths) - drvPaths2.insert(string(*i, 0, i->find('!'))); - writeStrings(drvPaths2, to); + for (auto & i : drvPaths) + drvPaths2.insert(string(i, 0, i.find('!'))); + conn->to << drvPaths2; } - processStderr(); - readInt(from); + conn->processStderr(); + readInt(conn->from); +} + + +BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(connections->get()); + conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn->processStderr(); + BuildResult res; + unsigned int status; + conn->from >> status >> res.errorMsg; + res.status = (BuildResult::Status) status; + return res; } void RemoteStore::ensurePath(const Path & path) { - openConnection(); - writeInt(wopEnsurePath, to); - writeString(path, to); - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopEnsurePath << path; + conn->processStderr(); + readInt(conn->from); } void RemoteStore::addTempRoot(const Path & path) { - openConnection(); - writeInt(wopAddTempRoot, to); - writeString(path, to); - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopAddTempRoot << path; + conn->processStderr(); + readInt(conn->from); } void RemoteStore::addIndirectRoot(const Path & path) { - openConnection(); - writeInt(wopAddIndirectRoot, to); - writeString(path, to); - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopAddIndirectRoot << path; + conn->processStderr(); + readInt(conn->from); } void RemoteStore::syncWithGC() { - openConnection(); - writeInt(wopSyncWithGC, to); - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopSyncWithGC; + conn->processStderr(); + readInt(conn->from); } Roots RemoteStore::findRoots() { - openConnection(); - writeInt(wopFindRoots, to); - processStderr(); - unsigned int count = readInt(from); + auto conn(connections->get()); + conn->to << wopFindRoots; + conn->processStderr(); + unsigned int count = readInt(conn->from); Roots result; while (count--) { - Path link = readString(from); - Path target = readStorePath(from); + Path link = readString(conn->from); + Path target = readStorePath(conn->from); result[link] = target; } return result; @@ -539,65 +473,66 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { - openConnection(false); + auto conn(connections->get()); - writeInt(wopCollectGarbage, to); - writeInt(options.action, to); - writeStrings(options.pathsToDelete, to); - writeInt(options.ignoreLiveness, to); - writeLongLong(options.maxFreed, to); - writeInt(0, to); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) { + conn->to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness + << options.maxFreed << 0; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 5) /* removed options */ - writeInt(0, to); - writeInt(0, to); - } + conn->to << 0 << 0; - processStderr(); + conn->processStderr(); - results.paths = readStrings<PathSet>(from); - results.bytesFreed = readLongLong(from); - readLongLong(from); // obsolete + results.paths = readStrings<PathSet>(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } } -PathSet RemoteStore::queryFailedPaths() +void RemoteStore::optimiseStore() { - openConnection(); - writeInt(wopQueryFailedPaths, to); - processStderr(); - return readStorePaths<PathSet>(from); + auto conn(connections->get()); + conn->to << wopOptimiseStore; + conn->processStderr(); + readInt(conn->from); } -void RemoteStore::clearFailedPaths(const PathSet & paths) +bool RemoteStore::verifyStore(bool checkContents, bool repair) { - openConnection(); - writeInt(wopClearFailedPaths, to); - writeStrings(paths, to); - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopVerifyStore << checkContents << repair; + conn->processStderr(); + return readInt(conn->from) != 0; } -void RemoteStore::optimiseStore() + +void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) { - openConnection(); - writeInt(wopOptimiseStore, to); - processStderr(); - readInt(from); + auto conn(connections->get()); + conn->to << wopAddSignatures << storePath << sigs; + conn->processStderr(); + readInt(conn->from); } -bool RemoteStore::verifyStore(bool checkContents, bool repair) + +RemoteStore::Connection::~Connection() { - openConnection(); - writeInt(wopVerifyStore, to); - writeInt(checkContents, to); - writeInt(repair, to); - processStderr(); - return readInt(from) != 0; + try { + to.flush(); + fd.close(); + } catch (...) { + ignoreException(); + } } -void RemoteStore::processStderr(Sink * sink, Source * source) + +void RemoteStore::Connection::processStderr(Sink * sink, Source * source) { to.flush(); unsigned int msg; @@ -606,7 +541,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) if (msg == STDERR_WRITE) { string s = readString(from); if (!sink) throw Error("no sink"); - (*sink)((const unsigned char *) s.data(), s.size()); + (*sink)(s); } else if (msg == STDERR_READ) { if (!source) throw Error("no source"); @@ -616,10 +551,8 @@ void RemoteStore::processStderr(Sink * sink, Source * source) writeString(buf, source->read(buf, len), to); to.flush(); } - else { - string s = readString(from); - writeToStderr(s); - } + else + printMsg(lvlError, chomp(readString(from))); } if (msg == STDERR_ERROR) { string error = readString(from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 030120db4067..0757f82e8964 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,5 +1,6 @@ #pragma once +#include <limits> #include <string> #include "store-api.hh" @@ -12,94 +13,96 @@ class Pipe; class Pid; struct FdSink; struct FdSource; +template<typename T> class Pool; -class RemoteStore : public StoreAPI +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public LocalFSStore { public: - RemoteStore(); - - ~RemoteStore(); + RemoteStore(size_t maxConnections = std::numeric_limits<size_t>::max()); /* Implementations of abstract store API methods. */ - bool isValidPath(const Path & path); - - PathSet queryValidPaths(const PathSet & paths); + std::string getUri() override; - PathSet queryAllValidPaths(); + bool isValidPathUncached(const Path & path) override; - ValidPathInfo queryPathInfo(const Path & path); + PathSet queryValidPaths(const PathSet & paths) override; - Hash queryPathHash(const Path & path); + PathSet queryAllValidPaths() override; - void queryReferences(const Path & path, PathSet & references); + std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override; - void queryReferrers(const Path & path, PathSet & referrers); + void queryReferrers(const Path & path, PathSet & referrers) override; - Path queryDeriver(const Path & path); + PathSet queryValidDerivers(const Path & path) override; - PathSet queryValidDerivers(const Path & path); + PathSet queryDerivationOutputs(const Path & path) override; - PathSet queryDerivationOutputs(const Path & path); + StringSet queryDerivationOutputNames(const Path & path) override; - StringSet queryDerivationOutputNames(const Path & path); + Path queryPathFromHashPart(const string & hashPart) override; - Path queryPathFromHashPart(const string & hashPart); - - PathSet querySubstitutablePaths(const PathSet & paths); + PathSet querySubstitutablePaths(const PathSet & paths) override; void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos); + SubstitutablePathInfos & infos) override; + + void addToStore(const ValidPathInfo & info, const std::string & nar, + bool repair) override; Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false); + PathFilter & filter = defaultPathFilter, bool repair = false) override; Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false); + const PathSet & references, bool repair = false) override; - void exportPath(const Path & path, bool sign, - Sink & sink); + void buildPaths(const PathSet & paths, BuildMode buildMode) override; - Paths importPaths(bool requireSignature, Source & source); + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; - void buildPaths(const PathSet & paths, BuildMode buildMode); + void ensurePath(const Path & path) override; - void ensurePath(const Path & path); + void addTempRoot(const Path & path) override; - void addTempRoot(const Path & path); + void addIndirectRoot(const Path & path) override; - void addIndirectRoot(const Path & path); + void syncWithGC() override; - void syncWithGC(); + Roots findRoots() override; - Roots findRoots(); + void collectGarbage(const GCOptions & options, GCResults & results) override; - void collectGarbage(const GCOptions & options, GCResults & results); + void optimiseStore() override; - PathSet queryFailedPaths(); + bool verifyStore(bool checkContents, bool repair) override; - void clearFailedPaths(const PathSet & paths); + void addSignatures(const Path & storePath, const StringSet & sigs) override; - void optimiseStore(); - - bool verifyStore(bool checkContents, bool repair); private: - AutoCloseFD fdSocket; - FdSink to; - FdSource from; - unsigned int daemonVersion; - bool initialised; - void openConnection(bool reserveSpace = true); + struct Connection + { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + + ~Connection(); + + void processStderr(Sink * sink = 0, Source * source = 0); + }; - void processStderr(Sink * sink = 0, Source * source = 0); + ref<Pool<Connection>> connections; - void connectToDaemon(); + ref<Connection> openConnection(); - void setOptions(); + void setOptions(ref<Connection> conn); }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000000..6ee27b48d61d --- /dev/null +++ b/src/libstore/s3-binary-cache-store.cc @@ -0,0 +1,258 @@ +#include "config.h" + +#if ENABLE_S3 + +#include "s3-binary-cache-store.hh" +#include "nar-info.hh" +#include "nar-info-disk-cache.hh" +#include "globals.hh" + +#include <aws/core/client/ClientConfiguration.h> +#include <aws/s3/S3Client.h> +#include <aws/s3/model/CreateBucketRequest.h> +#include <aws/s3/model/GetBucketLocationRequest.h> +#include <aws/s3/model/GetObjectRequest.h> +#include <aws/s3/model/HeadObjectRequest.h> +#include <aws/s3/model/PutObjectRequest.h> +#include <aws/s3/model/ListObjectsRequest.h> + +namespace nix { + +struct S3Error : public Error +{ + Aws::S3::S3Errors err; + S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) + : Error(fs), err(err) { }; +}; + +/* Helper: given an Outcome<R, E>, return R in case of success, or + throw an exception in case of an error. */ +template<typename R, typename E> +R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) +{ + if (!outcome.IsSuccess()) + throw S3Error( + outcome.GetError().GetErrorType(), + fs.s + ": " + outcome.GetError().GetMessage()); + return outcome.GetResultWithOwnership(); +} + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +{ + std::string bucketName; + + ref<Aws::Client::ClientConfiguration> config; + ref<Aws::S3::S3Client> client; + + Stats stats; + + S3BinaryCacheStoreImpl( + const StoreParams & params, const std::string & bucketName) + : S3BinaryCacheStore(params) + , bucketName(bucketName) + , config(makeConfig()) + , client(make_ref<Aws::S3::S3Client>(*config)) + { + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() + { + return "s3://" + bucketName; + } + + ref<Aws::Client::ClientConfiguration> makeConfig() + { + auto res = make_ref<Aws::Client::ClientConfiguration>(); + res->region = Aws::Region::US_EAST_1; // FIXME: make configurable + res->requestTimeoutMs = 600 * 1000; + return res; + } + + void init() + { + if (!diskCache->cacheExists(getUri())) { + + /* Create the bucket if it doesn't already exists. */ + // FIXME: HeadBucket would be more appropriate, but doesn't return + // an easily parsed 404 message. + auto res = client->GetBucketLocation( + Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); + + if (!res.IsSuccess()) { + if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) + throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage()); + + checkAws(format("AWS error creating bucket ‘%s’") % bucketName, + client->CreateBucket( + Aws::S3::Model::CreateBucketRequest() + .WithBucket(bucketName) + .WithCreateBucketConfiguration( + Aws::S3::Model::CreateBucketConfiguration() + /* .WithLocationConstraint( + Aws::S3::Model::BucketLocationConstraint::US) */ ))); + } + + BinaryCacheStore::init(); + + diskCache->createCache(getUri(), wantMassQuery_, priority); + } + } + + const Stats & getS3Stats() + { + return stats; + } + + /* This is a specialisation of isValidPath() that optimistically + fetches the .narinfo file, rather than first checking for its + existence via a HEAD request. Since .narinfos are small, doing + a GET is unlikely to be slower than HEAD. */ + bool isValidPathUncached(const Path & storePath) + { + try { + queryPathInfo(storePath); + return true; + } catch (InvalidPath & e) { + return false; + } + } + + bool fileExists(const std::string & path) + { + stats.head++; + + auto res = client->HeadObject( + Aws::S3::Model::HeadObjectRequest() + .WithBucket(bucketName) + .WithKey(path)); + + if (!res.IsSuccess()) { + auto & error = res.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME + && error.GetMessage().find("404") != std::string::npos) + return false; + throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage()); + } + + return true; + } + + void upsertFile(const std::string & path, const std::string & data) + { + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + auto stream = std::make_shared<std::stringstream>(data); + + request.SetBody(stream); + + stats.put++; + stats.putBytes += data.size(); + + auto now1 = std::chrono::steady_clock::now(); + + auto result = checkAws(format("AWS error uploading ‘%s’") % path, + client->PutObject(request)); + + auto now2 = std::chrono::steady_clock::now(); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + + printMsg(lvlInfo, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") + % bucketName % path % data.size() % duration); + + stats.putTimeMs += duration; + } + + std::shared_ptr<std::string> getFile(const std::string & path) + { + debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path); + + auto request = + Aws::S3::Model::GetObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetResponseStreamFactory([&]() { + return Aws::New<std::stringstream>("STRINGSTREAM"); + }); + + stats.get++; + + try { + + auto now1 = std::chrono::steady_clock::now(); + + auto result = checkAws(format("AWS error fetching ‘%s’") % path, + client->GetObject(request)); + + auto now2 = std::chrono::steady_clock::now(); + + auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str(); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + + printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") + % bucketName % path % res.size() % duration); + + stats.getBytes += res.size(); + stats.getTimeMs += duration; + + return std::make_shared<std::string>(res); + + } catch (S3Error & e) { + if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return 0; + throw; + } + } + + PathSet queryAllValidPaths() override + { + PathSet paths; + std::string marker; + + do { + debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker); + + auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName, + client->ListObjects( + Aws::S3::Model::ListObjectsRequest() + .WithBucket(bucketName) + .WithDelimiter("/") + .WithMarker(marker))); + + auto & contents = res.GetContents(); + + debug(format("got %d keys, next marker ‘%s’") + % contents.size() % res.GetNextMarker()); + + for (auto object : contents) { + auto & key = object.GetKey(); + if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; + paths.insert(settings.nixStore + "/" + key.substr(0, key.size() - 8)); + } + + marker = res.GetNextMarker(); + } while (!marker.empty()); + + return paths; + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, 5) != "s3://") return 0; + auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); + store->init(); + return store; +}); + +} + +#endif diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh new file mode 100644 index 000000000000..3f9bd891274b --- /dev/null +++ b/src/libstore/s3-binary-cache-store.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "binary-cache-store.hh" + +#include <atomic> + +namespace nix { + +class S3BinaryCacheStore : public BinaryCacheStore +{ +protected: + + S3BinaryCacheStore(const StoreParams & params) + : BinaryCacheStore(params) + { } + +public: + + struct Stats + { + std::atomic<uint64_t> put{0}; + std::atomic<uint64_t> putBytes{0}; + std::atomic<uint64_t> putTimeMs{0}; + std::atomic<uint64_t> get{0}; + std::atomic<uint64_t> getBytes{0}; + std::atomic<uint64_t> getTimeMs{0}; + std::atomic<uint64_t> head{0}; + }; + + const Stats & getS3Stats(); +}; + +} diff --git a/src/libstore/sandbox-defaults.sb.in b/src/libstore/sandbox-defaults.sb.in new file mode 100644 index 000000000000..b5e80085fbe2 --- /dev/null +++ b/src/libstore/sandbox-defaults.sb.in @@ -0,0 +1,63 @@ +(allow file-read* file-write-data (literal "/dev/null")) +(allow ipc-posix*) +(allow mach-lookup (global-name "com.apple.SecurityServer")) + +(allow file-read* + (literal "/dev/dtracehelper") + (literal "/dev/tty") + (literal "/dev/autofs_nowait") + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/private/var/run/systemkeychaincheck.done") + (literal "/private/etc/protocols") + (literal "/private/var/tmp") + (literal "/private/var/db") + (subpath "/private/var/db/mds")) + +(allow file-read* + (subpath "/usr/share/icu") + (subpath "/usr/share/locale") + (subpath "/usr/share/zoneinfo")) + +(allow file-write* + (literal "/dev/tty") + (literal "/dev/dtracehelper") + (literal "/mds")) + +(allow file-ioctl (literal "/dev/dtracehelper")) + +(allow file-read-metadata + (literal "/var") + (literal "/tmp") + ; symlinks + (literal "@sysconfdir@") + (literal "@sysconfdir@/nix") + (literal "@sysconfdir@/nix/nix.conf") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private@sysconfdir@/nix/nix.conf") + (literal "/private/var/run/resolv.conf")) + +; some builders use filehandles other than stdin/stdout +(allow file* + (subpath "/dev/fd") + (literal "/dev/ptmx") + (regex #"^/dev/[pt]ty.*$")) + +; allow everything inside TMP +(allow file* process-exec + (subpath (param "_GLOBAL_TMP_DIR")) + (subpath "/private/tmp")) + +(allow process-fork) +(allow sysctl-read) +(allow signal (target same-sandbox)) + +; allow getpwuid (for git and other packages) +(allow mach-lookup + (global-name "com.apple.system.notification_center") + (global-name "com.apple.system.opendirectoryd.libinfo")) + +; allow local networking +(allow network* (local ip) (remote unix-socket)) diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index c1b4a689afcb..91878af1580d 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -4,7 +4,9 @@ create table if not exists ValidPaths ( hash text not null, registrationTime integer not null, deriver text, - narSize integer + narSize integer, + ultimate integer, -- null implies "false" + sigs text -- space-separated ); create table if not exists Refs ( @@ -37,8 +39,3 @@ create table if not exists DerivationOutputs ( ); create index if not exists IndexDerivationOutputs on DerivationOutputs(path); - -create table if not exists FailedPaths ( - path text primary key not null, - time integer not null -); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc new file mode 100644 index 000000000000..816f9984d6eb --- /dev/null +++ b/src/libstore/sqlite.cc @@ -0,0 +1,172 @@ +#include "sqlite.hh" +#include "util.hh" + +#include <sqlite3.h> + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() +{ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; +} + +int SQLiteStmt::Use::step() +{ + return sqlite3_step(stmt); +} + +void SQLiteStmt::Use::exec() +{ + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +bool SQLiteStmt::Use::isNull(int col) +{ + return sqlite3_column_type(stmt, col) == SQLITE_NULL; +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh new file mode 100644 index 000000000000..d6b4a8d9117b --- /dev/null +++ b/src/libstore/sqlite.hh @@ -0,0 +1,103 @@ +#pragma once + +#include <functional> +#include <string> + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } + void create(sqlite3 * db, const std::string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + bool isNull(int col); + }; + + Use use() + { + return Use(*this); + } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template<typename T> +T retrySQLite(std::function<T()> fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d3cbd1e7dee2..f39d6b54787c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,21 +1,13 @@ -#include "store-api.hh" +#include "crypto.hh" #include "globals.hh" +#include "store-api.hh" #include "util.hh" - -#include <climits> +#include "nar-info-disk-cache.hh" namespace nix { -GCOptions::GCOptions() -{ - action = gcDeleteDead; - ignoreLiveness = false; - maxFreed = ULLONG_MAX; -} - - bool isInStore(const Path & path) { return isInDir(path, settings.nixStore); @@ -25,6 +17,7 @@ bool isInStore(const Path & path) bool isStorePath(const Path & path) { return isInStore(path) + && path.size() >= settings.nixStore.size() + 1 + storePathHashLen && path.find('/', settings.nixStore.size() + 1) == Path::npos; } @@ -71,7 +64,17 @@ Path followLinksToStorePath(const Path & path) string storePathToName(const Path & path) { assertStorePath(path); - return string(path, settings.nixStore.size() + 34); + auto l = settings.nixStore.size() + 1 + storePathHashLen; + assert(path.size() >= l); + return path.size() == l ? "" : string(path, l + 1); +} + + +string storePathToHash(const Path & path) +{ + assertStorePath(path); + assert(path.size() >= settings.nixStore.size() + 1 + storePathHashLen); + return string(path, settings.nixStore.size() + 1, storePathHashLen); } @@ -82,14 +85,14 @@ void checkStoreName(const string & name) reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") throw Error(format("illegal name: ‘%1%’") % name); - foreach (string::const_iterator, i, name) - if (!((*i >= 'A' && *i <= 'Z') || - (*i >= 'a' && *i <= 'z') || - (*i >= '0' && *i <= '9') || - validChars.find(*i) != string::npos)) + for (auto & i : name) + if (!((i >= 'A' && i <= 'Z') || + (i >= 'a' && i <= 'z') || + (i >= '0' && i <= '9') || + validChars.find(i) != string::npos)) { throw Error(format("invalid character ‘%1%’ in name ‘%2%’") - % *i % name); + % i % name); } } @@ -101,22 +104,22 @@ void checkStoreName(const string & name) where <store> = the location of the Nix store, usually /nix/store - + <name> = a human readable name for the path, typically obtained from the name attribute of the derivation, or the name of the source file from which the store path is created. For derivation outputs other than the default "out" output, the string "-<id>" is suffixed to <name>. - + <h> = base-32 representation of the first 160 bits of a SHA-256 hash of <s>; the hash part of the store name - + <s> = the string "<type>:sha256:<h2>:<store>:<name>"; note that it includes the location of the store as well as the name to make sure that changes to either of those are reflected in the hash (e.g. you won't get /nix/store/<h>-name1 and /nix/store/<h>-name2 with equal hash parts). - + <type> = one of: "text:<r1>:<r2>:...<rN>" for plain text files written to the store using @@ -138,14 +141,14 @@ void checkStoreName(const string & name) if <type> = "source": the serialisation of the path from which this store path is copied, as returned by hashPath() - if <type> = "output:out": + if <type> = "output:<id>": for non-fixed derivation outputs: the derivation (see hashDerivationModulo() in primops.cc) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:<rec><algo>:<hash>:", where - <rec> = "r:" for recursive (path) hashes, or "" or flat + <rec> = "r:" for recursive (path) hashes, or "" for flat (file) hashes <algo> = "md5", "sha1" or "sha256" <hash> = base-16 representation of the path or flat hash of @@ -219,45 +222,153 @@ Path computeStorePathForText(const string & name, const string & s, hacky, but we can't put them in `s' since that would be ambiguous. */ string type = "text"; - foreach (PathSet::const_iterator, i, references) { + for (auto & i : references) { type += ":"; - type += *i; + type += i; } return makeStorePath(type, hash, name); } +std::string Store::getUri() +{ + return ""; +} + + +bool Store::isValidPath(const Path & storePath) +{ + auto hashPart = storePathToHash(storePath); + + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + return *res != 0; + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + return res.first == NarInfoDiskCache::oValid; + } + } + + return isValidPathUncached(storePath); + + // FIXME: insert result into NARExistence table of diskCache. +} + + +ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) +{ + auto hashPart = storePathToHash(storePath); + + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + if (!*res) + throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + return ref<ValidPathInfo>(*res); + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + if (res.first == NarInfoDiskCache::oInvalid || + (res.second->path != storePath && storePathToName(storePath) != "")) + throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + return ref<ValidPathInfo>(res.second); + } + } + + auto info = queryPathInfoUncached(storePath); + + if (diskCache && info) + diskCache->upsertNarInfo(getUri(), hashPart, info); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } + + if (!info + || (info->path != storePath && storePathToName(storePath) != "")) + { + stats.narInfoMissing++; + throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + } + + return ref<ValidPathInfo>(info); +} + + /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string StoreAPI::makeValidityRegistration(const PathSet & paths, +string Store::makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash) { string s = ""; - - foreach (PathSet::iterator, i, paths) { - s += *i + "\n"; - ValidPathInfo info = queryPathInfo(*i); + for (auto & i : paths) { + s += i + "\n"; + + auto info = queryPathInfo(i); if (showHash) { - s += printHash(info.hash) + "\n"; - s += (format("%1%\n") % info.narSize).str(); + s += printHash(info->narHash) + "\n"; + s += (format("%1%\n") % info->narSize).str(); } - Path deriver = showDerivers ? info.deriver : ""; + Path deriver = showDerivers ? info->deriver : ""; s += deriver + "\n"; - s += (format("%1%\n") % info.references.size()).str(); + s += (format("%1%\n") % info->references.size()).str(); - foreach (PathSet::iterator, j, info.references) - s += *j + "\n"; + for (auto & j : info->references) + s += j + "\n"; } return s; } +const Store::Stats & Store::getStats() +{ + { + auto state_(state.lock()); + stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } + return stats; +} + + +void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, + const Path & storePath, bool repair) +{ + auto info = srcStore->queryPathInfo(storePath); + + StringSink sink; + srcStore->narFromPath({storePath}, sink); + + dstStore->addToStore(*info, *sink.s, repair); +} + + ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) { ValidPathInfo info; @@ -266,7 +377,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) if (hashGiven) { string s; getline(str, s); - info.hash = parseHash(htSHA256, s); + info.narHash = parseHash(htSHA256, s); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } @@ -286,22 +397,55 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string showPaths(const PathSet & paths) { string s; - foreach (PathSet::const_iterator, i, paths) { + for (auto & i : paths) { if (s.size() != 0) s += ", "; - s += "‘" + *i + "’"; + s += "‘" + i + "’"; } return s; } -void exportPaths(StoreAPI & store, const Paths & paths, - bool sign, Sink & sink) +std::string ValidPathInfo::fingerprint() const { - foreach (Paths::const_iterator, i, paths) { - writeInt(1, sink); - store.exportPath(*i, sign, sink); - } - writeInt(0, sink); + if (narSize == 0 || !narHash) + throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known") + % path); + return + "1;" + path + ";" + + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", references); +} + + +void ValidPathInfo::sign(const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint())); +} + + +unsigned int ValidPathInfo::checkSignatures(const PublicKeys & publicKeys) const +{ + unsigned int good = 0; + for (auto & sig : sigs) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(baseNameOf(r)); + return refs; } @@ -309,22 +453,102 @@ void exportPaths(StoreAPI & store, const Paths & paths, #include "local-store.hh" -#include "serialise.hh" #include "remote-store.hh" namespace nix { -std::shared_ptr<StoreAPI> store; +RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -std::shared_ptr<StoreAPI> openStore(bool reserveSpace) +ref<Store> openStoreAt(const std::string & uri_) { - if (getEnv("NIX_REMOTE") == "") - return std::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); - else - return std::shared_ptr<StoreAPI>(new RemoteStore()); + auto uri(uri_); + StoreParams params; + auto q = uri.find('?'); + if (q != std::string::npos) { + for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { + auto e = s.find('='); + if (e != std::string::npos) + params[s.substr(0, e)] = s.substr(e + 1); + } + uri = uri_.substr(0, q); + } + + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri, params); + if (store) return ref<Store>(store); + } + + throw Error(format("don't know how to open Nix store ‘%s’") % uri); +} + + +ref<Store> openStore() +{ + return openStoreAt(getEnv("NIX_REMOTE")); +} + + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ + enum { mDaemon, mLocal, mAuto } mode; + + if (uri == "daemon") mode = mDaemon; + else if (uri == "local") mode = mLocal; + else if (uri == "") mode = mAuto; + else return 0; + + if (mode == mAuto) { + if (LocalStore::haveWriteAccess()) + mode = mLocal; + else if (pathExists(settings.nixDaemonSocketFile)) + mode = mDaemon; + else + mode = mLocal; + } + + return mode == mDaemon + ? std::shared_ptr<Store>(std::make_shared<RemoteStore>()) + : std::shared_ptr<Store>(std::make_shared<LocalStore>()); +}); + + +std::list<ref<Store>> getDefaultSubstituters() +{ + struct State { + bool done = false; + std::list<ref<Store>> stores; + }; + static Sync<State> state_; + + auto state(state_.lock()); + + if (state->done) return state->stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (done.count(uri)) return; + done.insert(uri); + state->stores.push_back(openStoreAt(uri)); + }; + + for (auto uri : settings.get("substituters", Strings())) + addStore(uri); + + for (auto uri : settings.get("binary-caches", Strings())) + addStore(uri); + + for (auto uri : settings.get("extra-binary-caches", Strings())) + addStore(uri); + + state->done = true; + + return state->stores; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3764f3e54242..8c618bf3e771 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,15 +2,27 @@ #include "hash.hh" #include "serialise.hh" +#include "crypto.hh" +#include "lru-cache.hh" +#include "sync.hh" -#include <string> +#include <atomic> +#include <limits> #include <map> #include <memory> +#include <string> namespace nix { +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Magic header of exportPath() output (obsolete). */ +const uint32_t exportMagic = 0x4558494e; + + typedef std::map<Path, Path> Roots; @@ -36,21 +48,19 @@ struct GCOptions gcDeleteSpecific, } GCAction; - GCAction action; + GCAction action{gcDeleteDead}; /* If `ignoreLiveness' is set, then reachability from the roots is ignored (dangerous!). However, the paths must still be unreferenced *within* the store (i.e., there can be no other store paths that depend on them). */ - bool ignoreLiveness; + bool ignoreLiveness{false}; /* For `gcDeleteSpecific', the paths to delete. */ PathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed; - - GCOptions(); + unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; }; @@ -86,55 +96,138 @@ struct ValidPathInfo { Path path; Path deriver; - Hash hash; + Hash narHash; PathSet references; - time_t registrationTime; - unsigned long long narSize; // 0 = unknown - unsigned long long id; // internal use only - ValidPathInfo() : registrationTime(0), narSize(0) { } + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it was built + locally or is content-addressable (e.g. added via addToStore() + or the result of a fixed-output derivation). */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + bool operator == (const ValidPathInfo & i) const + { + return + path == i.path + && narHash == i.narHash + && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey & secretKey); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys. */ + unsigned int checkSignatures(const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + + Strings shortRefs() const; + + virtual ~ValidPathInfo() { } }; typedef list<ValidPathInfo> ValidPathInfos; -enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash }; -class StoreAPI +struct BuildResult { + enum Status { + Built = 0, + Substituted, + AlreadyValid, + PermanentFailure, + InputRejected, + OutputRejected, + TransientFailure, // possibly transient + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + } status = MiscFailure; + std::string errorMsg; + //time_t startTime = 0, stopTime = 0; + bool success() { + return status == Built || status == Substituted || status == AlreadyValid; + } +}; + + +struct BasicDerivation; +struct Derivation; +class FSAccessor; +class NarInfoDiskCache; + + +class Store : public std::enable_shared_from_this<Store> +{ +protected: + + struct State + { + LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024}; + }; + + Sync<State> state; + + std::shared_ptr<NarInfoDiskCache> diskCache; + public: - virtual ~StoreAPI() { } + virtual ~Store() { } + + virtual std::string getUri() = 0; /* Check whether a path is valid. */ - virtual bool isValidPath(const Path & path) = 0; + bool isValidPath(const Path & path); + +protected: + + virtual bool isValidPathUncached(const Path & path) = 0; + +public: /* Query which of the given paths is valid. */ virtual PathSet queryValidPaths(const PathSet & paths) = 0; - /* Query the set of all valid paths. */ + /* Query the set of all valid paths. Note that for some store + backends, the name part of store paths may be omitted + (i.e. you'll get /nix/store/<hash> rather than + /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the + full store path. */ virtual PathSet queryAllValidPaths() = 0; - /* Query information about a valid path. */ - virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + /* Query information about a valid path. It is permitted to omit + the name part of the store path. */ + ref<const ValidPathInfo> queryPathInfo(const Path & path); - /* Query the hash of a valid path. */ - virtual Hash queryPathHash(const Path & path) = 0; +protected: - /* Query the set of outgoing FS references for a store path. The - result is not cleared. */ - virtual void queryReferences(const Path & path, - PathSet & references) = 0; + virtual std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) = 0; + +public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const Path & path, PathSet & referrers) = 0; - /* Query the deriver of a store path. Return the empty string if - no deriver has been set. */ - virtual Path queryDeriver(const Path & path) = 0; - /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the derivation that was actually used to produce `path', which may @@ -160,6 +253,12 @@ public: virtual void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) = 0; + virtual bool wantMassQuery() { return false; } + + /* Import a path into the store. */ + virtual void addToStore(const ValidPathInfo & info, const std::string & nar, + bool repair = false) = 0; + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see @@ -173,16 +272,8 @@ public: virtual Path addTextToStore(const string & name, const string & s, const PathSet & references, bool repair = false) = 0; - /* Export a store path, that is, create a NAR dump of the store - path and append its references and its deriver. Optionally, a - cryptographic signature (created by OpenSSL) of the preceding - data is attached. */ - virtual void exportPath(const Path & path, bool sign, - Sink & sink) = 0; - - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. */ - virtual Paths importPaths(bool requireSignature, Source & source) = 0; + /* Write a NAR dump of a store path. */ + virtual void narFromPath(const Path & path, Sink & sink) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -194,6 +285,12 @@ public: not derivations, substitute them. */ virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; + /* Build a single non-materialized derivation (i.e. not from an + on-disk .drv file). Note that ‘drvPath’ is only used for + informational purposes. */ + virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) = 0; + /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ @@ -210,6 +307,10 @@ public: `path' has disappeared. */ virtual void addIndirectRoot(const Path & path) = 0; + /* Register a permanent GC root. */ + Path addPermRoot(const Path & storePath, + const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + /* Acquire the global GC lock, then immediately release it. This function must be called after registering a new permanent root, but before exiting. Otherwise, it is possible that a running @@ -238,13 +339,6 @@ public: /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - /* Return the set of paths that have failed to build.*/ - virtual PathSet queryFailedPaths() = 0; - - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ - virtual void clearFailedPaths(const PathSet & paths) = 0; - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ @@ -258,6 +352,86 @@ public: /* Check the integrity of the Nix store. Returns true if errors remain. */ virtual bool verifyStore(bool checkContents, bool repair) = 0; + + /* Return an object to access files in the Nix store. */ + virtual ref<FSAccessor> getFSAccessor() = 0; + + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + + /* Utility functions. */ + + /* Read a derivation, after ensuring its existence through + ensurePath(). */ + Derivation derivationFromPath(const Path & drvPath); + + /* Place in `paths' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly + or indirectly reached from it. `paths' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ + void computeFSClosure(const Path & path, + PathSet & paths, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + + /* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths + that will be substituted. */ + void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize); + + /* Sort a set of paths topologically under the references + relation. If p refers to q, then p preceeds q in this list. */ + Paths topoSortPaths(const PathSet & paths); + + /* Export multiple paths in the format expected by ‘nix-store + --import’. */ + void exportPaths(const Paths & paths, Sink & sink); + + void exportPath(const Path & path, Sink & sink); + + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + Paths importPaths(Source & source, + std::shared_ptr<FSAccessor> accessor); + + struct Stats + { + std::atomic<uint64_t> narInfoRead{0}; + std::atomic<uint64_t> narInfoReadAverted{0}; + std::atomic<uint64_t> narInfoMissing{0}; + std::atomic<uint64_t> narInfoWrite{0}; + std::atomic<uint64_t> pathInfoCacheSize{0}; + std::atomic<uint64_t> narRead{0}; + std::atomic<uint64_t> narReadBytes{0}; + std::atomic<uint64_t> narReadCompressedBytes{0}; + std::atomic<uint64_t> narWrite{0}; + std::atomic<uint64_t> narWriteAverted{0}; + std::atomic<uint64_t> narWriteBytes{0}; + std::atomic<uint64_t> narWriteCompressedBytes{0}; + std::atomic<uint64_t> narWriteCompressionTimeMs{0}; + }; + + const Stats & getStats(); + +protected: + + Stats stats; + +}; + + +class LocalFSStore : public Store +{ +public: + void narFromPath(const Path & path, Sink & sink) override; + ref<FSAccessor> getFSAccessor() override; }; @@ -272,6 +446,9 @@ bool isStorePath(const Path & path); /* Extract the name part of the given store path. */ string storePathToName(const Path & path); +/* Extract the hash part of the given store path. */ +string storePathToHash(const Path & path); + void checkStoreName(const string & name); @@ -326,30 +503,62 @@ Path computeStorePathForText(const string & name, const string & s, const PathSet & references); +/* Copy a path from one store to another. */ +void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, + const Path & storePath, bool repair = false); + + /* Remove the temporary roots file for this process. Any temporary root becomes garbage after this point unless it has been registered as a (permanent) root. */ void removeTempRoots(); -/* Register a permanent GC root. */ -Path addPermRoot(StoreAPI & store, const Path & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); +/* Return a Store object to access the Nix store denoted by + ‘uri’ (slight misnomer...). Supported values are: + + * ‘direct’: The Nix store in /nix/store and database in + /nix/var/nix/db, accessed directly. + + * ‘daemon’: The Nix store accessed via a Unix domain socket + connection to nix-daemon. + + * ‘file://<path>’: A binary cache stored in <path>. + + If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on + whether the user has write access to the local Nix store/database. + set to true *unless* you're going to collect garbage. */ +ref<Store> openStoreAt(const std::string & uri); -/* Sort a set of paths topologically under the references relation. - If p refers to q, then p preceeds q in this list. */ -Paths topoSortPaths(StoreAPI & store, const PathSet & paths); +/* Open the store indicated by the ‘NIX_REMOTE’ environment variable. */ +ref<Store> openStore(); -/* For now, there is a single global store API object, but we'll - purify that in the future. */ -extern std::shared_ptr<StoreAPI> store; +/* Return the default substituter stores, defined by the + ‘substituters’ option and various legacy options like + ‘binary-caches’. */ +std::list<ref<Store>> getDefaultSubstituters(); -/* Factory method: open the Nix database, either through the local or - remote implementation. */ -std::shared_ptr<StoreAPI> openStore(bool reserveSpace = true); +/* Store implementation registration. */ +typedef std::map<std::string, std::string> StoreParams; + +typedef std::function<std::shared_ptr<Store>( + const std::string & uri, const StoreParams & params)> OpenStore; + +struct RegisterStoreImplementation +{ + typedef std::vector<OpenStore> Implementations; + static Implementations * implementations; + + RegisterStoreImplementation(OpenStore fun) + { + if (!implementations) implementations = new Implementations; + implementations->push_back(fun); + } +}; + /* Display a set of paths in human-readable form (i.e., between quotes @@ -361,14 +570,9 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); -/* Export multiple paths in the format expected by ‘nix-store - --import’. */ -void exportPaths(StoreAPI & store, const Paths & paths, - bool sign, Sink & sink); - - MakeError(SubstError, Error) MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d037d7402ede..7ff0553a016c 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10e +#define PROTOCOL_VERSION 0x111 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -14,8 +14,8 @@ namespace nix { typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, - wopQueryPathHash = 4, - wopQueryReferences = 5, + wopQueryPathHash = 4, // obsolete + wopQueryReferences = 5, // obsolete wopQueryReferrers = 6, wopAddToStore = 7, wopAddTextToStore = 8, @@ -25,8 +25,8 @@ typedef enum { wopAddIndirectRoot = 12, wopSyncWithGC = 13, wopFindRoots = 14, - wopExportPath = 16, - wopQueryDeriver = 18, + wopExportPath = 16, // obsolete + wopQueryDeriver = 18, // obsolete wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, @@ -35,7 +35,7 @@ typedef enum { wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, - wopImportPaths = 27, + wopImportPaths = 27, // obsolete wopQueryDerivationOutputNames = 28, wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, @@ -43,7 +43,9 @@ typedef enum { wopQuerySubstitutablePaths = 32, wopQueryValidDerivers = 33, wopOptimiseStore = 34, - wopVerifyStore = 35 + wopVerifyStore = 35, + wopBuildDerivation = 36, + wopAddSignatures = 37, } WorkerOp; diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc index 3e21f43a2e9d..3cbdf878617a 100644 --- a/src/libutil/affinity.cc +++ b/src/libutil/affinity.cc @@ -2,14 +2,14 @@ #include "util.hh" #include "affinity.hh" -#if HAVE_SCHED_H +#if __linux__ #include <sched.h> #endif namespace nix { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ static bool didSaveAffinity = false; static cpu_set_t savedAffinity; #endif @@ -17,7 +17,7 @@ static cpu_set_t savedAffinity; void setAffinityTo(int cpu) { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; didSaveAffinity = true; printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); @@ -32,7 +32,7 @@ void setAffinityTo(int cpu) int lockToCurrentCPU() { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ int cpu = sched_getcpu(); if (cpu != -1) setAffinityTo(cpu); return cpu; @@ -44,7 +44,7 @@ int lockToCurrentCPU() void restoreAffinity() { -#if HAVE_SCHED_SETAFFINITY +#if __linux__ if (!didSaveAffinity) return; if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) printMsg(lvlError, "failed to restore affinity %1%"); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 9e16e04ae4b5..5363496c272e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -29,7 +29,7 @@ bool useCaseHack = false; #endif -static string archiveVersion1 = "nix-archive-1"; +const std::string narVersionMagic1 = "nix-archive-1"; static string caseHackSuffix = "~nix~case~hack~"; @@ -39,8 +39,7 @@ PathFilter defaultPathFilter; static void dumpContents(const Path & path, size_t size, Sink & sink) { - writeString("contents", sink); - writeLongLong(size, sink); + sink << "contents" << size; AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file ‘%1%’") % path); @@ -65,21 +64,17 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path ‘%1%’") % path); - writeString("(", sink); + sink << "("; if (S_ISREG(st.st_mode)) { - writeString("type", sink); - writeString("regular", sink); - if (st.st_mode & S_IXUSR) { - writeString("executable", sink); - writeString("", sink); - } + sink << "type" << "regular"; + if (st.st_mode & S_IXUSR) + sink << "executable" << ""; dumpContents(path, (size_t) st.st_size, sink); } else if (S_ISDIR(st.st_mode)) { - writeString("type", sink); - writeString("directory", sink); + sink << "type" << "directory"; /* If we're on a case-insensitive system like Mac OS X, undo the case hack applied by restorePath(). */ @@ -101,36 +96,34 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) for (auto & i : unhacked) if (filter(path + "/" + i.first)) { - writeString("entry", sink); - writeString("(", sink); - writeString("name", sink); - writeString(i.first, sink); - writeString("node", sink); + sink << "entry" << "(" << "name" << i.first << "node"; dump(path + "/" + i.second, sink, filter); - writeString(")", sink); + sink << ")"; } } - else if (S_ISLNK(st.st_mode)) { - writeString("type", sink); - writeString("symlink", sink); - writeString("target", sink); - writeString(readLink(path), sink); - } + else if (S_ISLNK(st.st_mode)) + sink << "type" << "symlink" << "target" << readLink(path); else throw Error(format("file ‘%1%’ has an unsupported type") % path); - writeString(")", sink); + sink << ")"; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) { - writeString(archiveVersion1, sink); + sink << narVersionMagic1; dump(path, sink, filter); } +void dumpString(const std::string & s, Sink & sink) +{ + sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +} + + static SerialisationError badArchive(string s) { return SerialisationError("bad archive: " + s); @@ -227,7 +220,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "executable" && type == tpRegular) { - readString(source); + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); sink.isExecutable(); } @@ -256,7 +250,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) if (i != names.end()) { printMsg(lvlDebug, format("case collision between ‘%1%’ and ‘%2%’") % i->first % name); name += caseHackSuffix; - name += int2String(++i->second); + name += std::to_string(++i->second); } else names[name] = 0; } @@ -288,7 +282,7 @@ void parseDump(ParseSink & sink, Source & source) /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ } - if (version != archiveVersion1) + if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); parse(sink, source, ""); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index c216e9768fd1..d58b91df0461 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -55,6 +55,9 @@ extern PathFilter defaultPathFilter; void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +void dumpString(const std::string & s, Sink & sink); + +/* FIXME: fix this API, it sucks. */ struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -76,4 +79,7 @@ void restorePath(const Path & path, Source & source); extern bool useCaseHack; +extern const std::string narVersionMagic1; + + } diff --git a/src/libutil/args.cc b/src/libutil/args.cc new file mode 100644 index 000000000000..115484f9e6c7 --- /dev/null +++ b/src/libutil/args.cc @@ -0,0 +1,180 @@ +#include "args.hh" +#include "hash.hh" + +namespace nix { + +void Args::parseCmdline(const Strings & _cmdline) +{ + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { + + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); + else { + cmdline.insert(next, string(arg, j)); + break; + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } + else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag ‘%1%’") % arg); + } + else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) + pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const string & programName, std::ostream & out) +{ + std::cout << "Usage: " << programName << " <FLAGS>..."; + for (auto & exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + } + std::cout << "\n"; + + auto s = description(); + if (s != "") + std::cout << "\nSummary: " << s << ".\n"; + + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream & out) +{ + Table2 table; + for (auto & flag : longFlags) + table.push_back(std::make_pair( + (flag.second.shortName ? std::string("-") + flag.second.shortName + ", " : " ") + + "--" + flag.first + renderLabels(flag.second.labels), + flag.second.description)); + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + assert(pos != end); + + auto process = [&](const std::string & name, const Flag & flag) -> bool { + ++pos; + Strings args; + for (size_t n = 0 ; n < flag.arity; ++n) { + if (pos == end) + throw UsageError(format("flag ‘%1%’ requires %2% argument(s)") + % name % flag.arity); + args.push_back(*pos++); + } + flag.handler(args); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, i->second); + } + + return false; +} + +bool Args::processArgs(const Strings & args, bool finish) +{ + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument ‘%1%’") % args.front()); + return true; + } + + auto & exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) + { + exp.handler(args); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty()) + throw UsageError("more arguments are required"); + + return res; +} + +void Args::mkHashTypeFlag(const std::string & name, HashType * ht) +{ + mkFlag1(0, name, "TYPE", "hash algorithm (‘md5’, ‘sha1’, ‘sha256’, or ‘sha512’)", [=](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + }); +} + +Strings argvToStrings(int argc, char * * argv) +{ + Strings args; + argc--; argv++; + while (argc--) args.push_back(*argv++); + return args; +} + +std::string renderLabels(const Strings & labels) +{ + std::string res; + for (auto label : labels) { + for (auto & c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream & out, const Table2 & table) +{ + size_t max = 0; + for (auto & row : table) + max = std::max(max, row.first.size()); + for (auto & row : table) { + out << " " << row.first + << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} diff --git a/src/libutil/args.hh b/src/libutil/args.hh new file mode 100644 index 000000000000..6aa08aacac9e --- /dev/null +++ b/src/libutil/args.hh @@ -0,0 +1,163 @@ +#pragma once + +#include <iostream> +#include <map> +#include <memory> + +#include "util.hh" + +namespace nix { + +MakeError(UsageError, nix::Error); + +enum HashType : char; + +class Args +{ +public: + + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings & cmdline); + + virtual void printHelp(const string & programName, std::ostream & out); + + virtual std::string description() { return ""; } + +protected: + + /* Flags. */ + struct Flag + { + char shortName; + std::string description; + Strings labels; + size_t arity; + std::function<void(Strings)> handler; + }; + + std::map<std::string, Flag> longFlags; + std::map<char, Flag> shortFlags; + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + + void printFlags(std::ostream & out); + + /* Positional arguments. */ + struct ExpectedArg + { + std::string label; + size_t arity; // 0 = any + std::function<void(Strings)> handler; + }; + + std::list<ExpectedArg> expectedArgs; + + virtual bool processArgs(const Strings & args, bool finish); + +public: + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag(char shortName, const std::string & longName, + const Strings & labels, const std::string & description, + size_t arity, std::function<void(Strings)> handler) + { + auto flag = Flag{shortName, description, labels, arity, handler}; + if (shortName) shortFlags[shortName] = flag; + longFlags[longName] = flag; + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void()> fun) + { + mkFlag(shortName, longName, {}, description, 0, std::bind(fun)); + } + + void mkFlag1(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + std::function<void(std::string)> fun) + { + mkFlag(shortName, longName, {label}, description, 1, [=](Strings ss) { + fun(ss.front()); + }); + } + + void mkFlag(char shortName, const std::string & name, + const std::string & description, bool * dest) + { + mkFlag(shortName, name, description, dest, true); + } + + void mkFlag(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + string * dest) + { + mkFlag1(shortName, longName, label, description, [=](std::string s) { + *dest = s; + }); + } + + void mkHashTypeFlag(const std::string & name, HashType * ht); + + template<class T> + void mkFlag(char shortName, const std::string & longName, const std::string & description, + T * dest, const T & value) + { + mkFlag(shortName, longName, {}, description, 0, [=](Strings ss) { + *dest = value; + }); + } + + template<class I> + void mkIntFlag(char shortName, const std::string & longName, + const std::string & description, I * dest) + { + mkFlag<I>(shortName, longName, description, [=](I n) { + *dest = n; + }); + } + + template<class I> + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function<void(I)> fun) + { + mkFlag(shortName, longName, {"N"}, description, 1, [=](Strings ss) { + I n; + if (!string2Int(ss.front(), n)) + throw UsageError(format("flag ‘--%1%’ requires a integer argument") % longName); + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string & label, string * dest) + { + expectedArgs.push_back(ExpectedArg{label, 1, [=](Strings ss) { + *dest = ss.front(); + }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string & label, Strings * dest) + { + expectedArgs.push_back(ExpectedArg{label, 0, [=](Strings ss) { + *dest = ss; + }}); + } + + friend class MultiCommand; +}; + +Strings argvToStrings(int argc, char * * argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings & labels); + +/* Helper function for printing 2-column tables. */ +typedef std::vector<std::pair<std::string, std::string>> Table2; + +void printTable(std::ostream & out, const Table2 & table); + +} diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc new file mode 100644 index 000000000000..a3bbb5170d9f --- /dev/null +++ b/src/libutil/compression.cc @@ -0,0 +1,276 @@ +#include "compression.hh" +#include "util.hh" +#include "finally.hh" + +#include <lzma.h> +#include <bzlib.h> +#include <cstdio> +#include <cstring> + +#include <iostream> + +namespace nix { + +static ref<std::string> decompressXZ(const std::string & in) +{ + lzma_stream strm(LZMA_STREAM_INIT); + + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw Error("unable to initialise lzma decoder"); + + Finally free([&]() { lzma_end(&strm); }); + + lzma_action action = LZMA_RUN; + uint8_t outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + if (strm.avail_in == 0) + action = LZMA_FINISH; + + lzma_ret ret = lzma_code(&strm, action); + + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == LZMA_STREAM_END) + return res; + + if (ret != LZMA_OK) + throw Error("error while decompressing xz file"); + } +} + +static ref<std::string> decompressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 decoder"); + + Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + + char outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK) + throw Error("error while decompressing bzip2 file"); + } +} + +ref<std::string> compress(const std::string & method, const std::string & in) +{ + StringSink ssink; + auto sink = makeCompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +ref<std::string> decompress(const std::string & method, const std::string & in) +{ + if (method == "none") + return make_ref<std::string>(in); + else if (method == "xz") + return decompressXZ(in); + else if (method == "bzip2") + return decompressBzip2(in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +struct NoneSink : CompressionSink +{ + Sink & nextSink; + NoneSink(Sink & nextSink) : nextSink(nextSink) { } + void finish() override { flush(); } + void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +}; + +struct XzSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzSink(Sink & nextSink) : nextSink(nextSink) + { + lzma_ret ret = lzma_easy_encoder( + &strm, 6, LZMA_CHECK_CRC64); + if (ret != LZMA_OK) + throw Error("unable to initialise lzma encoder"); + // FIXME: apply the x86 BCJ filter? + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzSink() + { + assert(finished); + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + + assert(!finished); + finished = true; + + while (true) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw Error("error while flushing xz file"); + + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == LZMA_STREAM_END) break; + } + } + + void write(const unsigned char * data, size_t len) override + { + assert(!finished); + + strm.next_in = data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, LZMA_RUN); + if (ret != LZMA_OK) + throw Error("error while compressing xz file"); + + if (strm.avail_out == 0) { + nextSink(outbuf, sizeof(outbuf)); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipSink : CompressionSink +{ + Sink & nextSink; + char outbuf[BUFSIZ]; + bz_stream strm; + bool finished = false; + + BzipSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 encoder"); + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~BzipSink() + { + assert(finished); + BZ2_bzCompressEnd(&strm); + } + + void finish() override + { + flush(); + + assert(!finished); + finished = true; + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, BZ_FINISH); + if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw Error("error while flushing bzip2 file"); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + nextSink((unsigned char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) break; + } + } + + void write(const unsigned char * data, size_t len) override + { + assert(!finished); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, BZ_RUN); + if (ret != BZ_OK) + Error("error while compressing bzip2 file"); + + if (strm.avail_out == 0) { + nextSink((unsigned char *) outbuf, sizeof(outbuf)); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink) +{ + if (method == "none") + return make_ref<NoneSink>(nextSink); + else if (method == "xz") + return make_ref<XzSink>(nextSink); + else if (method == "bzip2") + return make_ref<BzipSink>(nextSink); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +} diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh new file mode 100644 index 000000000000..eacf559d65e9 --- /dev/null +++ b/src/libutil/compression.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "ref.hh" +#include "types.hh" +#include "serialise.hh" + +#include <string> + +namespace nix { + +ref<std::string> compress(const std::string & method, const std::string & in); + +ref<std::string> decompress(const std::string & method, const std::string & in); + +struct CompressionSink : BufferedSink +{ + virtual void finish() = 0; +}; + +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink); + +MakeError(UnknownCompressionMethod, Error); + +} diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh new file mode 100644 index 000000000000..47c64deaecea --- /dev/null +++ b/src/libutil/finally.hh @@ -0,0 +1,12 @@ +#pragma once + +/* A trivial class to run a function at the end of a scope. */ +class Finally +{ +private: + std::function<void()> fun; + +public: + Finally(std::function<void()> fun) : fun(fun) { } + ~Finally() { fun(); } +}; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index a83ba0a81817..c17f1c4d5150 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -3,16 +3,8 @@ #include <iostream> #include <cstring> -#ifdef HAVE_OPENSSL #include <openssl/md5.h> #include <openssl/sha.h> -#else -extern "C" { -#include "md5.h" -#include "sha1.h" -#include "sha256.h" -} -#endif #include "hash.hh" #include "archive.hh" @@ -40,7 +32,8 @@ Hash::Hash(HashType type) if (type == htMD5) hashSize = md5HashSize; else if (type == htSHA1) hashSize = sha1HashSize; else if (type == htSHA256) hashSize = sha256HashSize; - else throw Error("unknown hash type"); + else if (type == htSHA512) hashSize = sha512HashSize; + else abort(); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -71,6 +64,12 @@ bool Hash::operator < (const Hash & h) const } +std::string Hash::to_string(bool base32) const +{ + return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this)); +} + + const string base16Chars = "0123456789abcdef"; @@ -85,15 +84,28 @@ string printHash(const Hash & hash) } +Hash parseHash(const string & s) +{ + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw BadHash(format("invalid hash ‘%s’") % s); + string hts = string(s, 0, colon); + HashType ht = parseHashType(hts); + if (ht == htUnknown) + throw BadHash(format("unknown hash type ‘%s’") % hts); + return parseHash16or32(ht, string(s, colon + 1)); +} + + Hash parseHash(HashType ht, const string & s) { Hash hash(ht); if (s.length() != hash.hashSize * 2) - throw Error(format("invalid hash ‘%1%’") % s); + throw BadHash(format("invalid hash ‘%1%’") % s); for (unsigned int i = 0; i < hash.hashSize; i++) { string s2(s, i * 2, 2); if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw Error(format("invalid hash ‘%1%’") % s); + throw BadHash(format("invalid hash ‘%1%’") % s); std::istringstream str(s2); int n; str >> std::hex >> n; @@ -103,20 +115,14 @@ Hash parseHash(HashType ht, const string & s) } -unsigned int hashLength32(const Hash & hash) -{ - return (hash.hashSize * 8 - 1) / 5 + 1; -} - - // omitted: E O U T const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; string printHash32(const Hash & hash) { - Hash hash2(hash); - unsigned int len = hashLength32(hash); + size_t len = hash.base32Len(); + assert(len); string s; s.reserve(len); @@ -144,7 +150,7 @@ string printHash16or32(const Hash & hash) Hash parseHash32(HashType ht, const string & s) { Hash hash(ht); - unsigned int len = hashLength32(ht); + size_t len = hash.base32Len(); assert(s.size() == len); for (unsigned int n = 0; n < len; ++n) { @@ -153,7 +159,7 @@ Hash parseHash32(HashType ht, const string & s) for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw Error(format("invalid base-32 hash ‘%1%’") % s); + throw BadHash(format("invalid base-32 hash ‘%1%’") % s); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -171,11 +177,11 @@ Hash parseHash16or32(HashType ht, const string & s) if (s.size() == hash.hashSize * 2) /* hexadecimal representation */ hash = parseHash(ht, s); - else if (s.size() == hashLength32(hash)) + else if (s.size() == hash.base32Len()) /* base-32 representation */ hash = parseHash32(ht, s); else - throw Error(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") + throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") % s % printHashType(ht)); return hash; } @@ -199,6 +205,7 @@ union Ctx MD5_CTX md5; SHA_CTX sha1; SHA256_CTX sha256; + SHA512_CTX sha512; }; @@ -207,6 +214,7 @@ static void start(HashType ht, Ctx & ctx) if (ht == htMD5) MD5_Init(&ctx.md5); else if (ht == htSHA1) SHA1_Init(&ctx.sha1); else if (ht == htSHA256) SHA256_Init(&ctx.sha256); + else if (ht == htSHA512) SHA512_Init(&ctx.sha512); } @@ -216,6 +224,7 @@ static void update(HashType ht, Ctx & ctx, if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); + else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); } @@ -224,6 +233,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash) if (ht == htMD5) MD5_Final(hash, &ctx.md5); else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); + else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); } @@ -321,6 +331,7 @@ HashType parseHashType(const string & s) if (s == "md5") return htMD5; else if (s == "sha1") return htSHA1; else if (s == "sha256") return htSHA256; + else if (s == "sha512") return htSHA512; else return htUnknown; } @@ -330,7 +341,8 @@ string printHashType(HashType ht) if (ht == htMD5) return "md5"; else if (ht == htSHA1) return "sha1"; else if (ht == htSHA256) return "sha256"; - else throw Error("cannot print unknown hash type"); + else if (ht == htSHA512) return "sha512"; + else abort(); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2c6f176ec74c..02e213fc7b35 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -7,30 +7,37 @@ namespace nix { -typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType; +MakeError(BadHash, Error); + + +enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; const int md5HashSize = 16; const int sha1HashSize = 20; const int sha256HashSize = 32; +const int sha512HashSize = 64; extern const string base32Chars; struct Hash { - static const unsigned int maxHashSize = 32; + static const unsigned int maxHashSize = 64; unsigned int hashSize; unsigned char hash[maxHashSize]; HashType type; - /* Create an unusable hash object. */ + /* Create an unset hash object. */ Hash(); /* Create a zero-filled hash object. */ Hash(HashType type); + /* Check whether a hash is set. */ + operator bool () const { return type != htUnknown; } + /* Check whether two hash are equal. */ bool operator == (const Hash & h2) const; @@ -39,18 +46,31 @@ struct Hash /* For sorting. */ bool operator < (const Hash & h) const; + + /* Returns the length of a base-16 representation of this hash. */ + size_t base16Len() const + { + return hashSize * 2; + } + + /* Returns the length of a base-32 representation of this hash. */ + size_t base32Len() const + { + return (hashSize * 8 - 1) / 5 + 1; + } + + std::string to_string(bool base32 = true) const; }; /* Convert a hash to a hexadecimal representation. */ string printHash(const Hash & hash); +Hash parseHash(const string & s); + /* Parse a hexadecimal representation of a hash code. */ Hash parseHash(HashType ht, const string & s); -/* Returns the length of a base-32 hash representation. */ -unsigned int hashLength32(const Hash & hash); - /* Convert a hash to a base-32 representation. */ string printHash32(const Hash & hash); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 8af2e78d9ce4..98cad00d6d95 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,10 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -ifeq ($(HAVE_OPENSSL), 1) - libutil_LDFLAGS = $(OPENSSL_LIBS) -else - libutil_SOURCES += $(d)/md5.c $(d)/sha1.c $(d)/sha256.c -endif +libutil_LDFLAGS = -llzma -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc new file mode 100644 index 000000000000..15bb1e175da6 --- /dev/null +++ b/src/libutil/logging.cc @@ -0,0 +1,79 @@ +#include "logging.hh" +#include "util.hh" + +namespace nix { + +Logger * logger = 0; + +class SimpleLogger : public Logger +{ +public: + + bool systemd, tty; + + SimpleLogger() + { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: c = '3'; break; + case lvlInfo: c = '5'; break; + case lvlTalkative: case lvlChatty: c = '6'; break; + default: c = '7'; + } + prefix = std::string("<") + c + ">"; + } + + writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; + +Verbosity verbosity = lvlInfo; + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % fs.s); + haveWarned = true; + } +} + +void writeToStderr(const string & s) +{ + try { + writeFull(STDERR_FILENO, s); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + +Logger * makeDefaultLogger() +{ + return new SimpleLogger(); +} + +} diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh new file mode 100644 index 000000000000..277dff280053 --- /dev/null +++ b/src/libutil/logging.hh @@ -0,0 +1,82 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +class Activity; + +class Logger +{ + friend class Activity; + +public: + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void setExpected(const std::string & label, uint64_t value = 1) { } + virtual void setProgress(const std::string & label, uint64_t value = 1) { } + virtual void incExpected(const std::string & label, uint64_t value = 1) { } + virtual void incProgress(const std::string & label, uint64_t value = 1) { } + +private: + + virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; + + virtual void stopActivity(Activity & activity) = 0; + +}; + +class Activity +{ +public: + Logger & logger; + + Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) + : logger(logger) + { + logger.startActivity(*this, lvl, fs); + } + + ~Activity() + { + logger.stopActivity(*this); + } +}; + +extern Logger * logger; + +Logger * makeDefaultLogger(); + +extern Verbosity verbosity; /* suppress msgs > this */ + +#define printMsg(level, f) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +} diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh new file mode 100644 index 000000000000..35983aa2c918 --- /dev/null +++ b/src/libutil/lru-cache.hh @@ -0,0 +1,90 @@ +#pragma once + +#include <map> +#include <list> + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template<typename Key, typename Value> +class LRUCache +{ +private: + + size_t maxSize; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map<Key, std::pair<LRUIterator, Value>>; + using LRU = std::list<typename Data::iterator>; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t maxSize) : maxSize(maxSize) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + erase(key); + + if (data.size() >= maxSize) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + // FIXME: use boost::optional? + Value * get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return 0; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return &i->second.second; + } + + size_t size() + { + return data.size(); + } + + void clear() + { + data.clear(); + lru.clear(); + } +}; + +} diff --git a/src/libutil/md32_common.h b/src/libutil/md32_common.h deleted file mode 100644 index 0cbcfaf8a20b..000000000000 --- a/src/libutil/md32_common.h +++ /dev/null @@ -1,620 +0,0 @@ -/* crypto/md32_common.h */ -/* ==================================================================== - * Copyright (c) 1999-2002 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * licensing@OpenSSL.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - -/* - * This is a generic 32 bit "collector" for message digest algorithms. - * Whenever needed it collects input character stream into chunks of - * 32 bit values and invokes a block function that performs actual hash - * calculations. - * - * Porting guide. - * - * Obligatory macros: - * - * DATA_ORDER_IS_BIG_ENDIAN or DATA_ORDER_IS_LITTLE_ENDIAN - * this macro defines byte order of input stream. - * HASH_CBLOCK - * size of a unit chunk HASH_BLOCK operates on. - * HASH_LONG - * has to be at lest 32 bit wide, if it's wider, then - * HASH_LONG_LOG2 *has to* be defined along - * HASH_CTX - * context structure that at least contains following - * members: - * typedef struct { - * ... - * HASH_LONG Nl,Nh; - * HASH_LONG data[HASH_LBLOCK]; - * unsigned int num; - * ... - * } HASH_CTX; - * HASH_UPDATE - * name of "Update" function, implemented here. - * HASH_TRANSFORM - * name of "Transform" function, implemented here. - * HASH_FINAL - * name of "Final" function, implemented here. - * HASH_BLOCK_HOST_ORDER - * name of "block" function treating *aligned* input message - * in host byte order, implemented externally. - * HASH_BLOCK_DATA_ORDER - * name of "block" function treating *unaligned* input message - * in original (data) byte order, implemented externally (it - * actually is optional if data and host are of the same - * "endianess"). - * HASH_MAKE_STRING - * macro convering context variables to an ASCII hash string. - * - * Optional macros: - * - * B_ENDIAN or L_ENDIAN - * defines host byte-order. - * HASH_LONG_LOG2 - * defaults to 2 if not states otherwise. - * HASH_LBLOCK - * assumed to be HASH_CBLOCK/4 if not stated otherwise. - * HASH_BLOCK_DATA_ORDER_ALIGNED - * alternative "block" function capable of treating - * aligned input message in original (data) order, - * implemented externally. - * - * MD5 example: - * - * #define DATA_ORDER_IS_LITTLE_ENDIAN - * - * #define HASH_LONG MD5_LONG - * #define HASH_LONG_LOG2 MD5_LONG_LOG2 - * #define HASH_CTX MD5_CTX - * #define HASH_CBLOCK MD5_CBLOCK - * #define HASH_LBLOCK MD5_LBLOCK - * #define HASH_UPDATE MD5_Update - * #define HASH_TRANSFORM MD5_Transform - * #define HASH_FINAL MD5_Final - * #define HASH_BLOCK_HOST_ORDER md5_block_host_order - * #define HASH_BLOCK_DATA_ORDER md5_block_data_order - * - * <appro@fy.chalmers.se> - */ - -#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN) -#error "DATA_ORDER must be defined!" -#endif - -#ifndef HASH_CBLOCK -#error "HASH_CBLOCK must be defined!" -#endif -#ifndef HASH_LONG -#error "HASH_LONG must be defined!" -#endif -#ifndef HASH_CTX -#error "HASH_CTX must be defined!" -#endif - -#ifndef HASH_UPDATE -#error "HASH_UPDATE must be defined!" -#endif -#ifndef HASH_TRANSFORM -#error "HASH_TRANSFORM must be defined!" -#endif -#ifndef HASH_FINAL -#error "HASH_FINAL must be defined!" -#endif - -#ifndef HASH_BLOCK_HOST_ORDER -#error "HASH_BLOCK_HOST_ORDER must be defined!" -#endif - -#if 0 -/* - * Moved below as it's required only if HASH_BLOCK_DATA_ORDER_ALIGNED - * isn't defined. - */ -#ifndef HASH_BLOCK_DATA_ORDER -#error "HASH_BLOCK_DATA_ORDER must be defined!" -#endif -#endif - -#ifndef HASH_LBLOCK -#define HASH_LBLOCK (HASH_CBLOCK/4) -#endif - -#ifndef HASH_LONG_LOG2 -#define HASH_LONG_LOG2 2 -#endif - -/* - * Engage compiler specific rotate intrinsic function if available. - */ -#undef ROTATE -#ifndef PEDANTIC -# if defined(_MSC_VER) || defined(__ICC) -# define ROTATE(a,n) _lrotl(a,n) -# elif defined(__MWERKS__) -# if defined(__POWERPC__) -# define ROTATE(a,n) __rlwinm(a,n,0,31) -# elif defined(__MC68K__) - /* Motorola specific tweak. <appro@fy.chalmers.se> */ -# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) ) -# else -# define ROTATE(a,n) __rol(a,n) -# endif -# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) - /* - * Some GNU C inline assembler templates. Note that these are - * rotates by *constant* number of bits! But that's exactly - * what we need here... - * <appro@fy.chalmers.se> - */ -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "roll %1,%0" \ - : "=r"(ret) \ - : "I"(n), "0"(a) \ - : "cc"); \ - ret; \ - }) -# elif defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__) -# define ROTATE(a,n) ({ register unsigned int ret; \ - asm ( \ - "rlwinm %0,%1,%2,0,31" \ - : "=r"(ret) \ - : "r"(a), "I"(n)); \ - ret; \ - }) -# endif -# endif -#endif /* PEDANTIC */ - -#if HASH_LONG_LOG2==2 /* Engage only if sizeof(HASH_LONG)== 4 */ -/* A nice byte order reversal from Wei Dai <weidai@eskimo.com> */ -#ifdef ROTATE -/* 5 instructions with rotate instruction, else 9 */ -#define REVERSE_FETCH32(a,l) ( \ - l=*(const HASH_LONG *)(a), \ - ((ROTATE(l,8)&0x00FF00FF)|(ROTATE((l&0x00FF00FF),24))) \ - ) -#else -/* 6 instructions with rotate instruction, else 8 */ -#define REVERSE_FETCH32(a,l) ( \ - l=*(const HASH_LONG *)(a), \ - l=(((l>>8)&0x00FF00FF)|((l&0x00FF00FF)<<8)), \ - ROTATE(l,16) \ - ) -/* - * Originally the middle line started with l=(((l&0xFF00FF00)>>8)|... - * It's rewritten as above for two reasons: - * - RISCs aren't good at long constants and have to explicitely - * compose 'em with several (well, usually 2) instructions in a - * register before performing the actual operation and (as you - * already realized:-) having same constant should inspire the - * compiler to permanently allocate the only register for it; - * - most modern CPUs have two ALUs, but usually only one has - * circuitry for shifts:-( this minor tweak inspires compiler - * to schedule shift instructions in a better way... - * - * <appro@fy.chalmers.se> - */ -#endif -#endif - -#ifndef ROTATE -#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n)))) -#endif - -/* - * Make some obvious choices. E.g., HASH_BLOCK_DATA_ORDER_ALIGNED - * and HASH_BLOCK_HOST_ORDER ought to be the same if input data - * and host are of the same "endianess". It's possible to mask - * this with blank #define HASH_BLOCK_DATA_ORDER though... - * - * <appro@fy.chalmers.se> - */ -#if defined(B_ENDIAN) -# if defined(DATA_ORDER_IS_BIG_ENDIAN) -# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 -# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER -# endif -# endif -#elif defined(L_ENDIAN) -# if defined(DATA_ORDER_IS_LITTLE_ENDIAN) -# if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2 -# define HASH_BLOCK_DATA_ORDER_ALIGNED HASH_BLOCK_HOST_ORDER -# endif -# endif -#endif - -#if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) -#ifndef HASH_BLOCK_DATA_ORDER -#error "HASH_BLOCK_DATA_ORDER must be defined!" -#endif -#endif - -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - -#ifndef PEDANTIC -# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM) -# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) - /* - * This gives ~30-40% performance improvement in SHA-256 compiled - * with gcc [on P4]. Well, first macro to be frank. We can pull - * this trick on x86* platforms only, because these CPUs can fetch - * unaligned data without raising an exception. - */ -# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - (c)+=4; (l)=r; }) -# define HOST_l2c(l,c) ({ unsigned int r=(l); \ - asm ("bswapl %0":"=r"(r):"0"(r)); \ - *((unsigned int *)(c))=r; (c)+=4; r; }) -# endif -# endif -#endif - -#ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++))) ), \ - l) -#endif -#define HOST_p_c2l(c,l,n) { \ - switch (n) { \ - case 0: l =((unsigned long)(*((c)++)))<<24; \ - case 1: l|=((unsigned long)(*((c)++)))<<16; \ - case 2: l|=((unsigned long)(*((c)++)))<< 8; \ - case 3: l|=((unsigned long)(*((c)++))); \ - } } -#define HOST_p_c2l_p(c,l,sc,len) { \ - switch (sc) { \ - case 0: l =((unsigned long)(*((c)++)))<<24; \ - if (--len == 0) break; \ - case 1: l|=((unsigned long)(*((c)++)))<<16; \ - if (--len == 0) break; \ - case 2: l|=((unsigned long)(*((c)++)))<< 8; \ - } } -/* NOTE the pointer is not incremented at the end of this */ -#define HOST_c2l_p(c,l,n) { \ - l=0; (c)+=n; \ - switch (n) { \ - case 3: l =((unsigned long)(*(--(c))))<< 8; \ - case 2: l|=((unsigned long)(*(--(c))))<<16; \ - case 1: l|=((unsigned long)(*(--(c))))<<24; \ - } } -#ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l) )&0xff), \ - l) -#endif - -#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - -#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) - /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */ -# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l) -# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l) -#endif - -#ifndef HOST_c2l -#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \ - l|=(((unsigned long)(*((c)++)))<< 8), \ - l|=(((unsigned long)(*((c)++)))<<16), \ - l|=(((unsigned long)(*((c)++)))<<24), \ - l) -#endif -#define HOST_p_c2l(c,l,n) { \ - switch (n) { \ - case 0: l =((unsigned long)(*((c)++))); \ - case 1: l|=((unsigned long)(*((c)++)))<< 8; \ - case 2: l|=((unsigned long)(*((c)++)))<<16; \ - case 3: l|=((unsigned long)(*((c)++)))<<24; \ - } } -#define HOST_p_c2l_p(c,l,sc,len) { \ - switch (sc) { \ - case 0: l =((unsigned long)(*((c)++))); \ - if (--len == 0) break; \ - case 1: l|=((unsigned long)(*((c)++)))<< 8; \ - if (--len == 0) break; \ - case 2: l|=((unsigned long)(*((c)++)))<<16; \ - } } -/* NOTE the pointer is not incremented at the end of this */ -#define HOST_c2l_p(c,l,n) { \ - l=0; (c)+=n; \ - switch (n) { \ - case 3: l =((unsigned long)(*(--(c))))<<16; \ - case 2: l|=((unsigned long)(*(--(c))))<< 8; \ - case 1: l|=((unsigned long)(*(--(c)))); \ - } } -#ifndef HOST_l2c -#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \ - *((c)++)=(unsigned char)(((l)>> 8)&0xff), \ - *((c)++)=(unsigned char)(((l)>>16)&0xff), \ - *((c)++)=(unsigned char)(((l)>>24)&0xff), \ - l) -#endif - -#endif - -/* - * Time for some action:-) - */ - -int HASH_UPDATE (HASH_CTX *c, const void *data_, size_t len) - { - const unsigned char *data=data_; - register HASH_LONG * p; - register HASH_LONG l; - size_t sw,sc,ew,ec; - - if (len==0) return 1; - - l=(c->Nl+(((HASH_LONG)len)<<3))&0xffffffffUL; - /* 95-05-24 eay Fixed a bug with the overflow handling, thanks to - * Wei Dai <weidai@eskimo.com> for pointing it out. */ - if (l < c->Nl) /* overflow */ - c->Nh++; - c->Nh+=(len>>29); /* might cause compiler warning on 16-bit */ - c->Nl=l; - - if (c->num != 0) - { - p=c->data; - sw=c->num>>2; - sc=c->num&0x03; - - if ((c->num+len) >= HASH_CBLOCK) - { - l=p[sw]; HOST_p_c2l(data,l,sc); p[sw++]=l; - for (; sw<HASH_LBLOCK; sw++) - { - HOST_c2l(data,l); p[sw]=l; - } - HASH_BLOCK_HOST_ORDER (c,p,1); - len-=(HASH_CBLOCK-c->num); - c->num=0; - /* drop through and do the rest */ - } - else - { - c->num+=(unsigned int)len; - if ((sc+len) < 4) /* ugly, add char's to a word */ - { - l=p[sw]; HOST_p_c2l_p(data,l,sc,len); p[sw]=l; - } - else - { - ew=(c->num>>2); - ec=(c->num&0x03); - if (sc) - l=p[sw]; - HOST_p_c2l(data,l,sc); - p[sw++]=l; - for (; sw < ew; sw++) - { - HOST_c2l(data,l); p[sw]=l; - } - if (ec) - { - HOST_c2l_p(data,l,ec); p[sw]=l; - } - } - return 1; - } - } - - sw=len/HASH_CBLOCK; - if (sw > 0) - { -#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) - /* - * Note that HASH_BLOCK_DATA_ORDER_ALIGNED gets defined - * only if sizeof(HASH_LONG)==4. - */ - if ((((size_t)data)%4) == 0) - { - /* data is properly aligned so that we can cast it: */ - HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,sw); - sw*=HASH_CBLOCK; - data+=sw; - len-=sw; - } - else -#if !defined(HASH_BLOCK_DATA_ORDER) - while (sw--) - { - memcpy (p=c->data,data,HASH_CBLOCK); - HASH_BLOCK_DATA_ORDER_ALIGNED(c,p,1); - data+=HASH_CBLOCK; - len-=HASH_CBLOCK; - } -#endif -#endif -#if defined(HASH_BLOCK_DATA_ORDER) - { - HASH_BLOCK_DATA_ORDER(c,data,sw); - sw*=HASH_CBLOCK; - data+=sw; - len-=sw; - } -#endif - } - - if (len!=0) - { - p = c->data; - c->num = len; - ew=len>>2; /* words to copy */ - ec=len&0x03; - for (; ew; ew--,p++) - { - HOST_c2l(data,l); *p=l; - } - HOST_c2l_p(data,l,ec); - *p=l; - } - return 1; - } - - -void HASH_TRANSFORM (HASH_CTX *c, const unsigned char *data) - { -#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED) - if ((((size_t)data)%4) == 0) - /* data is properly aligned so that we can cast it: */ - HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,1); - else -#if !defined(HASH_BLOCK_DATA_ORDER) - { - memcpy (c->data,data,HASH_CBLOCK); - HASH_BLOCK_DATA_ORDER_ALIGNED (c,c->data,1); - } -#endif -#endif -#if defined(HASH_BLOCK_DATA_ORDER) - HASH_BLOCK_DATA_ORDER (c,data,1); -#endif - } - - -int HASH_FINAL (unsigned char *md, HASH_CTX *c) - { - register HASH_LONG *p; - register unsigned long l; - register int i,j; - static const unsigned char end[4]={0x80,0x00,0x00,0x00}; - const unsigned char *cp=end; - - /* c->num should definitly have room for at least one more byte. */ - p=c->data; - i=c->num>>2; - j=c->num&0x03; - -#if 0 - /* purify often complains about the following line as an - * Uninitialized Memory Read. While this can be true, the - * following p_c2l macro will reset l when that case is true. - * This is because j&0x03 contains the number of 'valid' bytes - * already in p[i]. If and only if j&0x03 == 0, the UMR will - * occur but this is also the only time p_c2l will do - * l= *(cp++) instead of l|= *(cp++) - * Many thanks to Alex Tang <altitude@cic.net> for pickup this - * 'potential bug' */ -#ifdef PURIFY - if (j==0) p[i]=0; /* Yeah, but that's not the way to fix it:-) */ -#endif - l=p[i]; -#else - l = (j==0) ? 0 : p[i]; -#endif - HOST_p_c2l(cp,l,j); p[i++]=l; /* i is the next 'undefined word' */ - - if (i>(HASH_LBLOCK-2)) /* save room for Nl and Nh */ - { - if (i<HASH_LBLOCK) p[i]=0; - HASH_BLOCK_HOST_ORDER (c,p,1); - i=0; - } - for (; i<(HASH_LBLOCK-2); i++) - p[i]=0; - -#if defined(DATA_ORDER_IS_BIG_ENDIAN) - p[HASH_LBLOCK-2]=c->Nh; - p[HASH_LBLOCK-1]=c->Nl; -#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN) - p[HASH_LBLOCK-2]=c->Nl; - p[HASH_LBLOCK-1]=c->Nh; -#endif - HASH_BLOCK_HOST_ORDER (c,p,1); - -#ifndef HASH_MAKE_STRING -#error "HASH_MAKE_STRING must be defined!" -#else - HASH_MAKE_STRING(c,md); -#endif - - c->num=0; - /* clear stuff, HASH_BLOCK may be leaving some stuff on the stack - * but I'm not worried :-) - OPENSSL_cleanse((void *)c,sizeof(HASH_CTX)); - */ - return 1; - } - -#ifndef MD32_REG_T -#define MD32_REG_T long -/* - * This comment was originaly written for MD5, which is why it - * discusses A-D. But it basically applies to all 32-bit digests, - * which is why it was moved to common header file. - * - * In case you wonder why A-D are declared as long and not - * as MD5_LONG. Doing so results in slight performance - * boost on LP64 architectures. The catch is we don't - * really care if 32 MSBs of a 64-bit register get polluted - * with eventual overflows as we *save* only 32 LSBs in - * *either* case. Now declaring 'em long excuses the compiler - * from keeping 32 MSBs zeroed resulting in 13% performance - * improvement under SPARC Solaris7/64 and 5% under AlphaLinux. - * Well, to be honest it should say that this *prevents* - * performance degradation. - * <appro@fy.chalmers.se> - * Apparently there're LP64 compilers that generate better - * code if A-D are declared int. Most notably GCC-x86_64 - * generates better code. - * <appro@fy.chalmers.se> - */ -#endif diff --git a/src/libutil/md5.c b/src/libutil/md5.c deleted file mode 100644 index b31640cdcced..000000000000 --- a/src/libutil/md5.c +++ /dev/null @@ -1,365 +0,0 @@ -/* Functions to compute MD5 message digest of files or memory blocks. - according to the definition of MD5 in RFC 1321 from April 1992. - Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -/* Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995. */ - -#include <sys/types.h> - -#include <stdlib.h> -#include <string.h> - -#include "md5.h" - - -static md5_uint32 SWAP(md5_uint32 n) -{ - static int checked = 0; - static int bigendian = 0; - static md5_uint32 test; - - if (!checked) { - test = 1; - if (* (char *) &test == 0) - bigendian = 1; - checked = 1; - } - - if (bigendian) - return (((n) << 24) | (((n) & 0xff00) << 8) | (((n) >> 8) & 0xff00) | ((n) >> 24)); - else - return n; -} - - -/* This array contains the bytes used to pad the buffer to the next - 64-byte boundary. (RFC 1321, 3.1: Step 1) */ -static const unsigned char fillbuf[64] = { 0x80, 0 /* , 0, 0, ... */ }; - - -/* Initialize structure containing state of computation. - (RFC 1321, 3.3: Step 3) */ -void -MD5_Init (ctx) - struct MD5_CTX *ctx; -{ - ctx->A = 0x67452301; - ctx->B = 0xefcdab89; - ctx->C = 0x98badcfe; - ctx->D = 0x10325476; - - ctx->total[0] = ctx->total[1] = 0; - ctx->buflen = 0; -} - -/* Put result from CTX in first 16 bytes following RESBUF. The result - must be in little endian byte order. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -void * -md5_read_ctx (ctx, resbuf) - const struct MD5_CTX *ctx; - void *resbuf; -{ - ((md5_uint32 *) resbuf)[0] = SWAP (ctx->A); - ((md5_uint32 *) resbuf)[1] = SWAP (ctx->B); - ((md5_uint32 *) resbuf)[2] = SWAP (ctx->C); - ((md5_uint32 *) resbuf)[3] = SWAP (ctx->D); - - return resbuf; -} - -/* Process the remaining bytes in the internal buffer and the usual - prolog according to the standard and write the result to RESBUF. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -void * -MD5_Final (resbuf, ctx) - void *resbuf; - struct MD5_CTX *ctx; -{ - /* Take yet unprocessed bytes into account. */ - md5_uint32 bytes = ctx->buflen; - size_t pad; - - /* Now count remaining bytes. */ - ctx->total[0] += bytes; - if (ctx->total[0] < bytes) - ++ctx->total[1]; - - pad = bytes >= 56 ? 64 + 56 - bytes : 56 - bytes; - memcpy (&ctx->buffer[bytes], fillbuf, pad); - - /* Put the 64-bit file length in *bits* at the end of the buffer. */ - *(md5_uint32 *) &ctx->buffer[bytes + pad] = SWAP (ctx->total[0] << 3); - *(md5_uint32 *) &ctx->buffer[bytes + pad + 4] = SWAP ((ctx->total[1] << 3) | - (ctx->total[0] >> 29)); - - /* Process last bytes. */ - md5_process_block (ctx->buffer, bytes + pad + 8, ctx); - - return md5_read_ctx (ctx, resbuf); -} - -void -MD5_Update (ctx, buffer, len) - struct MD5_CTX *ctx; - const void *buffer; - size_t len; -{ - /* When we already have some bits in our internal buffer concatenate - both inputs first. */ - if (ctx->buflen != 0) - { - size_t left_over = ctx->buflen; - size_t add = 128 - left_over > len ? len : 128 - left_over; - - memcpy (&ctx->buffer[left_over], buffer, add); - ctx->buflen += add; - - if (ctx->buflen > 64) - { - md5_process_block (ctx->buffer, ctx->buflen & ~63, ctx); - - ctx->buflen &= 63; - /* The regions in the following copy operation cannot overlap. */ - memcpy (ctx->buffer, &ctx->buffer[(left_over + add) & ~63], - ctx->buflen); - } - - buffer = (const char *) buffer + add; - len -= add; - } - - /* Process available complete blocks. */ - if (len >= 64) - { -#if !_STRING_ARCH_unaligned -/* To check alignment gcc has an appropriate operator. Other - compilers don't. */ -# if __GNUC__ >= 2 -# define UNALIGNED_P(p) (((md5_uintptr) p) % __alignof__ (md5_uint32) != 0) -# else -# define UNALIGNED_P(p) (((md5_uintptr) p) % sizeof (md5_uint32) != 0) -# endif - if (UNALIGNED_P (buffer)) - while (len > 64) - { - md5_process_block (memcpy (ctx->buffer, buffer, 64), 64, ctx); - buffer = (const char *) buffer + 64; - len -= 64; - } - else -#endif - { - md5_process_block (buffer, len & ~63, ctx); - buffer = (const char *) buffer + (len & ~63); - len &= 63; - } - } - - /* Move remaining bytes in internal buffer. */ - if (len > 0) - { - size_t left_over = ctx->buflen; - - memcpy (&ctx->buffer[left_over], buffer, len); - left_over += len; - if (left_over >= 64) - { - md5_process_block (ctx->buffer, 64, ctx); - left_over -= 64; - memcpy (ctx->buffer, &ctx->buffer[64], left_over); - } - ctx->buflen = left_over; - } -} - - -/* These are the four functions used in the four steps of the MD5 algorithm - and defined in the RFC 1321. The first function is a little bit optimized - (as found in Colin Plumbs public domain implementation). */ -/* #define FF(b, c, d) ((b & c) | (~b & d)) */ -#define FF(b, c, d) (d ^ (b & (c ^ d))) -#define FG(b, c, d) FF (d, b, c) -#define FH(b, c, d) (b ^ c ^ d) -#define FI(b, c, d) (c ^ (b | ~d)) - -/* Process LEN bytes of BUFFER, accumulating context into CTX. - It is assumed that LEN % 64 == 0. */ - -void -md5_process_block (buffer, len, ctx) - const void *buffer; - size_t len; - struct MD5_CTX *ctx; -{ - md5_uint32 correct_words[16]; - const md5_uint32 *words = buffer; - size_t nwords = len / sizeof (md5_uint32); - const md5_uint32 *endp = words + nwords; - md5_uint32 A = ctx->A; - md5_uint32 B = ctx->B; - md5_uint32 C = ctx->C; - md5_uint32 D = ctx->D; - - /* First increment the byte count. RFC 1321 specifies the possible - length of the file up to 2^64 bits. Here we only compute the - number of bytes. Do a double word increment. */ - ctx->total[0] += len; - if (ctx->total[0] < len) - ++ctx->total[1]; - - /* Process all bytes in the buffer with 64 bytes in each round of - the loop. */ - while (words < endp) - { - md5_uint32 *cwp = correct_words; - md5_uint32 A_save = A; - md5_uint32 B_save = B; - md5_uint32 C_save = C; - md5_uint32 D_save = D; - - /* First round: using the given function, the context and a constant - the next context is computed. Because the algorithms processing - unit is a 32-bit word and it is determined to work on words in - little endian byte order we perhaps have to change the byte order - before the computation. To reduce the work for the next steps - we store the swapped words in the array CORRECT_WORDS. */ - -#define OP(a, b, c, d, s, T) \ - do \ - { \ - a += FF (b, c, d) + (*cwp++ = SWAP (*words)) + T; \ - ++words; \ - CYCLIC (a, s); \ - a += b; \ - } \ - while (0) - - /* It is unfortunate that C does not provide an operator for - cyclic rotation. Hope the C compiler is smart enough. */ -#define CYCLIC(w, s) (w = (w << s) | (w >> (32 - s))) - - /* Before we start, one word to the strange constants. - They are defined in RFC 1321 as - - T[i] = (int) (4294967296.0 * fabs (sin (i))), i=1..64 - */ - - /* Round 1. */ - OP (A, B, C, D, 7, 0xd76aa478); - OP (D, A, B, C, 12, 0xe8c7b756); - OP (C, D, A, B, 17, 0x242070db); - OP (B, C, D, A, 22, 0xc1bdceee); - OP (A, B, C, D, 7, 0xf57c0faf); - OP (D, A, B, C, 12, 0x4787c62a); - OP (C, D, A, B, 17, 0xa8304613); - OP (B, C, D, A, 22, 0xfd469501); - OP (A, B, C, D, 7, 0x698098d8); - OP (D, A, B, C, 12, 0x8b44f7af); - OP (C, D, A, B, 17, 0xffff5bb1); - OP (B, C, D, A, 22, 0x895cd7be); - OP (A, B, C, D, 7, 0x6b901122); - OP (D, A, B, C, 12, 0xfd987193); - OP (C, D, A, B, 17, 0xa679438e); - OP (B, C, D, A, 22, 0x49b40821); - - /* For the second to fourth round we have the possibly swapped words - in CORRECT_WORDS. Redefine the macro to take an additional first - argument specifying the function to use. */ -#undef OP -#define OP(f, a, b, c, d, k, s, T) \ - do \ - { \ - a += f (b, c, d) + correct_words[k] + T; \ - CYCLIC (a, s); \ - a += b; \ - } \ - while (0) - - /* Round 2. */ - OP (FG, A, B, C, D, 1, 5, 0xf61e2562); - OP (FG, D, A, B, C, 6, 9, 0xc040b340); - OP (FG, C, D, A, B, 11, 14, 0x265e5a51); - OP (FG, B, C, D, A, 0, 20, 0xe9b6c7aa); - OP (FG, A, B, C, D, 5, 5, 0xd62f105d); - OP (FG, D, A, B, C, 10, 9, 0x02441453); - OP (FG, C, D, A, B, 15, 14, 0xd8a1e681); - OP (FG, B, C, D, A, 4, 20, 0xe7d3fbc8); - OP (FG, A, B, C, D, 9, 5, 0x21e1cde6); - OP (FG, D, A, B, C, 14, 9, 0xc33707d6); - OP (FG, C, D, A, B, 3, 14, 0xf4d50d87); - OP (FG, B, C, D, A, 8, 20, 0x455a14ed); - OP (FG, A, B, C, D, 13, 5, 0xa9e3e905); - OP (FG, D, A, B, C, 2, 9, 0xfcefa3f8); - OP (FG, C, D, A, B, 7, 14, 0x676f02d9); - OP (FG, B, C, D, A, 12, 20, 0x8d2a4c8a); - - /* Round 3. */ - OP (FH, A, B, C, D, 5, 4, 0xfffa3942); - OP (FH, D, A, B, C, 8, 11, 0x8771f681); - OP (FH, C, D, A, B, 11, 16, 0x6d9d6122); - OP (FH, B, C, D, A, 14, 23, 0xfde5380c); - OP (FH, A, B, C, D, 1, 4, 0xa4beea44); - OP (FH, D, A, B, C, 4, 11, 0x4bdecfa9); - OP (FH, C, D, A, B, 7, 16, 0xf6bb4b60); - OP (FH, B, C, D, A, 10, 23, 0xbebfbc70); - OP (FH, A, B, C, D, 13, 4, 0x289b7ec6); - OP (FH, D, A, B, C, 0, 11, 0xeaa127fa); - OP (FH, C, D, A, B, 3, 16, 0xd4ef3085); - OP (FH, B, C, D, A, 6, 23, 0x04881d05); - OP (FH, A, B, C, D, 9, 4, 0xd9d4d039); - OP (FH, D, A, B, C, 12, 11, 0xe6db99e5); - OP (FH, C, D, A, B, 15, 16, 0x1fa27cf8); - OP (FH, B, C, D, A, 2, 23, 0xc4ac5665); - - /* Round 4. */ - OP (FI, A, B, C, D, 0, 6, 0xf4292244); - OP (FI, D, A, B, C, 7, 10, 0x432aff97); - OP (FI, C, D, A, B, 14, 15, 0xab9423a7); - OP (FI, B, C, D, A, 5, 21, 0xfc93a039); - OP (FI, A, B, C, D, 12, 6, 0x655b59c3); - OP (FI, D, A, B, C, 3, 10, 0x8f0ccc92); - OP (FI, C, D, A, B, 10, 15, 0xffeff47d); - OP (FI, B, C, D, A, 1, 21, 0x85845dd1); - OP (FI, A, B, C, D, 8, 6, 0x6fa87e4f); - OP (FI, D, A, B, C, 15, 10, 0xfe2ce6e0); - OP (FI, C, D, A, B, 6, 15, 0xa3014314); - OP (FI, B, C, D, A, 13, 21, 0x4e0811a1); - OP (FI, A, B, C, D, 4, 6, 0xf7537e82); - OP (FI, D, A, B, C, 11, 10, 0xbd3af235); - OP (FI, C, D, A, B, 2, 15, 0x2ad7d2bb); - OP (FI, B, C, D, A, 9, 21, 0xeb86d391); - - /* Add the starting values of the context. */ - A += A_save; - B += B_save; - C += C_save; - D += D_save; - } - - /* Put checksum in context given as argument. */ - ctx->A = A; - ctx->B = B; - ctx->C = C; - ctx->D = D; -} diff --git a/src/libutil/md5.h b/src/libutil/md5.h deleted file mode 100644 index 228d4972320f..000000000000 --- a/src/libutil/md5.h +++ /dev/null @@ -1,82 +0,0 @@ -/* Declaration of functions and data types used for MD5 sum computing - library functions. - Copyright (C) 1995,1996,1997,1999,2000,2001 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#ifndef _MD5_H -#define _MD5_H 1 - -#include <inttypes.h> -typedef uint32_t md5_uint32; -typedef uintptr_t md5_uintptr; - -/* Structure to save state of computation between the single steps. */ -struct MD5_CTX -{ - md5_uint32 A; - md5_uint32 B; - md5_uint32 C; - md5_uint32 D; - - md5_uint32 total[2]; - md5_uint32 buflen; - char buffer[128] __attribute__ ((__aligned__ (__alignof__ (md5_uint32)))); -}; - -/* - * The following three functions are build up the low level used in - * the functions `md5_stream' and `md5_buffer'. - */ - -/* Initialize structure containing state of computation. - (RFC 1321, 3.3: Step 3) */ -extern void MD5_Init (struct MD5_CTX *ctx); - -/* Starting with the result of former calls of this function (or the - initialization function update the context for the next LEN bytes - starting at BUFFER. - It is necessary that LEN is a multiple of 64!!! */ -extern void md5_process_block (const void *buffer, size_t len, - struct MD5_CTX *ctx); - -/* Starting with the result of former calls of this function (or the - initialization function update the context for the next LEN bytes - starting at BUFFER. - It is NOT required that LEN is a multiple of 64. */ -extern void MD5_Update (struct MD5_CTX *ctx, const void *buffer, size_t len); - -/* Process the remaining bytes in the buffer and put result from CTX - in first 16 bytes following RESBUF. The result is always in little - endian byte order, so that a byte-wise output yields to the wanted - ASCII representation of the message digest. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -extern void *MD5_Final (void *resbuf, struct MD5_CTX *ctx); - - -/* Put result from CTX in first 16 bytes following RESBUF. The result is - always in little endian byte order, so that a byte-wise output yields - to the wanted ASCII representation of the message digest. - - IMPORTANT: On some systems it is required that RESBUF is correctly - aligned for a 32 bits value. */ -extern void *md5_read_ctx (const struct MD5_CTX *ctx, void *resbuf); - - -#endif /* md5.h */ diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh new file mode 100644 index 000000000000..f291cd578388 --- /dev/null +++ b/src/libutil/pool.hh @@ -0,0 +1,151 @@ +#pragma once + +#include <functional> +#include <limits> +#include <list> +#include <memory> +#include <cassert> + +#include "sync.hh" +#include "ref.hh" + +namespace nix { + +/* This template class implements a simple pool manager of resources + of some type R, such as database connections. It is used as + follows: + + class Connection { ... }; + + Pool<Connection> pool; + + { + auto conn(pool.get()); + conn->exec("select ..."); + } + + Here, the Connection object referenced by ‘conn’ is automatically + returned to the pool when ‘conn’ goes out of scope. +*/ + +template <class R> +class Pool +{ +public: + + /* A function that produces new instances of R on demand. */ + typedef std::function<ref<R>()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + typedef std::function<bool(const ref<R> &)> Validator; + +private: + + Factory factory; + Validator validator; + + struct State + { + size_t inUse = 0; + size_t max; + std::vector<ref<R>> idle; + }; + + Sync<State> state; + + std::condition_variable wakeup; + +public: + + Pool(size_t max = std::numeric_limits<size_t>::max(), + const Factory & factory = []() { return make_ref<R>(); }, + const Validator & validator = [](ref<R> r) { return true; }) + : factory(factory) + , validator(validator) + { + auto state_(state.lock()); + state_->max = max; + } + + ~Pool() + { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle + { + private: + Pool & pool; + std::shared_ptr<R> r; + + friend Pool; + + Handle(Pool & pool, std::shared_ptr<R> r) : pool(pool), r(r) { } + + public: + Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle & l) = delete; + + ~Handle() + { + if (!r) return; + { + auto state_(pool.state.lock()); + state_->idle.push_back(ref<R>(r)); + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); + } + + R * operator -> () { return &*r; } + R & operator * () { return *r; } + }; + + Handle get() + { + { + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) + state_.wait(wakeup); + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); + } + } + + state_->inUse++; + } + + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + throw; + } + } + + unsigned int count() + { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } +}; + +} diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh new file mode 100644 index 000000000000..85afa28119a9 --- /dev/null +++ b/src/libutil/ref.hh @@ -0,0 +1,81 @@ +#pragma once + +#include <memory> +#include <exception> +#include <stdexcept> + +namespace nix { + +/* A simple non-nullable reference-counted pointer. Actually a wrapper + around std::shared_ptr that prevents non-null constructions. */ +template<typename T> +class ref +{ +private: + + std::shared_ptr<T> p; + +public: + + ref<T>(const ref<T> & r) + : p(r.p) + { } + + explicit ref<T>(const std::shared_ptr<T> & p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + explicit ref<T>(T * p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + T* operator ->() const + { + return &*p; + } + + T& operator *() const + { + return *p; + } + + operator std::shared_ptr<T> () + { + return p; + } + + template<typename T2> + ref<T2> cast() + { + return ref<T2>(std::dynamic_pointer_cast<T2>(p)); + } + + template<typename T2> + operator ref<T2> () + { + return ref<T2>((std::shared_ptr<T2>) p); + } + +private: + + template<typename T2, typename... Args> + friend ref<T2> + make_ref(Args&&... args); + +}; + +template<typename T, typename... Args> +inline ref<T> +make_ref(Args&&... args) +{ + auto p = std::make_shared<T>(std::forward<Args>(args)...); + return ref<T>(p); +} + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 92417507508a..5c45c890f7b6 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -16,11 +16,11 @@ BufferedSink::~BufferedSink() delete[] buffer; } - + void BufferedSink::operator () (const unsigned char * data, size_t len) { if (!buffer) buffer = new unsigned char[bufSize]; - + while (len) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ @@ -64,15 +64,25 @@ static void warnLargeDump() void FdSink::write(const unsigned char * data, size_t len) { + written += len; static bool warned = false; if (warn && !warned) { - written += len; if (written > threshold) { warnLargeDump(); warned = true; } } - writeFull(fd, data, len); + try { + writeFull(fd, data, len); + } catch (SysError & e) { + _good = true; + } +} + + +bool FdSink::good() +{ + return _good; } @@ -96,7 +106,7 @@ size_t BufferedSource::read(unsigned char * data, size_t len) if (!buffer) buffer = new unsigned char[bufSize]; if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); - + /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; memcpy(data, buffer + bufPosOut, n); @@ -119,12 +129,19 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) checkInterrupt(); n = ::read(fd, (char *) data, bufSize); } while (n == -1 && errno == EINTR); - if (n == -1) throw SysError("reading from file"); - if (n == 0) throw EndOfFile("unexpected end-of-file"); + if (n == -1) { _good = false; throw SysError("reading from file"); } + if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + read += n; return n; } +bool FdSource::good() +{ + return _good; +} + + size_t StringSource::read(unsigned char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); @@ -144,56 +161,39 @@ void writePadding(size_t len, Sink & sink) } -void writeInt(unsigned int n, Sink & sink) -{ - unsigned char buf[8]; - memset(buf, 0, sizeof(buf)); - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - sink(buf, sizeof(buf)); -} - - -void writeLongLong(unsigned long long n, Sink & sink) -{ - unsigned char buf[8]; - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - buf[4] = (n >> 32) & 0xff; - buf[5] = (n >> 40) & 0xff; - buf[6] = (n >> 48) & 0xff; - buf[7] = (n >> 56) & 0xff; - sink(buf, sizeof(buf)); -} - - void writeString(const unsigned char * buf, size_t len, Sink & sink) { - writeInt(len, sink); + sink << len; sink(buf, len); writePadding(len, sink); } -void writeString(const string & s, Sink & sink) +Sink & operator << (Sink & sink, const string & s) { writeString((const unsigned char *) s.data(), s.size(), sink); + return sink; } template<class T> void writeStrings(const T & ss, Sink & sink) { - writeInt(ss.size(), sink); - foreach (typename T::const_iterator, i, ss) - writeString(*i, sink); + sink << ss.size(); + for (auto & i : ss) + sink << i; } -template void writeStrings(const Paths & ss, Sink & sink); -template void writeStrings(const PathSet & ss, Sink & sink); +Sink & operator << (Sink & sink, const Strings & s) +{ + writeStrings(s, sink); + return sink; +} + +Sink & operator << (Sink & sink, const StringSet & s) +{ + writeStrings(s, sink); + return sink; +} void readPadding(size_t len, Source & source) @@ -247,7 +247,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source) return len; } - + string readString(Source & source) { size_t len = readInt(source); @@ -258,7 +258,20 @@ string readString(Source & source) return string((char *) buf, len); } - +Source & operator >> (Source & in, string & s) +{ + s = readString(in); + return in; +} + + +Source & operator >> (Source & in, unsigned int & n) +{ + n = readInt(in); + return in; +} + + template<class T> T readStrings(Source & source) { unsigned int count = readInt(source); @@ -275,11 +288,11 @@ template PathSet readStrings(Source & source); void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; - if (!warned && s.size() > threshold) { + if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } - s.append((const char *) data, len); + s->append((const char *) data, len); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6a6f028aa652..892ec4aa36de 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,16 +1,23 @@ #pragma once #include "types.hh" +#include "util.hh" namespace nix { /* Abstract destination of binary data. */ -struct Sink +struct Sink { virtual ~Sink() { } virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual bool good() { return true; } + + void operator () (const std::string & s) + { + (*this)((const unsigned char *) s.data(), s.size()); + } }; @@ -24,10 +31,15 @@ struct BufferedSink : Sink : bufSize(bufSize), bufPos(0), buffer(0) { } ~BufferedSink(); - void operator () (const unsigned char * data, size_t len); - + void operator () (const unsigned char * data, size_t len) override; + + void operator () (const std::string & s) + { + Sink::operator()(s); + } + void flush(); - + virtual void write(const unsigned char * data, size_t len) = 0; }; @@ -36,7 +48,7 @@ struct BufferedSink : Sink struct Source { virtual ~Source() { } - + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. It blocks until all the requested data is available, or throws an error if it is not going to be available. */ @@ -46,6 +58,8 @@ struct Source return the number of bytes stored. If blocks until at least one byte is available. */ virtual size_t read(unsigned char * data, size_t len) = 0; + + virtual bool good() { return true; } }; @@ -58,9 +72,9 @@ struct BufferedSource : Source BufferedSource(size_t bufSize = 32 * 1024) : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } ~BufferedSource(); - - size_t read(unsigned char * data, size_t len); - + + size_t read(unsigned char * data, size_t len) override; + /* Underlying read call, to be overridden. */ virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; @@ -72,14 +86,19 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; - bool warn; - size_t written; + bool warn = false; + size_t written = 0; - FdSink() : fd(-1), warn(false), written(0) { } - FdSink(int fd) : fd(fd), warn(false), written(0) { } + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } ~FdSink(); - - void write(const unsigned char * data, size_t len); + + void write(const unsigned char * data, size_t len) override; + + bool good() override; + +private: + bool _good = true; }; @@ -87,17 +106,24 @@ struct FdSink : BufferedSink struct FdSource : BufferedSource { int fd; + size_t read = 0; + FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } - size_t readUnbuffered(unsigned char * data, size_t len); + size_t readUnbuffered(unsigned char * data, size_t len) override; + bool good() override; +private: + bool _good = true; }; /* A sink that writes data to a string. */ struct StringSink : Sink { - string s; - void operator () (const unsigned char * data, size_t len); + ref<std::string> s; + StringSink() : s(make_ref<std::string>()) { }; + StringSink(ref<std::string> s) : s(s) { }; + void operator () (const unsigned char * data, size_t len) override; }; @@ -107,16 +133,32 @@ struct StringSource : Source const string & s; size_t pos; StringSource(const string & _s) : s(_s), pos(0) { } - size_t read(unsigned char * data, size_t len); + size_t read(unsigned char * data, size_t len) override; }; void writePadding(size_t len, Sink & sink); -void writeInt(unsigned int n, Sink & sink); -void writeLongLong(unsigned long long n, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); -void writeString(const string & s, Sink & sink); -template<class T> void writeStrings(const T & ss, Sink & sink); + +inline Sink & operator << (Sink & sink, uint64_t n) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (n >> 56) & 0xff; + sink(buf, sizeof(buf)); + return sink; +} + +Sink & operator << (Sink & sink, const string & s); +Sink & operator << (Sink & sink, const Strings & s); +Sink & operator << (Sink & sink, const StringSet & s); + void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); @@ -125,6 +167,9 @@ size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); template<class T> T readStrings(Source & source); +Source & operator >> (Source & in, string & s); +Source & operator >> (Source & in, unsigned int & n); + MakeError(SerialisationError, Error) diff --git a/src/libutil/sha1.c b/src/libutil/sha1.c deleted file mode 100644 index d9d294d15540..000000000000 --- a/src/libutil/sha1.c +++ /dev/null @@ -1,369 +0,0 @@ -/* $Id$ */ - -/* sha.c - Implementation of the Secure Hash Algorithm - * - * Copyright (C) 1995, A.M. Kuchling - * - * Distribute and use freely; there are no restrictions on further - * dissemination and usage except those imposed by the laws of your - * country of residence. - * - * Adapted to pike and some cleanup by Niels Mller. - */ - -/* $Id$ */ - -/* SHA: NIST's Secure Hash Algorithm */ - -/* Based on SHA code originally posted to sci.crypt by Peter Gutmann - in message <30ajo5$oe8@ccu2.auckland.ac.nz>. - Modified to test for endianness on creation of SHA objects by AMK. - Also, the original specification of SHA was found to have a weakness - by NSA/NIST. This code implements the fixed version of SHA. -*/ - -/* Here's the first paragraph of Peter Gutmann's posting: - -The following is my SHA (FIPS 180) code updated to allow use of the "fixed" -SHA, thanks to Jim Gillogly and an anonymous contributor for the information on -what's changed in the new version. The fix is a simple change which involves -adding a single rotate in the initial expansion function. It is unknown -whether this is an optimal solution to the problem which was discovered in the -SHA or whether it's simply a bandaid which fixes the problem with a minimum of -effort (for example the reengineering of a great many Capstone chips). -*/ - -#include "sha1.h" - -#include <string.h> - -void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src) -{ - unsigned int i; - - dest->count_l=src->count_l; - dest->count_h=src->count_h; - for(i=0; i<SHA_DIGESTLEN; i++) - dest->digest[i]=src->digest[i]; - for(i=0; i < src->index; i++) - dest->block[i] = src->block[i]; - dest->index = src->index; -} - - -/* The SHA f()-functions. The f1 and f3 functions can be optimized to - save one boolean operation each - thanks to Rich Schroeppel, - rcs@cs.arizona.edu for discovering this */ - -/*#define f1(x,y,z) ( ( x & y ) | ( ~x & z ) ) // Rounds 0-19 */ -#define f1(x,y,z) ( z ^ ( x & ( y ^ z ) ) ) /* Rounds 0-19 */ -#define f2(x,y,z) ( x ^ y ^ z ) /* Rounds 20-39 */ -/*#define f3(x,y,z) ( ( x & y ) | ( x & z ) | ( y & z ) ) // Rounds 40-59 */ -#define f3(x,y,z) ( ( x & y ) | ( z & ( x | y ) ) ) /* Rounds 40-59 */ -#define f4(x,y,z) ( x ^ y ^ z ) /* Rounds 60-79 */ - -/* The SHA Mysterious Constants */ - -#define K1 0x5A827999L /* Rounds 0-19 */ -#define K2 0x6ED9EBA1L /* Rounds 20-39 */ -#define K3 0x8F1BBCDCL /* Rounds 40-59 */ -#define K4 0xCA62C1D6L /* Rounds 60-79 */ - -/* SHA initial values */ - -#define h0init 0x67452301L -#define h1init 0xEFCDAB89L -#define h2init 0x98BADCFEL -#define h3init 0x10325476L -#define h4init 0xC3D2E1F0L - -/* 32-bit rotate left - kludged with shifts */ - -#define ROTL(n,X) ( ( (X) << (n) ) | ( (X) >> ( 32 - (n) ) ) ) - -/* The initial expanding function. The hash function is defined over an - 80-word expanded input array W, where the first 16 are copies of the input - data, and the remaining 64 are defined by - - W[ i ] = W[ i - 16 ] ^ W[ i - 14 ] ^ W[ i - 8 ] ^ W[ i - 3 ] - - This implementation generates these values on the fly in a circular - buffer - thanks to Colin Plumb, colin@nyx10.cs.du.edu for this - optimization. - - The updated SHA changes the expanding function by adding a rotate of 1 - bit. Thanks to Jim Gillogly, jim@rand.org, and an anonymous contributor - for this information */ - -#define expand(W,i) ( W[ i & 15 ] = \ - ROTL( 1, ( W[ i & 15 ] ^ W[ (i - 14) & 15 ] ^ \ - W[ (i - 8) & 15 ] ^ W[ (i - 3) & 15 ] ) ) ) - - -/* The prototype SHA sub-round. The fundamental sub-round is: - - a' = e + ROTL( 5, a ) + f( b, c, d ) + k + data; - b' = a; - c' = ROTL( 30, b ); - d' = c; - e' = d; - - but this is implemented by unrolling the loop 5 times and renaming the - variables ( e, a, b, c, d ) = ( a', b', c', d', e' ) each iteration. - This code is then replicated 20 times for each of the 4 functions, using - the next 20 values from the W[] array each time */ - -#define subRound(a, b, c, d, e, f, k, data) \ - ( e += ROTL( 5, a ) + f( b, c, d ) + k + data, b = ROTL( 30, b ) ) - -/* Initialize the SHA values */ - -void SHA1_Init(struct SHA_CTX *ctx) -{ - /* Set the h-vars to their initial values */ - ctx->digest[ 0 ] = h0init; - ctx->digest[ 1 ] = h1init; - ctx->digest[ 2 ] = h2init; - ctx->digest[ 3 ] = h3init; - ctx->digest[ 4 ] = h4init; - - /* Initialize bit count */ - ctx->count_l = ctx->count_h = 0; - - /* Initialize buffer */ - ctx->index = 0; -} - -/* Perform the SHA transformation. Note that this code, like MD5, seems to - break some optimizing compilers due to the complexity of the expressions - and the size of the basic block. It may be necessary to split it into - sections, e.g. based on the four subrounds - - Note that this function destroys the data area */ - -static void sha_transform(struct SHA_CTX *ctx, uint32_t *data ) -{ - uint32_t A, B, C, D, E; /* Local vars */ - - /* Set up first buffer and local data buffer */ - A = ctx->digest[0]; - B = ctx->digest[1]; - C = ctx->digest[2]; - D = ctx->digest[3]; - E = ctx->digest[4]; - - /* Heavy mangling, in 4 sub-rounds of 20 interations each. */ - subRound( A, B, C, D, E, f1, K1, data[ 0] ); - subRound( E, A, B, C, D, f1, K1, data[ 1] ); - subRound( D, E, A, B, C, f1, K1, data[ 2] ); - subRound( C, D, E, A, B, f1, K1, data[ 3] ); - subRound( B, C, D, E, A, f1, K1, data[ 4] ); - subRound( A, B, C, D, E, f1, K1, data[ 5] ); - subRound( E, A, B, C, D, f1, K1, data[ 6] ); - subRound( D, E, A, B, C, f1, K1, data[ 7] ); - subRound( C, D, E, A, B, f1, K1, data[ 8] ); - subRound( B, C, D, E, A, f1, K1, data[ 9] ); - subRound( A, B, C, D, E, f1, K1, data[10] ); - subRound( E, A, B, C, D, f1, K1, data[11] ); - subRound( D, E, A, B, C, f1, K1, data[12] ); - subRound( C, D, E, A, B, f1, K1, data[13] ); - subRound( B, C, D, E, A, f1, K1, data[14] ); - subRound( A, B, C, D, E, f1, K1, data[15] ); - subRound( E, A, B, C, D, f1, K1, expand( data, 16 ) ); - subRound( D, E, A, B, C, f1, K1, expand( data, 17 ) ); - subRound( C, D, E, A, B, f1, K1, expand( data, 18 ) ); - subRound( B, C, D, E, A, f1, K1, expand( data, 19 ) ); - - subRound( A, B, C, D, E, f2, K2, expand( data, 20 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 21 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 22 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 23 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 24 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 25 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 26 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 27 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 28 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 29 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 30 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 31 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 32 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 33 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 34 ) ); - subRound( A, B, C, D, E, f2, K2, expand( data, 35 ) ); - subRound( E, A, B, C, D, f2, K2, expand( data, 36 ) ); - subRound( D, E, A, B, C, f2, K2, expand( data, 37 ) ); - subRound( C, D, E, A, B, f2, K2, expand( data, 38 ) ); - subRound( B, C, D, E, A, f2, K2, expand( data, 39 ) ); - - subRound( A, B, C, D, E, f3, K3, expand( data, 40 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 41 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 42 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 43 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 44 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 45 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 46 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 47 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 48 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 49 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 50 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 51 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 52 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 53 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 54 ) ); - subRound( A, B, C, D, E, f3, K3, expand( data, 55 ) ); - subRound( E, A, B, C, D, f3, K3, expand( data, 56 ) ); - subRound( D, E, A, B, C, f3, K3, expand( data, 57 ) ); - subRound( C, D, E, A, B, f3, K3, expand( data, 58 ) ); - subRound( B, C, D, E, A, f3, K3, expand( data, 59 ) ); - - subRound( A, B, C, D, E, f4, K4, expand( data, 60 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 61 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 62 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 63 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 64 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 65 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 66 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 67 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 68 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 69 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 70 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 71 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 72 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 73 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 74 ) ); - subRound( A, B, C, D, E, f4, K4, expand( data, 75 ) ); - subRound( E, A, B, C, D, f4, K4, expand( data, 76 ) ); - subRound( D, E, A, B, C, f4, K4, expand( data, 77 ) ); - subRound( C, D, E, A, B, f4, K4, expand( data, 78 ) ); - subRound( B, C, D, E, A, f4, K4, expand( data, 79 ) ); - - /* Build message digest */ - ctx->digest[0] += A; - ctx->digest[1] += B; - ctx->digest[2] += C; - ctx->digest[3] += D; - ctx->digest[4] += E; -} - -#if 1 - -#ifndef EXTRACT_UCHAR -#define EXTRACT_UCHAR(p) (*(unsigned char *)(p)) -#endif - -#define STRING2INT(s) ((((((EXTRACT_UCHAR(s) << 8) \ - | EXTRACT_UCHAR(s+1)) << 8) \ - | EXTRACT_UCHAR(s+2)) << 8) \ - | EXTRACT_UCHAR(s+3)) -#else -uint32_t STRING2INT(unsigned char *s) -{ - uint32_t r; - unsigned int i; - - for (i = 0, r = 0; i < 4; i++, s++) - r = (r << 8) | *s; - return r; -} -#endif - -static void sha_block(struct SHA_CTX *ctx, const unsigned char *block) -{ - uint32_t data[SHA_DATALEN]; - unsigned int i; - - /* Update block count */ - if (!++ctx->count_l) - ++ctx->count_h; - - /* Endian independent conversion */ - for (i = 0; i<SHA_DATALEN; i++, block += 4) - data[i] = STRING2INT(block); - - sha_transform(ctx, data); -} - -void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len) -{ - if (ctx->index) - { /* Try to fill partial block */ - unsigned left = SHA_DATASIZE - ctx->index; - if (len < left) - { - memcpy(ctx->block + ctx->index, buffer, len); - ctx->index += len; - return; /* Finished */ - } - else - { - memcpy(ctx->block + ctx->index, buffer, left); - sha_block(ctx, ctx->block); - buffer += left; - len -= left; - } - } - while (len >= SHA_DATASIZE) - { - sha_block(ctx, buffer); - buffer += SHA_DATASIZE; - len -= SHA_DATASIZE; - } - if ((ctx->index = len)) /* This assignment is intended */ - /* Buffer leftovers */ - memcpy(ctx->block, buffer, len); -} - -/* Final wrapup - pad to SHA_DATASIZE-byte boundary with the bit pattern - 1 0* (64-bit count of bits processed, MSB-first) */ - -void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx) -{ - uint32_t data[SHA_DATALEN]; - unsigned int i; - unsigned int words; - - i = ctx->index; - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - ctx->block[i++] = 0x80; - - /* Fill rest of word */ - for( ; i & 3; i++) - ctx->block[i] = 0; - - /* i is now a multiple of the word size 4 */ - words = i >> 2; - for (i = 0; i < words; i++) - data[i] = STRING2INT(ctx->block + 4*i); - - if (words > (SHA_DATALEN-2)) - { /* No room for length in this block. Process it and - * pad with another one */ - for (i = words ; i < SHA_DATALEN; i++) - data[i] = 0; - sha_transform(ctx, data); - for (i = 0; i < (SHA_DATALEN-2); i++) - data[i] = 0; - } - else - for (i = words ; i < SHA_DATALEN - 2; i++) - data[i] = 0; - /* Theres 512 = 2^9 bits in one block */ - data[SHA_DATALEN-2] = (ctx->count_h << 9) | (ctx->count_l >> 23); - data[SHA_DATALEN-1] = (ctx->count_l << 9) | (ctx->index << 3); - sha_transform(ctx, data); - sha_digest(ctx, s); -} - -void sha_digest(struct SHA_CTX *ctx, unsigned char *s) -{ - unsigned int i; - - for (i = 0; i < SHA_DIGESTLEN; i++) - { - *s++ = ctx->digest[i] >> 24; - *s++ = 0xff & (ctx->digest[i] >> 16); - *s++ = 0xff & (ctx->digest[i] >> 8); - *s++ = 0xff & ctx->digest[i]; - } -} diff --git a/src/libutil/sha1.h b/src/libutil/sha1.h deleted file mode 100644 index 715040dd48df..000000000000 --- a/src/libutil/sha1.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _SHA_H -#define _SHA_H - -#include <inttypes.h> - -/* The SHA block size and message digest sizes, in bytes */ - -#define SHA_DATASIZE 64 -#define SHA_DATALEN 16 -#define SHA_DIGESTSIZE 20 -#define SHA_DIGESTLEN 5 -/* The structure for storing SHA info */ - -struct SHA_CTX { - uint32_t digest[SHA_DIGESTLEN]; /* Message digest */ - uint32_t count_l, count_h; /* 64-bit block count */ - uint8_t block[SHA_DATASIZE]; /* SHA data buffer */ - unsigned int index; /* index into buffer */ -}; - -void SHA1_Init(struct SHA_CTX *ctx); -void SHA1_Update(struct SHA_CTX *ctx, const unsigned char *buffer, uint32_t len); -void SHA1_Final(unsigned char *s, struct SHA_CTX *ctx); -void sha_digest(struct SHA_CTX *ctx, unsigned char *s); -void sha_copy(struct SHA_CTX *dest, struct SHA_CTX *src); - - -#endif /* !_SHA_H */ diff --git a/src/libutil/sha256.c b/src/libutil/sha256.c deleted file mode 100644 index 63ed0ba43011..000000000000 --- a/src/libutil/sha256.c +++ /dev/null @@ -1,238 +0,0 @@ -/* crypto/sha/sha256.c */ -/* ==================================================================== - * Copyright (c) 2004 The OpenSSL Project. All rights reserved - * according to the OpenSSL license [found in ./md32_common.h]. - * ==================================================================== - */ - -#include <stdlib.h> -#include <string.h> - -#include "sha256.h" - -int SHA224_Init (SHA256_CTX *c) - { - c->h[0]=0xc1059ed8UL; c->h[1]=0x367cd507UL; - c->h[2]=0x3070dd17UL; c->h[3]=0xf70e5939UL; - c->h[4]=0xffc00b31UL; c->h[5]=0x68581511UL; - c->h[6]=0x64f98fa7UL; c->h[7]=0xbefa4fa4UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA224_DIGEST_LENGTH; - return 1; - } - -int SHA256_Init (SHA256_CTX *c) - { - c->h[0]=0x6a09e667UL; c->h[1]=0xbb67ae85UL; - c->h[2]=0x3c6ef372UL; c->h[3]=0xa54ff53aUL; - c->h[4]=0x510e527fUL; c->h[5]=0x9b05688cUL; - c->h[6]=0x1f83d9abUL; c->h[7]=0x5be0cd19UL; - c->Nl=0; c->Nh=0; - c->num=0; c->md_len=SHA256_DIGEST_LENGTH; - return 1; - } - -unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md) - { - SHA256_CTX c; - static unsigned char m[SHA224_DIGEST_LENGTH]; - - if (md == NULL) md=m; - SHA224_Init(&c); - SHA256_Update(&c,d,n); - SHA256_Final(md,&c); - return(md); - } - -unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md) - { - SHA256_CTX c; - static unsigned char m[SHA256_DIGEST_LENGTH]; - - if (md == NULL) md=m; - SHA256_Init(&c); - SHA256_Update(&c,d,n); - SHA256_Final(md,&c); - return(md); - } - -int SHA224_Update(SHA256_CTX *c, const void *data, size_t len) -{ return SHA256_Update (c,data,len); } -int SHA224_Final (unsigned char *md, SHA256_CTX *c) -{ return SHA256_Final (md,c); } - -#define DATA_ORDER_IS_BIG_ENDIAN - -#define HASH_LONG uint32_t -#define HASH_LONG_LOG2 2 -#define HASH_CTX SHA256_CTX -#define HASH_CBLOCK SHA_CBLOCK -#define HASH_LBLOCK SHA_LBLOCK -/* - * Note that FIPS180-2 discusses "Truncation of the Hash Function Output." - * default: case below covers for it. It's not clear however if it's - * permitted to truncate to amount of bytes not divisible by 4. I bet not, - * but if it is, then default: case shall be extended. For reference. - * Idea behind separate cases for pre-defined lenghts is to let the - * compiler decide if it's appropriate to unroll small loops. - */ -#define HASH_MAKE_STRING(c,s) do { \ - unsigned long ll; \ - unsigned int n; \ - switch ((c)->md_len) \ - { case SHA224_DIGEST_LENGTH: \ - for (n=0;n<SHA224_DIGEST_LENGTH/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - case SHA256_DIGEST_LENGTH: \ - for (n=0;n<SHA256_DIGEST_LENGTH/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - default: \ - if ((c)->md_len > SHA256_DIGEST_LENGTH) \ - return 0; \ - for (n=0;n<(c)->md_len/4;n++) \ - { ll=(c)->h[n]; HOST_l2c(ll,(s)); } \ - break; \ - } \ - } while (0) - -#define HASH_UPDATE SHA256_Update -#define HASH_TRANSFORM SHA256_Transform -#define HASH_FINAL SHA256_Final -#define HASH_BLOCK_HOST_ORDER sha256_block_host_order -#define HASH_BLOCK_DATA_ORDER sha256_block_data_order -void sha256_block_host_order (SHA256_CTX *ctx, const void *in, size_t num); -void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num); - -#include "md32_common.h" - -static const uint32_t K256[64] = { - 0x428a2f98UL,0x71374491UL,0xb5c0fbcfUL,0xe9b5dba5UL, - 0x3956c25bUL,0x59f111f1UL,0x923f82a4UL,0xab1c5ed5UL, - 0xd807aa98UL,0x12835b01UL,0x243185beUL,0x550c7dc3UL, - 0x72be5d74UL,0x80deb1feUL,0x9bdc06a7UL,0xc19bf174UL, - 0xe49b69c1UL,0xefbe4786UL,0x0fc19dc6UL,0x240ca1ccUL, - 0x2de92c6fUL,0x4a7484aaUL,0x5cb0a9dcUL,0x76f988daUL, - 0x983e5152UL,0xa831c66dUL,0xb00327c8UL,0xbf597fc7UL, - 0xc6e00bf3UL,0xd5a79147UL,0x06ca6351UL,0x14292967UL, - 0x27b70a85UL,0x2e1b2138UL,0x4d2c6dfcUL,0x53380d13UL, - 0x650a7354UL,0x766a0abbUL,0x81c2c92eUL,0x92722c85UL, - 0xa2bfe8a1UL,0xa81a664bUL,0xc24b8b70UL,0xc76c51a3UL, - 0xd192e819UL,0xd6990624UL,0xf40e3585UL,0x106aa070UL, - 0x19a4c116UL,0x1e376c08UL,0x2748774cUL,0x34b0bcb5UL, - 0x391c0cb3UL,0x4ed8aa4aUL,0x5b9cca4fUL,0x682e6ff3UL, - 0x748f82eeUL,0x78a5636fUL,0x84c87814UL,0x8cc70208UL, - 0x90befffaUL,0xa4506cebUL,0xbef9a3f7UL,0xc67178f2UL }; - -/* - * FIPS specification refers to right rotations, while our ROTATE macro - * is left one. This is why you might notice that rotation coefficients - * differ from those observed in FIPS document by 32-N... - */ -#define Sigma0(x) (ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10)) -#define Sigma1(x) (ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7)) -#define sigma0(x) (ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3)) -#define sigma1(x) (ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10)) - -#define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) -#define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) - -#define ROUND_00_15(i,a,b,c,d,e,f,g,h) do { \ - T1 += h + Sigma1(e) + Ch(e,f,g) + K256[i]; \ - h = Sigma0(a) + Maj(a,b,c); \ - d += T1; h += T1; } while (0) - -#define ROUND_16_63(i,a,b,c,d,e,f,g,h,X) do { \ - s0 = X[(i+1)&0x0f]; s0 = sigma0(s0); \ - s1 = X[(i+14)&0x0f]; s1 = sigma1(s1); \ - T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f]; \ - ROUND_00_15(i,a,b,c,d,e,f,g,h); } while (0) - -static void sha256_block (SHA256_CTX *ctx, const void *in, size_t num, int host) - { - uint32_t a,b,c,d,e,f,g,h,s0,s1,T1; - uint32_t X[16]; - int i; - const unsigned char *data=in; - - while (num--) { - - a = ctx->h[0]; b = ctx->h[1]; c = ctx->h[2]; d = ctx->h[3]; - e = ctx->h[4]; f = ctx->h[5]; g = ctx->h[6]; h = ctx->h[7]; - - if (host) - { - const uint32_t *W=(const uint32_t *)data; - - T1 = X[0] = W[0]; ROUND_00_15(0,a,b,c,d,e,f,g,h); - T1 = X[1] = W[1]; ROUND_00_15(1,h,a,b,c,d,e,f,g); - T1 = X[2] = W[2]; ROUND_00_15(2,g,h,a,b,c,d,e,f); - T1 = X[3] = W[3]; ROUND_00_15(3,f,g,h,a,b,c,d,e); - T1 = X[4] = W[4]; ROUND_00_15(4,e,f,g,h,a,b,c,d); - T1 = X[5] = W[5]; ROUND_00_15(5,d,e,f,g,h,a,b,c); - T1 = X[6] = W[6]; ROUND_00_15(6,c,d,e,f,g,h,a,b); - T1 = X[7] = W[7]; ROUND_00_15(7,b,c,d,e,f,g,h,a); - T1 = X[8] = W[8]; ROUND_00_15(8,a,b,c,d,e,f,g,h); - T1 = X[9] = W[9]; ROUND_00_15(9,h,a,b,c,d,e,f,g); - T1 = X[10] = W[10]; ROUND_00_15(10,g,h,a,b,c,d,e,f); - T1 = X[11] = W[11]; ROUND_00_15(11,f,g,h,a,b,c,d,e); - T1 = X[12] = W[12]; ROUND_00_15(12,e,f,g,h,a,b,c,d); - T1 = X[13] = W[13]; ROUND_00_15(13,d,e,f,g,h,a,b,c); - T1 = X[14] = W[14]; ROUND_00_15(14,c,d,e,f,g,h,a,b); - T1 = X[15] = W[15]; ROUND_00_15(15,b,c,d,e,f,g,h,a); - - data += SHA256_CBLOCK; - } - else - { - uint32_t l; - - HOST_c2l(data,l); T1 = X[0] = l; ROUND_00_15(0,a,b,c,d,e,f,g,h); - HOST_c2l(data,l); T1 = X[1] = l; ROUND_00_15(1,h,a,b,c,d,e,f,g); - HOST_c2l(data,l); T1 = X[2] = l; ROUND_00_15(2,g,h,a,b,c,d,e,f); - HOST_c2l(data,l); T1 = X[3] = l; ROUND_00_15(3,f,g,h,a,b,c,d,e); - HOST_c2l(data,l); T1 = X[4] = l; ROUND_00_15(4,e,f,g,h,a,b,c,d); - HOST_c2l(data,l); T1 = X[5] = l; ROUND_00_15(5,d,e,f,g,h,a,b,c); - HOST_c2l(data,l); T1 = X[6] = l; ROUND_00_15(6,c,d,e,f,g,h,a,b); - HOST_c2l(data,l); T1 = X[7] = l; ROUND_00_15(7,b,c,d,e,f,g,h,a); - HOST_c2l(data,l); T1 = X[8] = l; ROUND_00_15(8,a,b,c,d,e,f,g,h); - HOST_c2l(data,l); T1 = X[9] = l; ROUND_00_15(9,h,a,b,c,d,e,f,g); - HOST_c2l(data,l); T1 = X[10] = l; ROUND_00_15(10,g,h,a,b,c,d,e,f); - HOST_c2l(data,l); T1 = X[11] = l; ROUND_00_15(11,f,g,h,a,b,c,d,e); - HOST_c2l(data,l); T1 = X[12] = l; ROUND_00_15(12,e,f,g,h,a,b,c,d); - HOST_c2l(data,l); T1 = X[13] = l; ROUND_00_15(13,d,e,f,g,h,a,b,c); - HOST_c2l(data,l); T1 = X[14] = l; ROUND_00_15(14,c,d,e,f,g,h,a,b); - HOST_c2l(data,l); T1 = X[15] = l; ROUND_00_15(15,b,c,d,e,f,g,h,a); - } - - for (i=16;i<64;i+=8) - { - ROUND_16_63(i+0,a,b,c,d,e,f,g,h,X); - ROUND_16_63(i+1,h,a,b,c,d,e,f,g,X); - ROUND_16_63(i+2,g,h,a,b,c,d,e,f,X); - ROUND_16_63(i+3,f,g,h,a,b,c,d,e,X); - ROUND_16_63(i+4,e,f,g,h,a,b,c,d,X); - ROUND_16_63(i+5,d,e,f,g,h,a,b,c,X); - ROUND_16_63(i+6,c,d,e,f,g,h,a,b,X); - ROUND_16_63(i+7,b,c,d,e,f,g,h,a,X); - } - - ctx->h[0] += a; ctx->h[1] += b; ctx->h[2] += c; ctx->h[3] += d; - ctx->h[4] += e; ctx->h[5] += f; ctx->h[6] += g; ctx->h[7] += h; - - } - } - -/* - * Idea is to trade couple of cycles for some space. On IA-32 we save - * about 4K in "big footprint" case. In "small footprint" case any gain - * is appreciated:-) - */ -void HASH_BLOCK_HOST_ORDER (SHA256_CTX *ctx, const void *in, size_t num) -{ sha256_block (ctx,in,num,1); } - -void HASH_BLOCK_DATA_ORDER (SHA256_CTX *ctx, const void *in, size_t num) -{ sha256_block (ctx,in,num,0); } - - diff --git a/src/libutil/sha256.h b/src/libutil/sha256.h deleted file mode 100644 index 0686b84f0e08..000000000000 --- a/src/libutil/sha256.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef _SHA256_H -#define _SHA256_H 1 - -#include <inttypes.h> - -#define SHA_LBLOCK 16 -#define SHA_CBLOCK (SHA_LBLOCK*4) /* SHA treats input data as a - * contiguous array of 32 bit - * wide big-endian values. */ - -#define SHA256_CBLOCK (SHA_LBLOCK*4) /* SHA-256 treats input data as a - * contiguous array of 32 bit - * wide big-endian values. */ -#define SHA224_DIGEST_LENGTH 28 -#define SHA256_DIGEST_LENGTH 32 - -typedef struct SHA256state_st - { - uint32_t h[8]; - uint32_t Nl,Nh; - uint32_t data[SHA_LBLOCK]; - unsigned int num,md_len; - } SHA256_CTX; - -int SHA224_Init(SHA256_CTX *c); -int SHA224_Update(SHA256_CTX *c, const void *data, size_t len); -int SHA224_Final(unsigned char *md, SHA256_CTX *c); -unsigned char *SHA224(const unsigned char *d, size_t n,unsigned char *md); -int SHA256_Init(SHA256_CTX *c); -int SHA256_Update(SHA256_CTX *c, const void *data, size_t len); -int SHA256_Final(unsigned char *md, SHA256_CTX *c); -unsigned char *SHA256(const unsigned char *d, size_t n,unsigned char *md); -void SHA256_Transform(SHA256_CTX *c, const unsigned char *data); - -#endif diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh new file mode 100644 index 000000000000..ebe64ffbdab7 --- /dev/null +++ b/src/libutil/sync.hh @@ -0,0 +1,78 @@ +#pragma once + +#include <mutex> +#include <condition_variable> +#include <cassert> + +namespace nix { + +/* This template class ensures synchronized access to a value of type + T. It is used as follows: + + struct Data { int x; ... }; + + Sync<Data> data; + + { + auto data_(data.lock()); + data_->x = 123; + } + + Here, "data" is automatically unlocked when "data_" goes out of + scope. +*/ + +template<class T, class M = std::mutex> +class Sync +{ +private: + M mutex; + T data; + +public: + + Sync() { } + Sync(const T & data) : data(data) { } + + class Lock + { + private: + Sync * s; + std::unique_lock<M> lk; + friend Sync; + Lock(Sync * s) : s(s), lk(s->mutex) { } + public: + Lock(Lock && l) : s(l.s) { abort(); } + Lock(const Lock & l) = delete; + ~Lock() { } + T * operator -> () { return &s->data; } + T & operator * () { return s->data; } + + void wait(std::condition_variable & cv) + { + assert(s); + cv.wait(lk); + } + + template<class Rep, class Period, class Predicate> + bool wait_for(std::condition_variable & cv, + const std::chrono::duration<Rep, Period> & duration, + Predicate pred) + { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template<class Clock, class Duration> + std::cv_status wait_until(std::condition_variable & cv, + const std::chrono::time_point<Clock, Duration> & duration) + { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } +}; + +} diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc new file mode 100644 index 000000000000..32363ecf0098 --- /dev/null +++ b/src/libutil/thread-pool.cc @@ -0,0 +1,102 @@ +#include "thread-pool.hh" +#include "affinity.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _maxThreads) + : maxThreads(_maxThreads) +{ + restoreAffinity(); // FIXME + + if (!maxThreads) { + maxThreads = std::thread::hardware_concurrency(); + if (!maxThreads) maxThreads = 1; + } + + debug(format("starting pool of %d threads") % maxThreads); +} + +ThreadPool::~ThreadPool() +{ + std::vector<std::thread> workers; + { + auto state(state_.lock()); + state->quit = true; + std::swap(workers, state->workers); + } + + debug(format("reaping %d worker threads") % workers.size()); + + work.notify_all(); + + for (auto & thr : workers) + thr.join(); +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state(state_.lock()); + assert(!state->quit); + state->left.push(t); + if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads) + state->workers.emplace_back(&ThreadPool::workerEntry, this); + work.notify_one(); +} + +void ThreadPool::process() +{ + while (true) { + auto state(state_.lock()); + if (state->exception) + std::rethrow_exception(state->exception); + if (state->left.empty() && !state->pending) break; + state.wait(done); + } +} + +void ThreadPool::workerEntry() +{ + bool didWork = false; + + while (true) { + work_t w; + { + auto state(state_.lock()); + while (true) { + if (state->quit || state->exception) return; + if (didWork) { + assert(state->pending); + state->pending--; + didWork = false; + } + if (!state->left.empty()) break; + if (!state->pending) + done.notify_all(); + state.wait(work); + } + w = state->left.front(); + state->left.pop(); + state->pending++; + } + + try { + w(); + } catch (std::exception & e) { + auto state(state_.lock()); + if (state->exception) { + if (!dynamic_cast<Interrupted*>(&e)) + printMsg(lvlError, format("error: %s") % e.what()); + } else { + state->exception = std::current_exception(); + work.notify_all(); + done.notify_all(); + } + } + + didWork = true; + } +} + +} + + diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh new file mode 100644 index 000000000000..78b63467d62e --- /dev/null +++ b/src/libutil/thread-pool.hh @@ -0,0 +1,117 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include <queue> +#include <functional> +#include <thread> +#include <map> + +namespace nix { + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t maxThreads = 0); + + ~ThreadPool(); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t maxThreads; + + struct State + { + std::queue<work_t> left; + size_t pending = 0; + std::exception_ptr exception; + std::vector<std::thread> workers; + bool quit = false; + }; + + Sync<State> state_; + + std::condition_variable work, done; + + void workerEntry(); +}; + +/* Process in parallel a set of items of type T that have a partial + ordering between them. Thus, any item is only processed after all + its dependencies have been processed. */ +template<typename T> +void processGraph( + ThreadPool & pool, + const std::set<T> & nodes, + std::function<std::set<T>(const T &)> getEdges, + std::function<void(const T &)> processNode) +{ + struct Graph { + std::set<T> left; + std::map<T, std::set<T>> refs, rrefs; + std::function<void(T)> wrap; + }; + + ref<Sync<Graph>> graph_ = make_ref<Sync<Graph>>(); + + auto wrapWork = [&pool, graph_, processNode](const T & node) { + processNode(node); + + /* Enqueue work for all nodes that were waiting on this one. */ + { + auto graph(graph_->lock()); + graph->left.erase(node); + for (auto & rref : graph->rrefs[node]) { + auto & refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) + pool.enqueue(std::bind(graph->wrap, rref)); + } + } + }; + + { + auto graph(graph_->lock()); + graph->left = nodes; + graph->wrap = wrapWork; + } + + /* Build the dependency graph; enqueue all nodes with no + dependencies. */ + for (auto & node : nodes) { + auto refs = getEdges(node); + { + auto graph(graph_->lock()); + for (auto & ref : refs) + if (ref != node && graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + pool.enqueue(std::bind(graph->wrap, node)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 160884ee1ad7..bd192b8506b2 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -2,9 +2,12 @@ #include "config.h" +#include "ref.hh" + #include <string> #include <list> #include <set> +#include <memory> #include <boost/format.hpp> @@ -86,14 +89,4 @@ typedef list<Path> Paths; typedef set<Path> PathSet; -typedef enum { - lvlError = 0, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - - } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 596b79e10e69..67558cc0b33c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -149,10 +149,20 @@ Path dirOf(const Path & path) string baseNameOf(const Path & path) { - Path::size_type pos = path.rfind('/'); + if (path.empty()) + return ""; + + Path::size_type last = path.length() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + Path::size_type pos = path.rfind('/', last); if (pos == string::npos) - throw Error(format("invalid file name ‘%1%’") % path); - return string(path, pos + 1); + pos = 0; + else + pos += 1; + + return string(path, pos, last - pos + 1); } @@ -199,7 +209,7 @@ Path readLink(const Path & path) else if (rlsize > st.st_size) throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%") % path % rlsize % st.st_size); - return string(buf, st.st_size); + return string(buf, rlsize); } @@ -223,7 +233,13 @@ DirEntries readDirectory(const Path & path) checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, dirent->d_type); + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); } if (errno) throw SysError(format("reading directory ‘%1%’") % path); @@ -304,9 +320,11 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); - printMsg(lvlVomit, format("%1%") % path); - - struct stat st = lstat(path); + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + if (errno == ENOENT) return; + throw SysError(format("getting status of ‘%1%’") % path); + } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) bytesFreed += st.st_blocks * 512; @@ -322,8 +340,10 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) _deletePath(path + "/" + i.name, bytesFreed); } - if (remove(path.c_str()) == -1) + if (remove(path.c_str()) == -1) { + if (errno == ENOENT) return; throw SysError(format("cannot unlink ‘%1%’") % path); + } } @@ -336,8 +356,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - startNest(nest, lvlDebug, - format("recursively deleting path ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -383,6 +402,18 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +Path getCacheDir() +{ + Path cacheDir = getEnv("XDG_CACHE_HOME"); + if (cacheDir.empty()) { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set"); + cacheDir = homeDir + "/.cache"; + } + return cacheDir; +} + + Paths createDirs(const Path & path) { Paths created; @@ -424,101 +455,6 @@ void replaceSymlink(const Path & target, const Path & link) } -LogType logType = ltPretty; -Verbosity verbosity = lvlInfo; - -static int nestingLevel = 0; - - -Nest::Nest() -{ - nest = false; -} - - -Nest::~Nest() -{ - close(); -} - - -static string escVerbosity(Verbosity level) -{ - return int2String((int) level); -} - - -void Nest::open(Verbosity level, const FormatOrString & fs) -{ - if (level <= verbosity) { - if (logType == ltEscapes) - std::cerr << "\033[" << escVerbosity(level) << "p" - << fs.s << "\n"; - else - printMsg_(level, fs); - nest = true; - nestingLevel++; - } -} - - -void Nest::close() -{ - if (nest) { - nestingLevel--; - if (logType == ltEscapes) - std::cerr << "\033[q"; - nest = false; - } -} - - -void printMsg_(Verbosity level, const FormatOrString & fs) -{ - checkInterrupt(); - if (level > verbosity) return; - string prefix; - if (logType == ltPretty) - for (int i = 0; i < nestingLevel; i++) - prefix += "| "; - else if (logType == ltEscapes && level != lvlInfo) - prefix = "\033[" + escVerbosity(level) + "s"; - string s = (format("%1%%2%\n") % prefix % fs.s).str(); - if (!isatty(STDERR_FILENO)) s = filterANSIEscapes(s); - writeToStderr(s); -} - - -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - printMsg(lvlError, format("warning: %1%") % fs.s); - haveWarned = true; - } -} - - -void writeToStderr(const string & s) -{ - try { - if (_writeToStderr) - _writeToStderr((const unsigned char *) s.data(), s.size()); - else - writeFull(STDERR_FILENO, s); - } catch (SysError & e) { - /* Ignore failing writes to stderr if we're in an exception - handler, otherwise throw an exception. We need to ignore - write errors in exception handlers to ensure that cleanup - code runs to completion if the other side of stderr has - been closed unexpectedly. */ - if (!std::uncaught_exception()) throw; - } -} - - -void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; - - void readFull(int fd, unsigned char * buf, size_t count) { while (count) { @@ -578,6 +514,8 @@ string drainFD(int fd) ////////////////////////////////////////////////////////////////////// +AutoDelete::AutoDelete() : del{false} {} + AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) { del = true; @@ -605,6 +543,12 @@ void AutoDelete::cancel() del = false; } +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + ////////////////////////////////////////////////////////////////////// @@ -901,7 +845,8 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { auto wrapper = [&]() { - if (!options.allowVfork) _writeToStderr = 0; + if (!options.allowVfork) + logger = makeDefaultLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -1022,13 +967,15 @@ void restoreSIGPIPE() volatile sig_atomic_t _isInterrupted = 0; +thread_local bool interruptThrown = false; + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!std::uncaught_exception()) { - _isInterrupted = 0; + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; throw Interrupted("interrupted by the user"); } } @@ -1060,9 +1007,9 @@ template vector<string> tokenizeString(const string & s, const string & separato string concatStringsSep(const string & sep, const Strings & ss) { string s; - foreach (Strings::const_iterator, i, ss) { + for (auto & i : ss) { if (s.size() != 0) s += sep; - s += *i; + s += i; } return s; } @@ -1071,9 +1018,9 @@ string concatStringsSep(const string & sep, const Strings & ss) string concatStringsSep(const string & sep, const StringSet & ss) { string s; - foreach (StringSet::const_iterator, i, ss) { + for (auto & i : ss) { if (s.size() != 0) s += sep; - s += *i; + s += i; } return s; } @@ -1135,6 +1082,12 @@ bool statusOk(int status) } +bool hasPrefix(const string & s, const string & suffix) +{ + return s.compare(0, suffix.size(), suffix) == 0; +} + + bool hasSuffix(const string & s, const string & suffix) { return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 187e05ece050..ab43637a574c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <sys/types.h> #include <sys/stat.h> @@ -8,20 +9,19 @@ #include <unistd.h> #include <signal.h> #include <functional> - +#include <limits> #include <cstdio> +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif namespace nix { -#define foreach(it_type, it, collection) \ - for (it_type it = (collection).begin(); it != (collection).end(); ++it) - -#define foreach_reverse(it_type, it, collection) \ - for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) - - /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); @@ -93,8 +93,8 @@ string readLine(int fd); void writeLine(int fd, string s); /* Delete a path; i.e., in the case of a directory, it is deleted - recursively. Don't use this at home, kids. The second variant - returns the number of bytes and blocks freed. */ + recursively. It's not an error if the path does not exist. The + second variant returns the number of bytes and blocks freed. */ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); @@ -103,6 +103,9 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +/* Return the path to $XDG_CACHE_HOME/.cache. */ +Path getCacheDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -114,62 +117,6 @@ void createSymlink(const Path & target, const Path & link); void replaceSymlink(const Path & target, const Path & link); -template<class T, class A> -T singleton(const A & a) -{ - T t; - t.insert(a); - return t; -} - - -/* Messages. */ - - -typedef enum { - ltPretty, /* nice, nested output */ - ltEscapes, /* nesting indicated using escape codes (for log2xml) */ - ltFlat /* no nesting */ -} LogType; - -extern LogType logType; -extern Verbosity verbosity; /* suppress msgs > this */ - -class Nest -{ -private: - bool nest; -public: - Nest(); - ~Nest(); - void open(Verbosity level, const FormatOrString & fs); - void close(); -}; - -void printMsg_(Verbosity level, const FormatOrString & fs); - -#define startNest(varName, level, f) \ - Nest varName; \ - if (level <= verbosity) { \ - varName.open(level, (f)); \ - } - -#define printMsg(level, f) \ - do { \ - if (level <= verbosity) { \ - printMsg_(level, (f)); \ - } \ - } while (0) - -#define debug(f) printMsg(lvlDebug, f) - -void warnOnce(bool & haveWarned, const FormatOrString & fs); - -void writeToStderr(const string & s); - -extern void (*_writeToStderr) (const unsigned char * buf, size_t count); - - /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); @@ -205,9 +152,12 @@ class AutoDelete bool del; bool recursive; public: + AutoDelete(); AutoDelete(const Path & p, bool recursive = true); ~AutoDelete(); void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } }; @@ -313,6 +263,8 @@ void restoreSIGPIPE(); extern volatile sig_atomic_t _isInterrupted; +extern thread_local bool interruptThrown; + void _interrupted(); void inline checkInterrupt() @@ -356,19 +308,26 @@ bool statusOk(int status); /* Parse a string into an integer. */ template<class N> bool string2Int(const string & s, N & n) { + if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed) + return false; std::istringstream str(s); str >> n; return str && str.get() == EOF; } -template<class N> string int2String(N n) +/* Parse a string into a float. */ +template<class N> bool string2Float(const string & s, N & n) { - std::ostringstream str; - str << n; - return str.str(); + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; } +/* Return true iff `s' starts with `prefix'. */ +bool hasPrefix(const string & s, const string & prefix); + + /* Return true iff `s' ends in `suffix'. */ bool hasSuffix(const string & s, const string & suffix); @@ -415,4 +374,14 @@ string base64Encode(const string & s); string base64Decode(const string & s); +/* Get a value for the specified key from an associate container, or a + default value if the key doesn't exist. */ +template <class T> +string get(const T & map, const string & key, const string & def = "") +{ + auto i = map.find(key); + return i == map.end() ? def : i->second; +} + + } diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc index 01794001b2c6..98bd058d18be 100644 --- a/src/libutil/xml-writer.cc +++ b/src/libutil/xml-writer.cc @@ -73,10 +73,10 @@ void XMLWriter::writeEmptyElement(const string & name, void XMLWriter::writeAttrs(const XMLAttrs & attrs) { - for (XMLAttrs::const_iterator i = attrs.begin(); i != attrs.end(); ++i) { - output << " " << i->first << "=\""; - for (unsigned int j = 0; j < i->second.size(); ++j) { - char c = i->second[j]; + for (auto & i : attrs) { + output << " " << i.first << "=\""; + for (unsigned int j = 0; j < i.second.size(); ++j) { + char c = i.second[j]; if (c == '"') output << """; else if (c == '<') output << "<"; else if (c == '>') output << ">"; diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index c8dc9099ca09..3aa348581b19 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include <iostream> +#include <cerrno> using namespace nix; @@ -28,7 +29,12 @@ void removeOldGenerations(std::string dir) auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type; if (type == DT_LNK && canWrite) { - auto link = readLink(path); + std::string link; + try { + link = readLink(path); + } catch (SysError & e) { + if (e.errNo == ENOENT) continue; + } if (link.find("link") != string::npos) { printMsg(lvlInfo, format("removing old generations of profile %1%") % path); if (deleteOlderThan != "") @@ -45,11 +51,12 @@ void removeOldGenerations(std::string dir) int main(int argc, char * * argv) { bool removeOld = false; - Strings extraArgs; return handleExceptions(argv[0], [&]() { initNix(); + GCOptions options; + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-collect-garbage"); @@ -61,8 +68,12 @@ int main(int argc, char * * argv) deleteOlderThan = getArg(*arg, arg, end); } else if (*arg == "--dry-run") dryRun = true; + else if (*arg == "--max-freed") { + long long maxFreed = getIntArg<long long>(*arg, arg, end, true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } else - extraArgs.push_back(*arg); + return false; return true; }); @@ -71,8 +82,7 @@ int main(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { - store = openStore(false); - GCOptions options; + auto store = openStore(); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index ad8b0d133d82..9757086c650e 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -7,6 +7,7 @@ #include "affinity.hh" #include "globals.hh" #include "monitor-fd.hh" +#include "derivations.hh" #include <algorithm> @@ -32,29 +33,43 @@ using namespace nix; static FdSource from(STDIN_FILENO); static FdSink to(STDOUT_FILENO); -bool canSendStderr; +static bool canSendStderr; +static Logger * defaultLogger; -/* This function is called anytime we want to write something to - stderr. If we're in a state where the protocol allows it (i.e., - when canSendStderr), send the message to the client over the - socket. */ -static void tunnelStderr(const unsigned char * buf, size_t count) + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +class TunnelLogger : public Logger { - if (canSendStderr) { - try { - writeInt(STDERR_NEXT, to); - writeString(buf, count, to); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - canSendStderr = false; - throw; - } - } else - writeFull(STDERR_FILENO, buf, count); -} + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + if (canSendStderr) { + try { + to << STDERR_NEXT << (fs.s + "\n"); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + canSendStderr = false; + throw; + } + } else + defaultLogger->log(lvl, fs); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; /* startWork() means that we're starting an operation for which we @@ -72,11 +87,10 @@ static void stopWork(bool success = true, const string & msg = "", unsigned int canSendStderr = false; if (success) - writeInt(STDERR_LAST, to); + to << STDERR_LAST; else { - writeInt(STDERR_ERROR, to); - writeString(msg, to); - if (status != 0) writeInt(status, to); + to << STDERR_ERROR << msg; + if (status != 0) to << status; } } @@ -87,7 +101,7 @@ struct TunnelSink : Sink TunnelSink(Sink & to) : to(to) { } virtual void operator () (const unsigned char * data, size_t len) { - writeInt(STDERR_WRITE, to); + to << STDERR_WRITE; writeString(data, len, to); } }; @@ -99,8 +113,7 @@ struct TunnelSource : BufferedSource TunnelSource(Source & from) : from(from) { } size_t readUnbuffered(unsigned char * data, size_t len) { - writeInt(STDERR_READ, to); - writeInt(len, to); + to << STDERR_READ << len; to.flush(); size_t n = readString(data, len, from); if (n == 0) throw EndOfFile("unexpected end-of-file"); @@ -150,7 +163,7 @@ struct SavingSourceAdapter : Source }; -static void performOp(bool trusted, unsigned int clientVersion, +static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVersion, Source & from, Sink & to, unsigned int op) { switch (op) { @@ -166,7 +179,7 @@ static void performOp(bool trusted, unsigned int clientVersion, assertStorePath(path); bool result = store->isValidPath(path); stopWork(); - writeInt(result, to); + to << result; break; } @@ -175,16 +188,16 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); PathSet res = store->queryValidPaths(paths); stopWork(); - writeStrings(res, to); + to << res; break; } case wopHasSubstitutes: { Path path = readStorePath(from); startWork(); - PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path)); + PathSet res = store->querySubstitutablePaths({path}); stopWork(); - writeInt(res.find(path) != res.end(), to); + to << (res.find(path) != res.end()); break; } @@ -193,16 +206,16 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); PathSet res = store->querySubstitutablePaths(paths); stopWork(); - writeStrings(res, to); + to << res; break; } case wopQueryPathHash: { Path path = readStorePath(from); startWork(); - Hash hash = store->queryPathHash(path); + auto hash = store->queryPathInfo(path)->narHash; stopWork(); - writeString(printHash(hash), to); + to << printHash(hash); break; } @@ -214,14 +227,14 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); PathSet paths; if (op == wopQueryReferences) - store->queryReferences(path, paths); + paths = store->queryPathInfo(path)->references; else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); stopWork(); - writeStrings(paths, to); + to << paths; break; } @@ -231,16 +244,16 @@ static void performOp(bool trusted, unsigned int clientVersion, StringSet names; names = store->queryDerivationOutputNames(path); stopWork(); - writeStrings(names, to); + to << names; break; } case wopQueryDeriver: { Path path = readStorePath(from); startWork(); - Path deriver = store->queryDeriver(path); + auto deriver = store->queryPathInfo(path)->deriver; stopWork(); - writeString(deriver, to); + to << deriver; break; } @@ -249,7 +262,7 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); Path path = store->queryPathFromHashPart(hashPart); stopWork(); - writeString(path, to); + to << path; break; } @@ -279,11 +292,10 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - Path path = dynamic_cast<LocalStore *>(store.get()) - ->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo); + Path path = store->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo); stopWork(); - writeString(path, to); + to << path; break; } @@ -294,36 +306,59 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); Path path = store->addTextToStore(suffix, s, refs); stopWork(); - writeString(path, to); + to << path; break; } case wopExportPath: { Path path = readStorePath(from); - bool sign = readInt(from) == 1; + readInt(from); // obsolete startWork(); TunnelSink sink(to); - store->exportPath(path, sign, sink); + store->exportPath(path, sink); stopWork(); - writeInt(1, to); + to << 1; break; } case wopImportPaths: { startWork(); TunnelSource source(from); - Paths paths = store->importPaths(!trusted, source); + Paths paths = store->importPaths(source, 0); stopWork(); - writeStrings(paths, to); + to << paths; break; } case wopBuildPaths: { PathSet drvs = readStorePaths<PathSet>(from); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode)readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not supported when building through the Nix daemon"); + } startWork(); - store->buildPaths(drvs); + store->buildPaths(drvs, mode); stopWork(); - writeInt(1, to); + to << 1; + break; + } + + case wopBuildDerivation: { + Path drvPath = readStorePath(from); + BasicDerivation drv; + from >> drv; + BuildMode buildMode = (BuildMode) readInt(from); + startWork(); + if (!trusted) + throw Error("you are not privileged to build derivations"); + auto res = store->buildDerivation(drvPath, drv, buildMode); + stopWork(); + to << res.status << res.errorMsg; break; } @@ -332,7 +367,7 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); store->ensurePath(path); stopWork(); - writeInt(1, to); + to << 1; break; } @@ -341,7 +376,7 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); store->addTempRoot(path); stopWork(); - writeInt(1, to); + to << 1; break; } @@ -350,7 +385,7 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); store->addIndirectRoot(path); stopWork(); - writeInt(1, to); + to << 1; break; } @@ -358,7 +393,7 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); store->syncWithGC(); stopWork(); - writeInt(1, to); + to << 1; break; } @@ -366,11 +401,9 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); Roots roots = store->findRoots(); stopWork(); - writeInt(roots.size(), to); - for (Roots::iterator i = roots.begin(); i != roots.end(); ++i) { - writeString(i->first, to); - writeString(i->second, to); - } + to << roots.size(); + for (auto & i : roots) + to << i.first << i.second; break; } @@ -395,9 +428,7 @@ static void performOp(bool trusted, unsigned int clientVersion, store->collectGarbage(options, results); stopWork(); - writeStrings(results.paths, to); - writeLongLong(results.bytesFreed, to); - writeLongLong(0, to); // obsolete + to << results.paths << results.bytesFreed << 0 /* obsolete */; break; } @@ -407,17 +438,17 @@ static void performOp(bool trusted, unsigned int clientVersion, settings.keepGoing = readInt(from) != 0; settings.set("build-fallback", readInt(from) ? "true" : "false"); verbosity = (Verbosity) readInt(from); - settings.set("build-max-jobs", int2String(readInt(from))); - settings.set("build-max-silent-time", int2String(readInt(from))); + settings.set("build-max-jobs", std::to_string(readInt(from))); + settings.set("build-max-silent-time", std::to_string(readInt(from))); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.useBuildHook = readInt(from) != 0; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { - settings.buildVerbosity = (Verbosity) readInt(from); - logType = (LogType) readInt(from); - settings.printBuildTrace = readInt(from) != 0; + settings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace } if (GET_PROTOCOL_MINOR(clientVersion) >= 6) - settings.set("build-cores", int2String(readInt(from))); + settings.set("build-cores", std::to_string(readInt(from))); if (GET_PROTOCOL_MINOR(clientVersion) >= 10) settings.set("build-use-substitutes", readInt(from) ? "true" : "false"); if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { @@ -441,18 +472,15 @@ static void performOp(bool trusted, unsigned int clientVersion, Path path = absPath(readString(from)); startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos(singleton<PathSet>(path), infos); + store->querySubstitutablePathInfos({path}, infos); stopWork(); SubstitutablePathInfos::iterator i = infos.find(path); if (i == infos.end()) - writeInt(0, to); + to << 0; else { - writeInt(1, to); - writeString(i->second.deriver, to); - writeStrings(i->second.references, to); - writeLongLong(i->second.downloadSize, to); + to << 1 << i->second.deriver << i->second.references << i->second.downloadSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 7) - writeLongLong(i->second.narSize, to); + to << i->second.narSize; } break; } @@ -463,13 +491,10 @@ static void performOp(bool trusted, unsigned int clientVersion, SubstitutablePathInfos infos; store->querySubstitutablePathInfos(paths, infos); stopWork(); - writeInt(infos.size(), to); - foreach (SubstitutablePathInfos::iterator, i, infos) { - writeString(i->first, to); - writeString(i->second.deriver, to); - writeStrings(i->second.references, to); - writeLongLong(i->second.downloadSize, to); - writeLongLong(i->second.narSize, to); + to << infos.size(); + for (auto & i : infos) { + to << i.first << i.second.deriver << i.second.references + << i.second.downloadSize << i.second.narSize; } break; } @@ -478,37 +503,33 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); PathSet paths = store->queryAllValidPaths(); stopWork(); - writeStrings(paths, to); - break; - } - - case wopQueryFailedPaths: { - startWork(); - PathSet paths = store->queryFailedPaths(); - stopWork(); - writeStrings(paths, to); - break; - } - - case wopClearFailedPaths: { - PathSet paths = readStrings<PathSet>(from); - startWork(); - store->clearFailedPaths(paths); - stopWork(); - writeInt(1, to); + to << paths; break; } case wopQueryPathInfo: { Path path = readStorePath(from); + std::shared_ptr<const ValidPathInfo> info; startWork(); - ValidPathInfo info = store->queryPathInfo(path); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath &) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } stopWork(); - writeString(info.deriver, to); - writeString(printHash(info.hash), to); - writeStrings(info.references, to); - writeInt(info.registrationTime, to); - writeLongLong(info.narSize, to); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) + to << 1; + to << info->deriver << printHash(info->narHash) << info->references + << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs; + } + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } break; } @@ -516,7 +537,7 @@ static void performOp(bool trusted, unsigned int clientVersion, startWork(); store->optimiseStore(); stopWork(); - writeInt(1, to); + to << 1; break; case wopVerifyStore: { @@ -527,7 +548,19 @@ static void performOp(bool trusted, unsigned int clientVersion, throw Error("you are not privileged to repair paths"); bool errors = store->verifyStore(checkContents, repair); stopWork(); - writeInt(errors, to); + to << errors; + break; + } + + case wopAddSignatures: { + Path path = readStorePath(from); + StringSet sigs = readStrings<StringSet>(from); + startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + stopWork(); + to << 1; break; } @@ -542,22 +575,21 @@ static void processConnection(bool trusted) MonitorFdHup monitor(from.fd); canSendStderr = false; - _writeToStderr = tunnelStderr; + defaultLogger = logger; + logger = new TunnelLogger(); /* Exchange the greeting. */ unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - writeInt(WORKER_MAGIC_2, to); - writeInt(PROTOCOL_VERSION, to); + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to.flush(); unsigned int clientVersion = readInt(from); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) setAffinityTo(readInt(from)); - bool reserveSpace = true; if (GET_PROTOCOL_MINOR(clientVersion) >= 11) - reserveSpace = readInt(from) != 0; + readInt(from); // obsolete reserveSpace /* Send startup error messages to the client. */ startWork(); @@ -575,56 +607,56 @@ static void processConnection(bool trusted) #endif /* Open the store. */ - store = std::shared_ptr<StoreAPI>(new LocalStore(reserveSpace)); + auto store = make_ref<LocalStore>(); stopWork(); to.flush(); - } catch (Error & e) { - stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); - to.flush(); - return; - } + /* Process client requests. */ + unsigned int opCount = 0; + + while (true) { + WorkerOp op; + try { + op = (WorkerOp) readInt(from); + } catch (Interrupted & e) { + break; + } catch (EndOfFile & e) { + break; + } - /* Process client requests. */ - unsigned int opCount = 0; + opCount++; + + try { + performOp(store, trusted, clientVersion, from, to, op); + } catch (Error & e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = canSendStderr; + stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); + if (!errorAllowed) throw; + } catch (std::bad_alloc & e) { + stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); + throw; + } - while (true) { - WorkerOp op; - try { - op = (WorkerOp) readInt(from); - } catch (Interrupted & e) { - break; - } catch (EndOfFile & e) { - break; - } + to.flush(); - opCount++; + assert(!canSendStderr); + }; - try { - performOp(trusted, clientVersion, from, to, op); - } catch (Error & e) { - /* If we're not in a state where we can send replies, then - something went wrong processing the input of the - client. This can happen especially if I/O errors occur - during addTextToStore() / importPath(). If that - happens, just send the error message and exit. */ - bool errorAllowed = canSendStderr; - stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); - if (!errorAllowed) throw; - } catch (std::bad_alloc & e) { - stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); - throw; - } + canSendStderr = false; + _isInterrupted = false; + printMsg(lvlDebug, format("%1% operations") % opCount); + } catch (Error & e) { + stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); to.flush(); - - assert(!canSendStderr); - }; - - canSendStderr = false; - _isInterrupted = false; - printMsg(lvlDebug, format("%1% operations") % opCount); + return; + } } @@ -693,6 +725,10 @@ static PeerInfo getPeerInfo(int remote) #elif defined(LOCAL_PEERCRED) +#if !defined(SOL_LOCAL) +#define SOL_LOCAL 0 +#endif + xucred cred; socklen_t credLen = sizeof(cred); if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) @@ -721,7 +757,7 @@ static void daemonLoop(char * * argv) /* Handle socket-based activation by systemd. */ if (getEnv("LISTEN_FDS") != "") { - if (getEnv("LISTEN_PID") != int2String(getpid()) || getEnv("LISTEN_FDS") != "1") + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; } @@ -775,10 +811,6 @@ static void daemonLoop(char * * argv) while (1) { try { - /* Important: the server process *cannot* open the SQLite - database, because it doesn't like forks very much. */ - assert(!store); - /* Accept a connection. */ struct sockaddr_un remoteAddr; socklen_t remoteAddrLen = sizeof(remoteAddr); @@ -797,10 +829,10 @@ static void daemonLoop(char * * argv) PeerInfo peer = getPeerInfo(remote); struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - string user = pw ? pw->pw_name : int2String(peer.uid); + string user = pw ? pw->pw_name : std::to_string(peer.uid); struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - string group = gr ? gr->gr_name : int2String(peer.gid); + string group = gr ? gr->gr_name : std::to_string(peer.gid); Strings trustedUsers = settings.get("trusted-users", Strings({"root"})); Strings allowedUsers = settings.get("allowed-users", Strings({"*"})); @@ -812,7 +844,7 @@ static void daemonLoop(char * * argv) throw Error(format("user ‘%1%’ is not allowed to connect to the Nix daemon") % user); printMsg(lvlInfo, format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) - % (peer.pidKnown ? int2String(peer.pid) : "<unknown>") + % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") % (peer.uidKnown ? user : "<unknown>")); /* Fork a child to handle the connection. */ @@ -833,7 +865,7 @@ static void daemonLoop(char * * argv) /* For debugging, stuff the pid into argv[1]. */ if (peer.pidKnown && argv[1]) { - string processName = int2String(peer.pid); + string processName = std::to_string(peer.pid); strncpy(argv[1], processName.c_str(), strlen(argv[1])); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 3f82345ce43b..6b6c245d3637 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,17 +1,17 @@ -#include "profiles.hh" -#include "names.hh" -#include "globals.hh" -#include "misc.hh" -#include "shared.hh" -#include "eval.hh" -#include "get-drvs.hh" #include "attr-path.hh" #include "common-opts.hh" -#include "xml-writer.hh" +#include "derivations.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "globals.hh" +#include "names.hh" +#include "profiles.hh" +#include "shared.hh" #include "store-api.hh" #include "user-env.hh" #include "util.hh" #include "value-to-json.hh" +#include "xml-writer.hh" #include <cerrno> #include <ctime> @@ -223,8 +223,8 @@ static int comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) static bool isPrebuilt(EvalState & state, DrvInfo & elem) { Path path = elem.queryOutPath(); - if (store->isValidPath(path)) return true; - PathSet ps = store->querySubstitutablePaths(singleton<PathSet>(path)); + if (state.store->isValidPath(path)) return true; + PathSet ps = state.store->querySubstitutablePaths({path}); return ps.find(path) != ps.end(); } @@ -232,9 +232,9 @@ static bool isPrebuilt(EvalState & state, DrvInfo & elem) static void checkSelectorUse(DrvNames & selectors) { /* Check that all selectors have been used. */ - foreach (DrvNames::iterator, i, selectors) - if (i->hits == 0 && i->fullName != "*") - throw Error(format("selector ‘%1%’ matches no derivations") % i->fullName); + for (auto & i : selectors) + if (i.hits == 0 && i.fullName != "*") + throw Error(format("selector ‘%1%’ matches no derivations") % i.fullName); } @@ -248,7 +248,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, DrvInfos elems; set<unsigned int> done; - foreach (DrvNames::iterator, i, selectors) { + for (auto & i : selectors) { typedef list<std::pair<DrvInfo, unsigned int> > Matches; Matches matches; unsigned int n = 0; @@ -256,8 +256,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, j != allElems.end(); ++j, ++n) { DrvName drvName(j->name); - if (i->matches(drvName)) { - i->hits++; + if (i.matches(drvName)) { + i.hits++; matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); } } @@ -276,47 +276,47 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, Newest newest; StringSet multiple; - for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) { - DrvName drvName(j->first.name); + for (auto & j : matches) { + DrvName drvName(j.first.name); int d = 1; Newest::iterator k = newest.find(drvName.name); if (k != newest.end()) { - d = j->first.system == k->second.first.system ? 0 : - j->first.system == settings.thisSystem ? 1 : + d = j.first.system == k->second.first.system ? 0 : + j.first.system == settings.thisSystem ? 1 : k->second.first.system == settings.thisSystem ? -1 : 0; if (d == 0) - d = comparePriorities(state, j->first, k->second.first); + d = comparePriorities(state, j.first, k->second.first); if (d == 0) d = compareVersions(drvName.version, DrvName(k->second.first.name).version); } if (d > 0) { newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, *j)); - multiple.erase(j->first.name); + newest.insert(Newest::value_type(drvName.name, j)); + multiple.erase(j.first.name); } else if (d == 0) { - multiple.insert(j->first.name); + multiple.insert(j.first.name); } } matches.clear(); - for (Newest::iterator j = newest.begin(); j != newest.end(); ++j) { - if (multiple.find(j->second.first.name) != multiple.end()) + for (auto & j : newest) { + if (multiple.find(j.second.first.name) != multiple.end()) printMsg(lvlInfo, format("warning: there are multiple derivations named ‘%1%’; using the first one") - % j->second.first.name); - matches.push_back(j->second); + % j.second.first.name); + matches.push_back(j.second); } } /* Insert only those elements in the final list that we haven't inserted before. */ - for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) - if (done.find(j->second) == done.end()) { - done.insert(j->second); - elems.push_back(j->first); + for (auto & j : matches) + if (done.find(j.second) == done.end()) { + done.insert(j.second); + elems.push_back(j.first); } } @@ -370,8 +370,8 @@ static void queryInstSources(EvalState & state, Value vArg; loadSourceExpr(state, instSource.nixExprPath, vArg); - foreach (Strings::const_iterator, i, args) { - Expr * eFun = state.parseExprFromString(*i, absPath(".")); + for (auto & i : args) { + Expr * eFun = state.parseExprFromString(i, absPath(".")); Value vFun, vTmp; state.eval(eFun, vFun); mkApp(vTmp, vFun, vArg); @@ -386,8 +386,8 @@ static void queryInstSources(EvalState & state, derivations). */ case srcStorePaths: { - foreach (Strings::const_iterator, i, args) { - Path path = followLinksToStorePath(*i); + for (auto & i : args) { + Path path = followLinksToStorePath(i); string name = baseNameOf(path); string::size_type dash = name.find('-'); @@ -398,7 +398,7 @@ static void queryInstSources(EvalState & state, if (isDerivation(path)) { elem.setDrvPath(path); - elem.setOutPath(findOutput(derivationFromPath(*store, path), "out")); + elem.setOutPath(state.store->derivationFromPath(path).findOutput("out")); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); @@ -424,8 +424,8 @@ static void queryInstSources(EvalState & state, case srcAttrPath: { Value vRoot; loadSourceExpr(state, instSource.nixExprPath, vRoot); - foreach (Strings::const_iterator, i, args) { - Value & v(*findAlongAttrPath(state, *i, *instSource.autoArgs, vRoot)); + for (auto & i : args) { + Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); getDerivations(state, v, "", *instSource.autoArgs, elems, true); } break; @@ -437,15 +437,15 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { PathSet targets; - foreach (DrvInfos::iterator, i, elems) { - Path drvPath = i->queryDrvPath(); + for (auto & i : elems) { + Path drvPath = i.queryDrvPath(); if (drvPath != "") targets.insert(drvPath); else - targets.insert(i->queryOutPath()); + targets.insert(i.queryOutPath()); } - printMissing(*store, targets); + printMissing(state.store, targets); } @@ -465,19 +465,19 @@ static void installDerivations(Globals & globals, queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); /* If --prebuilt-only is given, filter out source-only packages. */ - foreach (DrvInfos::iterator, i, newElemsTmp) - if (!globals.prebuiltOnly || isPrebuilt(*globals.state, *i)) - newElems.push_back(*i); + for (auto & i : newElemsTmp) + if (!globals.prebuiltOnly || isPrebuilt(*globals.state, i)) + newElems.push_back(i); StringSet newNames; - for (DrvInfos::iterator i = newElems.begin(); i != newElems.end(); ++i) { + for (auto & i : newElems) { /* `forceName' is a hack to get package names right in some one-click installs, namely those where the name used in the path is not the one we want (e.g., `java-front' versus `java-front-0.9pre15899'). */ if (globals.forceName != "") - i->name = globals.forceName; - newNames.insert(DrvName(i->name).name); + i.name = globals.forceName; + newNames.insert(DrvName(i.name).name); } @@ -491,18 +491,18 @@ static void installDerivations(Globals & globals, if (!globals.removeAll) { DrvInfos installedElems = queryInstalled(*globals.state, profile); - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + for (auto & i : installedElems) { + DrvName drvName(i.name); if (!globals.preserveInstalled && newNames.find(drvName.name) != newNames.end() && - !keep(*i)) - printMsg(lvlInfo, format("replacing old ‘%1%’") % i->name); + !keep(i)) + printMsg(lvlInfo, format("replacing old ‘%1%’") % i.name); else - allElems.push_back(*i); + allElems.push_back(i); } - foreach (DrvInfos::iterator, i, newElems) - printMsg(lvlInfo, format("installing ‘%1%’") % i->name); + for (auto & i : newElems) + printMsg(lvlInfo, format("installing ‘%1%’") % i.name); } printMissing(*globals.state, newElems); @@ -555,13 +555,13 @@ static void upgradeDerivations(Globals & globals, /* Go through all installed derivations. */ DrvInfos newElems; - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + for (auto & i : installedElems) { + DrvName drvName(i.name); try { - if (keep(*i)) { - newElems.push_back(*i); + if (keep(i)) { + newElems.push_back(i); continue; } @@ -570,14 +570,16 @@ static void upgradeDerivations(Globals & globals, constraints specified by upgradeType. If there are multiple matches, take the one with the highest priority. If there are still multiple matches, - take the one with the highest version. */ + take the one with the highest version. + Do not upgrade if it would decrease the priority. */ DrvInfos::iterator bestElem = availElems.end(); - DrvName bestName; - foreach (DrvInfos::iterator, j, availElems) { + string bestVersion; + for (auto j = availElems.begin(); j != availElems.end(); ++j) { + if (comparePriorities(*globals.state, i, *j) > 0) + continue; DrvName newName(j->name); if (newName.name == drvName.name) { - int d = comparePriorities(*globals.state, *i, *j); - if (d == 0) d = compareVersions(drvName.version, newName.version); + int d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || (upgradeType == utLeq && d <= 0) || (upgradeType == utEq && d == 0) || @@ -586,28 +588,30 @@ static void upgradeDerivations(Globals & globals, int d2 = -1; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); - if (d2 == 0) d2 = compareVersions(bestName.version, newName.version); + if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); } if (d2 < 0 && (!globals.prebuiltOnly || isPrebuilt(*globals.state, *j))) { bestElem = j; - bestName = newName; + bestVersion = newName.version; } } } } if (bestElem != availElems.end() && - i->queryOutPath() != + i.queryOutPath() != bestElem->queryOutPath()) { + const char * action = compareVersions(drvName.version, bestVersion) <= 0 + ? "upgrading" : "downgrading"; printMsg(lvlInfo, - format("upgrading ‘%1%’ to ‘%2%’") - % i->name % bestElem->name); + format("%1% ‘%2%’ to ‘%3%’") + % action % i.name % bestElem->name); newElems.push_back(*bestElem); - } else newElems.push_back(*i); + } else newElems.push_back(i); } catch (Error & e) { - e.addPrefix(format("while trying to find an upgrade for ‘%1%’:\n") % i->name); + e.addPrefix(format("while trying to find an upgrade for ‘%1%’:\n") % i.name); throw; } } @@ -666,13 +670,13 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); /* Update all matching derivations. */ - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); - foreach (DrvNames::iterator, j, selectors) - if (j->matches(drvName)) { - printMsg(lvlInfo, format("setting flag on ‘%1%’") % i->name); - j->hits++; - setMetaFlag(*globals.state, *i, flagName, flagValue); + for (auto & i : installedElems) { + DrvName drvName(i.name); + for (auto & j : selectors) + if (j.matches(drvName)) { + printMsg(lvlInfo, format("setting flag on ‘%1%’") % i.name); + j.hits++; + setMetaFlag(*globals.state, i, flagName, flagValue); break; } } @@ -706,19 +710,19 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) drv.name = globals.forceName; if (drv.queryDrvPath() != "") { - PathSet paths = singleton<PathSet>(drv.queryDrvPath()); - printMissing(*store, paths); + PathSet paths = {drv.queryDrvPath()}; + printMissing(globals.state->store, paths); if (globals.dryRun) return; - store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); } else { - printMissing(*store, singleton<PathSet>(drv.queryOutPath())); + printMissing(globals.state->store, {drv.queryOutPath()}); if (globals.dryRun) return; - store->ensurePath(drv.queryOutPath()); + globals.state->store->ensurePath(drv.queryOutPath()); } debug(format("switching to new user environment")); - Path generation = createGeneration(globals.profile, drv.queryOutPath()); + Path generation = createGeneration(globals.state->store, globals.profile, drv.queryOutPath()); switchLink(globals.profile, generation); } @@ -732,20 +736,20 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, DrvInfos installedElems = queryInstalled(*globals.state, profile); DrvInfos newElems; - foreach (DrvInfos::iterator, i, installedElems) { - DrvName drvName(i->name); + for (auto & i : installedElems) { + DrvName drvName(i.name); bool found = false; - foreach (Strings::iterator, j, selectors) + for (auto & j : selectors) /* !!! the repeated calls to followLinksToStorePath() are expensive, should pre-compute them. */ - if ((isPath(*j) && i->queryOutPath() == followLinksToStorePath(*j)) - || DrvName(*j).matches(drvName)) + if ((isPath(j) && i.queryOutPath() == followLinksToStorePath(j)) + || DrvName(j).matches(drvName)) { - printMsg(lvlInfo, format("uninstalling ‘%1%’") % i->name); + printMsg(lvlInfo, format("uninstalling ‘%1%’") % i.name); found = true; break; } - if (!found) newElems.push_back(*i); + if (!found) newElems.push_back(i); } if (globals.dryRun) return; @@ -788,18 +792,18 @@ void printTable(Table & table) vector<unsigned int> widths; widths.resize(nrColumns); - foreach (Table::iterator, i, table) { - assert(i->size() == nrColumns); + for (auto & i : table) { + assert(i.size() == nrColumns); Strings::iterator j; unsigned int column; - for (j = i->begin(), column = 0; j != i->end(); ++j, ++column) + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) if (j->size() > widths[column]) widths[column] = j->size(); } - foreach (Table::iterator, i, table) { + for (auto & i : table) { Strings::iterator j; unsigned int column; - for (j = i->begin(), column = 0; j != i->end(); ++j, ++column) { + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { string s = *j; replace(s.begin(), s.end(), '\n', ' '); cout << s; @@ -828,8 +832,8 @@ static VersionDiff compareVersionAgainstSet( VersionDiff diff = cvUnavail; version = "?"; - for (DrvInfos::const_iterator i = elems.begin(); i != elems.end(); ++i) { - DrvName name2(i->name); + for (auto & i : elems) { + DrvName name2(i.name); if (name.name == name2.name) { int d = compareVersions(name.version, name2.version); if (d < 0) { @@ -855,21 +859,21 @@ static VersionDiff compareVersionAgainstSet( static void queryJSON(Globals & globals, vector<DrvInfo> & elems) { JSONObject topObj(cout); - foreach (vector<DrvInfo>::iterator, i, elems) { - topObj.attr(i->attrPath); + for (auto & i : elems) { + topObj.attr(i.attrPath); JSONObject pkgObj(cout); - pkgObj.attr("name", i->name); - pkgObj.attr("system", i->system); + pkgObj.attr("name", i.name); + pkgObj.attr("system", i.system); pkgObj.attr("meta"); JSONObject metaObj(cout); - StringSet metaNames = i->queryMetaNames(); - foreach (StringSet::iterator, j, metaNames) { - metaObj.attr(*j); - Value * v = i->queryMeta(*j); + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + metaObj.attr(j); + Value * v = i.queryMeta(j); if (!v) { - printMsg(lvlError, format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i->name % *j); + printMsg(lvlError, format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i.name % j); cout << "null"; } else { PathSet context; @@ -944,8 +948,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Sort them by name. */ /* !!! */ vector<DrvInfo> elems; - for (DrvInfos::iterator i = elems_.begin(); i != elems_.end(); ++i) - elems.push_back(*i); + for (auto & i : elems_) elems.push_back(i); sort(elems.begin(), elems.end(), cmpElemByName); @@ -954,9 +957,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) PathSet installed; /* installed paths */ if (printStatus) { - for (DrvInfos::iterator i = installedElems.begin(); - i != installedElems.end(); ++i) - installed.insert(i->queryOutPath()); + for (auto & i : installedElems) + installed.insert(i.queryOutPath()); } @@ -964,15 +966,15 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) PathSet validPaths, substitutablePaths; if (printStatus || globals.prebuiltOnly) { PathSet paths; - foreach (vector<DrvInfo>::iterator, i, elems) + for (auto & i : elems) try { - paths.insert(i->queryOutPath()); + paths.insert(i.queryOutPath()); } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i->name); - i->setFailed(); + printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i.name); + i.setFailed(); } - validPaths = store->queryValidPaths(paths); - substitutablePaths = store->querySubstitutablePaths(paths); + validPaths = globals.state->store->queryValidPaths(paths); + substitutablePaths = globals.state->store->querySubstitutablePaths(paths); } @@ -990,15 +992,15 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); XMLOpenElement xmlRoot(xml, "items"); - foreach (vector<DrvInfo>::iterator, i, elems) { + for (auto & i : elems) { try { - if (i->hasFailed()) continue; + if (i.hasFailed()) continue; - startNest(nest, lvlDebug, format("outputting query result ‘%1%’") % i->attrPath); + Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); if (globals.prebuiltOnly && - validPaths.find(i->queryOutPath()) == validPaths.end() && - substitutablePaths.find(i->queryOutPath()) == substitutablePaths.end()) + validPaths.find(i.queryOutPath()) == validPaths.end() && + substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end()) continue; /* For table output. */ @@ -1008,7 +1010,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) XMLAttrs attrs; if (printStatus) { - Path outPath = i->queryOutPath(); + Path outPath = i.queryOutPath(); bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); bool isInstalled = installed.find(outPath) != installed.end(); bool isValid = validPaths.find(outPath) != validPaths.end(); @@ -1024,14 +1026,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } if (xmlOutput) - attrs["attrPath"] = i->attrPath; + attrs["attrPath"] = i.attrPath; else if (printAttrPath) - columns.push_back(i->attrPath); + columns.push_back(i.attrPath); if (xmlOutput) - attrs["name"] = i->name; + attrs["name"] = i.name; else if (printName) - columns.push_back(i->name); + columns.push_back(i.name); if (compareVersions) { /* Compare this element against the versions of the @@ -1039,7 +1041,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) elements, or the set of installed elements. !!! This is O(N * M), should be O(N * lg M). */ string version; - VersionDiff diff = compareVersionAgainstSet(*i, otherElems, version); + VersionDiff diff = compareVersionAgainstSet(i, otherElems, version); char ch; switch (diff) { @@ -1064,13 +1066,13 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } if (xmlOutput) { - if (i->system != "") attrs["system"] = i->system; + if (i.system != "") attrs["system"] = i.system; } else if (printSystem) - columns.push_back(i->system); + columns.push_back(i.system); if (printDrvPath) { - string drvPath = i->queryDrvPath(); + string drvPath = i.queryDrvPath(); if (xmlOutput) { if (drvPath != "") attrs["drvPath"] = drvPath; } else @@ -1078,18 +1080,18 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } if (printOutPath && !xmlOutput) { - DrvInfo::Outputs outputs = i->queryOutputs(); + DrvInfo::Outputs outputs = i.queryOutputs(); string s; - foreach (DrvInfo::Outputs::iterator, j, outputs) { + for (auto & j : outputs) { if (!s.empty()) s += ';'; - if (j->first != "out") { s += j->first; s += "="; } - s += j->second; + if (j.first != "out") { s += j.first; s += "="; } + s += j.second; } columns.push_back(s); } if (printDescription) { - string descr = i->queryMetaString("description"); + string descr = i.queryMetaString("description"); if (xmlOutput) { if (descr != "") attrs["description"] = descr; } else @@ -1100,22 +1102,22 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (printOutPath || printMeta) { XMLOpenElement item(xml, "item", attrs); if (printOutPath) { - DrvInfo::Outputs outputs = i->queryOutputs(); - foreach (DrvInfo::Outputs::iterator, j, outputs) { + DrvInfo::Outputs outputs = i.queryOutputs(); + for (auto & j : outputs) { XMLAttrs attrs2; - attrs2["name"] = j->first; - attrs2["path"] = j->second; + attrs2["name"] = j.first; + attrs2["path"] = j.second; xml.writeEmptyElement("output", attrs2); } } if (printMeta) { - StringSet metaNames = i->queryMetaNames(); - foreach (StringSet::iterator, j, metaNames) { + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { XMLAttrs attrs2; - attrs2["name"] = *j; - Value * v = i->queryMeta(*j); + attrs2["name"] = j; + Value * v = i.queryMeta(j); if (!v) - printMsg(lvlError, format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i->name % *j); + printMsg(lvlError, format("derivation ‘%1%’ has invalid meta attribute ‘%2%’") % i.name % j); else { if (v->type == tString) { attrs2["type"] = "string"; @@ -1125,20 +1127,36 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs2["type"] = "int"; attrs2["value"] = (format("%1%") % v->integer).str(); xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tFloat) { + attrs2["type"] = "float"; + attrs2["value"] = (format("%1%") % v->fpoint).str(); + xml.writeEmptyElement("meta", attrs2); } else if (v->type == tBool) { attrs2["type"] = "bool"; attrs2["value"] = v->boolean ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tList) { + } else if (v->isList()) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); - for (unsigned int j = 0; j < v->list.length; ++j) { - if (v->list.elems[j]->type != tString) continue; + for (unsigned int j = 0; j < v->listSize(); ++j) { + if (v->listElems()[j]->type != tString) continue; XMLAttrs attrs3; - attrs3["value"] = v->list.elems[j]->string.s; + attrs3["value"] = v->listElems()[j]->string.s; xml.writeEmptyElement("string", attrs3); } + } else if (v->type == tAttrs) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + Bindings & attrs = *v->attrs; + for (auto &i : attrs) { + Attr & a(*attrs.find(i.name)); + if(a.value->type != tString) continue; + XMLAttrs attrs3; + attrs3["type"] = i.name; + attrs3["value"] = a.value->string.s; + xml.writeEmptyElement("string", attrs3); } + } } } } @@ -1150,9 +1168,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) cout.flush(); } catch (AssertionError & e) { - printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i->name); + printMsg(lvlTalkative, format("skipping derivation named ‘%1%’ which gives an assertion failure") % i.name); } catch (Error & e) { - e.addPrefix(format("while querying the derivation named ‘%1%’:\n") % i->name); + e.addPrefix(format("while querying the derivation named ‘%1%’:\n") % i.name); throw; } } @@ -1187,10 +1205,10 @@ static void switchGeneration(Globals & globals, int dstGen) Generations gens = findGenerations(globals.profile, curGen); Generation dst; - for (Generations::iterator i = gens.begin(); i != gens.end(); ++i) - if ((dstGen == prevGen && i->number < curGen) || - (dstGen >= 0 && i->number == dstGen)) - dst = *i; + for (auto & i : gens) + if ((dstGen == prevGen && i.number < curGen) || + (dstGen >= 0 && i.number == dstGen)) + dst = i; if (!dst) { if (dstGen == prevGen) @@ -1250,14 +1268,14 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs RunPager pager; - for (Generations::iterator i = gens.begin(); i != gens.end(); ++i) { + for (auto & i : gens) { tm t; - if (!localtime_r(&i->creationTime, &t)) throw Error("cannot convert time"); + if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") - % i->number + % i.number % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday % t.tm_hour % t.tm_min % t.tm_sec - % (i->number == curGen ? "(current)" : ""); + % (i.number == curGen ? "(current)" : ""); } } @@ -1275,7 +1293,7 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr std::set<unsigned int> gens; for (auto & i : opArgs) { unsigned int n; - if (!string2Int(i, n) || n < 0) + if (!string2Int(i, n)) throw UsageError(format("invalid generation number ‘%1%’") % i); gens.insert(n); } @@ -1284,6 +1302,12 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr } +static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) +{ + printVersion("nix-env"); +} + + int main(int argc, char * * argv) { return handleExceptions(argv[0], [&]() { @@ -1313,7 +1337,7 @@ int main(int argc, char * * argv) if (*arg == "--help") showManPage("nix-env"); else if (*arg == "--version") - printVersion("nix-env"); + op = opVersion; else if (*arg == "--install" || *arg == "-i") op = opInstall; else if (parseAutoArgs(arg, end, autoArgs_)) @@ -1374,9 +1398,9 @@ int main(int argc, char * * argv) if (!op) throw UsageError("no operation specified"); - store = openStore(); + auto store = openStore(); - globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath)); + globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath, store)); globals.state->repair = repair; if (file != "") diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 3bc31b9eafeb..ca27a7248107 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -33,63 +33,63 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Build the components in the user environment, if they don't exist already. */ PathSet drvsToBuild; - foreach (DrvInfos::iterator, i, elems) - if (i->queryDrvPath() != "") - drvsToBuild.insert(i->queryDrvPath()); + for (auto & i : elems) + if (i.queryDrvPath() != "") + drvsToBuild.insert(i.queryDrvPath()); debug(format("building user environment dependencies")); - store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ PathSet references; Value manifest; state.mkList(manifest, elems.size()); unsigned int n = 0; - foreach (DrvInfos::iterator, i, elems) { + for (auto & i : elems) { /* Create a pseudo-derivation containing the name, system, output paths, and optionally the derivation path, as well as the meta attributes. */ - Path drvPath = keepDerivations ? i->queryDrvPath() : ""; + Path drvPath = keepDerivations ? i.queryDrvPath() : ""; Value & v(*state.allocValue()); - manifest.list.elems[n++] = &v; + manifest.listElems()[n++] = &v; state.mkAttrs(v, 16); mkString(*state.allocAttr(v, state.sType), "derivation"); - mkString(*state.allocAttr(v, state.sName), i->name); - if (!i->system.empty()) - mkString(*state.allocAttr(v, state.sSystem), i->system); - mkString(*state.allocAttr(v, state.sOutPath), i->queryOutPath()); + mkString(*state.allocAttr(v, state.sName), i.name); + if (!i.system.empty()) + mkString(*state.allocAttr(v, state.sSystem), i.system); + mkString(*state.allocAttr(v, state.sOutPath), i.queryOutPath()); if (drvPath != "") - mkString(*state.allocAttr(v, state.sDrvPath), i->queryDrvPath()); + mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); - // Copy each output. - DrvInfo::Outputs outputs = i->queryOutputs(); + // Copy each output meant for installation. + DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; - foreach (DrvInfo::Outputs::iterator, j, outputs) { - mkString(*(vOutputs.list.elems[m++] = state.allocValue()), j->first); - Value & vOutputs = *state.allocAttr(v, state.symbols.create(j->first)); + for (auto & j : outputs) { + mkString(*(vOutputs.listElems()[m++] = state.allocValue()), j.first); + Value & vOutputs = *state.allocAttr(v, state.symbols.create(j.first)); state.mkAttrs(vOutputs, 2); - mkString(*state.allocAttr(vOutputs, state.sOutPath), j->second); + mkString(*state.allocAttr(vOutputs, state.sOutPath), j.second); /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ - store->addTempRoot(j->second); - store->ensurePath(j->second); + state.store->addTempRoot(j.second); + state.store->ensurePath(j.second); - references.insert(j->second); + references.insert(j.second); } // Copy the meta attributes. Value & vMeta = *state.allocAttr(v, state.sMeta); state.mkAttrs(vMeta, 16); - StringSet metaNames = i->queryMetaNames(); - foreach (StringSet::iterator, j, metaNames) { - Value * v = i->queryMeta(*j); + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + Value * v = i.queryMeta(j); if (!v) continue; - vMeta.attrs->push_back(Attr(state.symbols.create(*j), v)); + vMeta.attrs->push_back(Attr(state.symbols.create(j), v)); } vMeta.attrs->sort(); v.attrs->sort(); @@ -100,7 +100,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ - Path manifestFile = store->addTextToStore("env-manifest.nix", + Path manifestFile = state.store->addTextToStore("env-manifest.nix", (format("%1%") % manifest).str(), references); /* Get the environment builder expression. */ @@ -112,7 +112,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Value args, topLevel; state.mkAttrs(args, 3); mkString(*state.allocAttr(args, state.symbols.create("manifest")), - manifestFile, singleton<PathSet>(manifestFile)); + manifestFile, {manifestFile}); args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); args.attrs->sort(); mkApp(topLevel, envBuilder, args); @@ -128,7 +128,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Realise the resulting store expression. */ debug("building user environment"); - store->buildPaths(singleton<PathSet>(topLevelDrv), state.repair ? bmRepair : bmNormal); + state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ PathLocks lock; @@ -141,7 +141,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, } debug(format("switching to new user environment")); - Path generation = createGeneration(profile, topLevelOut); + Path generation = createGeneration(state.store, profile, topLevelOut); switchLink(profile, generation); return true; diff --git a/src/nix-hash/local.mk b/src/nix-hash/local.mk deleted file mode 100644 index 7c290ca8466e..000000000000 --- a/src/nix-hash/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-hash - -nix-hash_DIR := $(d) - -nix-hash_SOURCES := $(d)/nix-hash.cc - -nix-hash_LIBS = libmain libstore libutil libformat diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc deleted file mode 100644 index 8035162aea37..000000000000 --- a/src/nix-hash/nix-hash.cc +++ /dev/null @@ -1,63 +0,0 @@ -#include "hash.hh" -#include "shared.hh" - -#include <iostream> - -using namespace nix; - - -int main(int argc, char * * argv) -{ - HashType ht = htMD5; - bool flat = false; - bool base32 = false; - bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; - - Strings ss; - - return handleExceptions(argv[0], [&]() { - initNix(); - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-hash"); - else if (*arg == "--version") - printVersion("nix-hash"); - else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; - else if (*arg == "--truncate") truncate = true; - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - if (ht == htUnknown) - throw UsageError(format("unknown hash type ‘%1%’") % s); - } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - ss.push_back(*arg); - return true; - }); - - if (op == opHash) { - for (auto & i : ss) { - Hash h = flat ? hashFile(ht, i) : hashPath(ht, i).first; - if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - (base32 ? printHash32(h) : printHash(h)); - } - } - - else { - for (auto & i : ss) { - Hash h = parseHash16or32(ht, i); - std::cout << format("%1%\n") % - (op == opTo16 ? printHash(h) : printHash32(h)); - } - } - }); -} - diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index bea04180e513..7dce08400e82 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,7 +9,6 @@ #include "util.hh" #include "store-api.hh" #include "common-opts.hh" -#include "misc.hh" #include <map> #include <iostream> @@ -20,7 +19,7 @@ using namespace nix; static Expr * parseStdin(EvalState & state) { - startNest(nest, lvlTalkative, format("parsing standard input")); + Activity act(*logger, lvlTalkative, format("parsing standard input")); return state.parseExprFromString(drainFD(0), absPath(".")); } @@ -45,8 +44,8 @@ void processExpr(EvalState & state, const Strings & attrPaths, Value vRoot; state.eval(e, vRoot); - foreach (Strings::const_iterator, i, attrPaths) { - Value & v(*findAlongAttrPath(state, *i, autoArgs, vRoot)); + for (auto & i : attrPaths) { + Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); state.forceValue(v); PathSet context; @@ -67,11 +66,11 @@ void processExpr(EvalState & state, const Strings & attrPaths, } else { DrvInfos drvs; getDerivations(state, v, "", autoArgs, drvs, false); - foreach (DrvInfos::iterator, i, drvs) { - Path drvPath = i->queryDrvPath(); + for (auto & i : drvs) { + Path drvPath = i.queryDrvPath(); /* What output do we want? */ - string outputName = i->queryOutputName(); + string outputName = i.queryOutputName(); if (outputName == "") throw Error(format("derivation ‘%1%’ lacks an ‘outputName’ attribute ") % drvPath); @@ -79,8 +78,8 @@ void processExpr(EvalState & state, const Strings & attrPaths, printGCWarning(); else { Path rootName = gcRoot; - if (++rootNr > 1) rootName += "-" + int2String(rootNr); - drvPath = addPermRoot(*store, drvPath, rootName, indirectRoot); + if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); + drvPath = state.store->addPermRoot(drvPath, rootName, indirectRoot); } std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); } @@ -158,9 +157,9 @@ int main(int argc, char * * argv) if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; - store = openStore(); + auto store = openStore(); - EvalState state(searchPath); + EvalState state(searchPath, store); state.repair = repair; Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); @@ -168,9 +167,9 @@ int main(int argc, char * * argv) if (attrPaths.empty()) attrPaths.push_back(""); if (findFile) { - foreach (Strings::iterator, i, files) { - Path p = state.findFile(*i); - if (p == "") throw Error(format("unable to find ‘%1%’") % *i); + for (auto & i : files) { + Path p = state.findFile(i); + if (p == "") throw Error(format("unable to find ‘%1%’") % i); std::cout << p << std::endl; } return; diff --git a/src/nix-log2xml/local.mk b/src/nix-log2xml/local.mk deleted file mode 100644 index 09c848c17f40..000000000000 --- a/src/nix-log2xml/local.mk +++ /dev/null @@ -1,5 +0,0 @@ -programs += nix-log2xml - -nix-log2xml_DIR := $(d) - -nix-log2xml_SOURCES := $(d)/log2xml.cc diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc deleted file mode 100644 index 31cea60c3809..000000000000 --- a/src/nix-log2xml/log2xml.cc +++ /dev/null @@ -1,201 +0,0 @@ -#include <vector> -#include <iostream> -#include <cstdio> -#include <string> -#include <cstring> - -using namespace std; - - -struct Decoder -{ - enum { stTop, stEscape, stCSI } state; - string line; - bool inHeader; - int level; - vector<int> args; - bool newNumber; - int priority; - bool ignoreLF; - int lineNo, charNo; - bool warning; - bool error; - - Decoder() - { - state = stTop; - line = ""; - inHeader = false; - level = 0; - priority = 1; - ignoreLF = false; - lineNo = 1; - charNo = 0; - warning = false; - error = false; - } - - void pushChar(char c); - - void finishLine(); - - void decodeFile(istream & st); -}; - - -void Decoder::pushChar(char c) -{ - if (c == '\n') { - lineNo++; - charNo = 0; - } else charNo++; - - switch (state) { - - case stTop: - if (c == '\e') { - state = stEscape; - } else if (c == '\n' && !ignoreLF) { - finishLine(); - } else line += c; - break; - - case stEscape: - if (c == '[') { - state = stCSI; - args.clear(); - newNumber = true; - } else - state = stTop; /* !!! wrong */ - break; - - case stCSI: - if (c >= 0x40 && c != 0x7e) { - state = stTop; - switch (c) { - case 'p': - if (line.size()) finishLine(); - level++; - inHeader = true; - cout << "<nest>" << endl; - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'q': - if (line.size()) finishLine(); - if (level > 0) { - level--; - cout << "</nest>" << endl; - } else - cerr << "not enough nesting levels at line " - << lineNo << ", character " << charNo << endl; - break; - case 's': - if (line.size()) finishLine(); - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'a': - ignoreLF = true; - break; - case 'b': - ignoreLF = false; - break; - case 'e': - error = true; - break; - case 'w': - warning = true; - break; - } - } else if (c >= '0' && c <= '9') { - int n = 0; - if (!newNumber) { - n = args.back() * 10; - args.pop_back(); - } - n += c - '0'; - args.push_back(n); - } - break; - - } -} - - -void Decoder::finishLine() -{ - string storeDir = "/nix/store/"; - int sz = storeDir.size(); - string tag = inHeader ? "head" : "line"; - cout << "<" << tag; - if (priority != 1) cout << " priority='" << priority << "'"; - if (warning) cout << " warning='1'"; - if (error) cout << " error='1'"; - cout << ">"; - - for (unsigned int i = 0; i < line.size(); i++) { - - if (line[i] == '<') cout << "<"; - else if (line[i] == '&') cout << "&"; - else if (line[i] == '\r') ; /* ignore carriage return */ - else if (line[i] == '\n') cout << "\n"; - else if (line[i] >= 0 && line[i] < 32 && line[i] != 9) cout << "�"; - else if (i + sz + 33 < line.size() && - string(line, i, sz) == storeDir && - line[i + sz + 32] == '-') - { - int j = i + sz + 32; - /* skip name */ - while (!strchr("/\n\r\t ()[]:;?<>", line[j])) j++; - int k = j; - while (!strchr("\n\r\t ()[]:;?<>", line[k])) k++; - // !!! escaping - cout << "<storeref>" - << "<storedir>" - << string(line, i, sz) - << "</storedir>" - << "<hash>" - << string(line, i + sz, 32) - << "</hash>" - << "<name>" - << string(line, i + sz + 32, j - (i + sz + 32)) - << "</name>" - << "<path>" - << string(line, j, k - j) - << "</path>" - << "</storeref>"; - i = k - 1; - } else cout << line[i]; - } - - cout << "</" << tag << ">" << endl; - line = ""; - inHeader = false; - priority = 1; - warning = false; - error = false; -} - - -void Decoder::decodeFile(istream & st) -{ - int c; - - cout << "<logfile>" << endl; - - while ((c = st.get()) != EOF) { - pushChar(c); - } - - if (line.size()) finishLine(); - - while (level--) cout << "</nest>" << endl; - - cout << "</logfile>" << endl; -} - - -int main(int argc, char * * argv) -{ - Decoder dec; - dec.decodeFile(cin); -} diff --git a/src/nix-log2xml/logfile.css b/src/nix-log2xml/logfile.css deleted file mode 100644 index ed390d64a9ef..000000000000 --- a/src/nix-log2xml/logfile.css +++ /dev/null @@ -1,86 +0,0 @@ -body { - font-family: sans-serif; - background: white; -} - - -ul.nesting, ul.toplevel { - padding: 0; - margin: 0; -} - -ul.toplevel { - list-style-type: none; -} - -ul.nesting li.line, ul.nesting li.lastline { - position: relative; - list-style-type: none; -} - -ul.nesting li.line { - padding-left: 1.1em; -} - -ul.nesting li.lastline { - padding-left: 1.2em; // for the 0.1em border-left in .lastline > .lineconn -} - -li.line { - border-left: 0.1em solid #6185a0; -} - -li.line > span.lineconn, li.lastline > span.lineconn { - position: absolute; - height: 0.65em; - left: 0em; - width: 1em; - border-bottom: 0.1em solid #6185a0; -} - -li.lastline > span.lineconn { - border-left: 0.1em solid #6185a0; -} - - -em.storeref { - color: #500000; - position: relative; - width: 100%; -} - -em.storeref:hover { - background-color: #eeeeee; -} - -*.popup { - display: none; -/* background: url('http://losser.st-lab.cs.uu.nl/~mbravenb/menuback.png') repeat; */ - background: #ffffcd; - border: solid #555555 1px; - position: absolute; - top: 0em; - left: 0em; - margin: 0; - padding: 0; - z-index: 100; -} - -em.storeref:hover span.popup { - display: inline; -} - - -.toggle { - text-decoration: none; -} - -.showTree, .hideTree { - font-family: monospace; - font-size: larger; -} - -.error { - color: #ff0000; - font-weight: bold; -} \ No newline at end of file diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk new file mode 100644 index 000000000000..3e7735406af0 --- /dev/null +++ b/src/nix-prefetch-url/local.mk @@ -0,0 +1,7 @@ +programs += nix-prefetch-url + +nix-prefetch-url_DIR := $(d) + +nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc + +nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc new file mode 100644 index 000000000000..64da10513711 --- /dev/null +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -0,0 +1,210 @@ +#include "hash.hh" +#include "shared.hh" +#include "download.hh" +#include "store-api.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "common-opts.hh" +#include "attr-path.hh" + +#include <iostream> + +using namespace nix; + + +/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUri(EvalState & state, string uri) +{ + if (string(uri, 0, 9) != "mirror://") return uri; + + string s(uri, 9); + auto p = s.find('/'); + if (p == string::npos) throw Error("invalid mirror URI"); + string mirrorName(s, 0, p); + + Value vMirrors; + state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error(format("unknown mirror name ‘%1%’") % mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error(format("mirror URI ‘%1%’ did not expand to anything") % uri); + + string mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + + +int main(int argc, char * * argv) +{ + return handleExceptions(argv[0], [&]() { + initNix(); + initGC(); + + HashType ht = htSHA256; + std::vector<string> args; + Strings searchPath; + bool printPath = getEnv("PRINT_PATH") != ""; + bool fromExpr = false; + string attrPath; + std::map<string, string> autoArgs_; + bool unpack = false; + string name; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (parseAutoArgs(arg, end, autoArgs_)) + ; + else if (parseSearchPathArg(arg, end, searchPath)) + ; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + auto store = openStore(); + EvalState state(searchPath, store); + + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + + /* If -A is given, get the URI from the specified Nix + expression. */ + string uri; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URI"); + uri = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(state, args.empty() ? "." : args[0])); + Value vRoot; + state.evalFile(path, vRoot); + Value & v(*findAlongAttrPath(state, attrPath, autoArgs, vRoot)); + state.forceAttrs(v); + + /* Extract the URI. */ + auto attr = v.attrs->find(state.symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a ‘urls’ attribute"); + state.forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("‘urls’ list is empty"); + uri = state.forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state.symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printMsg(lvlInfo, "warning: this does not look like a fetchurl call"); + else + unpack = state.forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (name.empty()) { + attr = v.attrs->find(state.symbols.create("name")); + if (attr != v.attrs->end()) + name = state.forceString(*attr->value); + } + } + + /* Figure out a name in the Nix store. */ + if (name.empty()) + name = baseNameOf(uri); + if (name.empty()) + throw Error(format("cannot figure out file name for ‘%1%’") % uri); + + /* If an expected hash is given, the file may already exist in + the store. */ + Hash hash, expectedHash(ht); + Path storePath; + if (args.size() == 2) { + expectedHash = parseHash16or32(ht, args[1]); + storePath = makeFixedOutputPath(unpack, ht, expectedHash, name); + if (store->isValidPath(storePath)) + hash = expectedHash; + else + storePath.clear(); + } + + if (storePath.empty()) { + + auto actualUri = resolveMirrorUri(state, uri); + + /* Download the file. */ + auto result = makeDownloader()->download(actualUri, DownloadOptions()); + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + writeFile(tmpFile, *result.data); + + /* Optionally unpack the file. */ + if (unpack) { + printMsg(lvlInfo, "unpacking..."); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + if (hasSuffix(baseNameOf(uri), ".zip")) + runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}, ""); + else + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}, ""); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + /* FIXME: inefficient; addToStore() will also hash + this. */ + hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data); + + if (expectedHash != Hash(ht) && expectedHash != hash) + throw Error(format("hash mismatch for ‘%1%’") % uri); + + /* Copy the file to the Nix store. FIXME: if RemoteStore + implemented addToStoreFromDump() and downloadFile() + supported a sink, we could stream the download directly + into the Nix store. */ + storePath = store->addToStore(name, tmpFile, unpack, ht); + + assert(storePath == makeFixedOutputPath(unpack, ht, hash, name)); + } + + if (!printPath) + printMsg(lvlInfo, format("path is ‘%1%’") % storePath); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << storePath << std::endl; + }); +} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index a333d7351010..356a82510124 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -20,8 +20,8 @@ static string nextColour() { static int n = 0; static string colours[] = - { "black", "red", "green", "blue" - , "magenta", "burlywood" }; + { "black", "red", "green", "blue" + , "magenta", "burlywood" }; return colours[n++ % (sizeof(colours) / sizeof(string))]; } @@ -29,7 +29,7 @@ static string nextColour() static string makeEdge(const string & src, const string & dst) { format f = format("%1% -> %2% [color = %3%];\n") - % dotQuote(src) % dotQuote(dst) % dotQuote(nextColour()); + % dotQuote(src) % dotQuote(dst) % dotQuote(nextColour()); return f.str(); } @@ -38,8 +38,8 @@ static string makeNode(const string & id, const string & label, const string & colour) { format f = format("%1% [label = %2%, shape = box, " - "style = filled, fillcolor = %3%];\n") - % dotQuote(id) % dotQuote(label) % dotQuote(colour); + "style = filled, fillcolor = %3%];\n") + % dotQuote(id) % dotQuote(label) % dotQuote(colour); return f.str(); } @@ -65,98 +65,92 @@ void printClosure(const Path & nePath, const StoreExpr & fs) PathSet doneSet; for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) { - cout << makeEdge(pathLabel(nePath, *i), nePath); + cout << makeEdge(pathLabel(nePath, *i), nePath); } while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) == doneSet.end()) { - doneSet.insert(path); - - ClosureElems::const_iterator elem = fs.closure.elems.find(path); - if (elem == fs.closure.elems.end()) - throw Error(format("bad closure, missing path ‘%1%’") % path); - - for (StringSet::const_iterator i = elem->second.refs.begin(); - i != elem->second.refs.end(); ++i) - { - workList.insert(*i); - cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); - } - - cout << makeNode(pathLabel(nePath, path), - symbolicName(path), "#ff0000"); - } + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) == doneSet.end()) { + doneSet.insert(path); + + ClosureElems::const_iterator elem = fs.closure.elems.find(path); + if (elem == fs.closure.elems.end()) + throw Error(format("bad closure, missing path ‘%1%’") % path); + + for (StringSet::const_iterator i = elem->second.refs.begin(); + i != elem->second.refs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); + } + + cout << makeNode(pathLabel(nePath, path), + symbolicName(path), "#ff0000"); + } } } #endif -void printDotGraph(const PathSet & roots) +void printDotGraph(ref<Store> store, const PathSet & roots) { PathSet workList(roots); PathSet doneSet; - + cout << "digraph G {\n"; while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); + Path path = *(workList.begin()); + workList.erase(path); - if (doneSet.find(path) != doneSet.end()) continue; + if (doneSet.find(path) != doneSet.end()) continue; doneSet.insert(path); cout << makeNode(path, symbolicName(path), "#ff0000"); - - PathSet references; - store->queryReferences(path, references); - - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) - { - if (*i != path) { + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); + } + } + +#if 0 + StoreExpr ne = storeExprFromPath(path); + + string label, colour; + + if (ne.type == StoreExpr::neDerivation) { + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); ++i) + { workList.insert(*i); cout << makeEdge(*i, path); } + + label = "derivation"; + colour = "#00ff00"; + for (StringPairs::iterator i = ne.derivation.env.begin(); + i != ne.derivation.env.end(); ++i) + if (i->first == "name") label = i->second; } - - -#if 0 - StoreExpr ne = storeExprFromPath(path); - - string label, colour; - - if (ne.type == StoreExpr::neDerivation) { - for (PathSet::iterator i = ne.derivation.inputs.begin(); - i != ne.derivation.inputs.end(); ++i) - { - workList.insert(*i); - cout << makeEdge(*i, path); - } - - label = "derivation"; - colour = "#00ff00"; - for (StringPairs::iterator i = ne.derivation.env.begin(); - i != ne.derivation.env.end(); ++i) - if (i->first == "name") label = i->second; - } - - else if (ne.type == StoreExpr::neClosure) { - label = "<closure>"; - colour = "#00ffff"; - printClosure(path, ne); - } - - else abort(); - - cout << makeNode(path, label, colour); + + else if (ne.type == StoreExpr::neClosure) { + label = "<closure>"; + colour = "#00ffff"; + printClosure(path, ne); + } + + else abort(); + + cout << makeNode(path, label, colour); #endif } cout << "}\n"; } - + } diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 68410d84156d..e2b5fc72fbe1 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -4,6 +4,8 @@ namespace nix { -void printDotGraph(const PathSet & roots); +class Store; + +void printDotGraph(ref<Store> store, const PathSet & roots); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 23b97ca9e5aa..0038fff036a1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1,14 +1,15 @@ -#include "globals.hh" -#include "misc.hh" #include "archive.hh" -#include "shared.hh" +#include "derivations.hh" #include "dotgraph.hh" -#include "xmlgraph.hh" +#include "globals.hh" #include "local-store.hh" -#include "util.hh" +#include "monitor-fd.hh" #include "serve-protocol.hh" +#include "shared.hh" +#include "util.hh" #include "worker-protocol.hh" -#include "monitor-fd.hh" +#include "xmlgraph.hh" +#include "compression.hh" #include <iostream> #include <algorithm> @@ -18,8 +19,6 @@ #include <sys/stat.h> #include <fcntl.h> -#include <bzlib.h> - #if HAVE_SODIUM #include <sodium.h> #endif @@ -37,20 +36,21 @@ static Path gcRoot; static int rootNr = 0; static bool indirectRoot = false; static bool noOutput = false; +static std::shared_ptr<Store> store; -LocalStore & ensureLocalStore() +ref<LocalStore> ensureLocalStore() { - LocalStore * store2(dynamic_cast<LocalStore *>(store.get())); + auto store2 = std::dynamic_pointer_cast<LocalStore>(store); if (!store2) throw Error("you don't have sufficient rights to use this command"); - return *store2; + return ref<LocalStore>(store2); } static Path useDeriver(Path path) { if (isDerivation(path)) return path; - Path drvPath = store->queryDeriver(path); + Path drvPath = store->queryPathInfo(path)->deriver; if (drvPath == "") throw Error(format("deriver of path ‘%1%’ is not known") % path); return drvPath; @@ -64,26 +64,26 @@ static PathSet realisePath(Path path, bool build = true) DrvPathWithOutputs p = parseDrvPathWithOutputs(path); if (isDerivation(p.first)) { - if (build) store->buildPaths(singleton<PathSet>(path)); - Derivation drv = derivationFromPath(*store, p.first); + if (build) store->buildPaths({path}); + Derivation drv = store->derivationFromPath(p.first); rootNr++; if (p.second.empty()) - foreach (DerivationOutputs::iterator, i, drv.outputs) p.second.insert(i->first); + for (auto & i : drv.outputs) p.second.insert(i.first); PathSet outputs; - foreach (StringSet::iterator, j, p.second) { - DerivationOutputs::iterator i = drv.outputs.find(*j); + for (auto & j : p.second) { + DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) - throw Error(format("derivation ‘%1%’ does not have an output named ‘%2%’") % p.first % *j); + throw Error(format("derivation ‘%1%’ does not have an output named ‘%2%’") % p.first % j); Path outPath = i->second.path; if (gcRoot == "") printGCWarning(); else { Path rootName = gcRoot; - if (rootNr > 1) rootName += "-" + int2String(rootNr); + if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (i->first != "out") rootName += "-" + i->first; - outPath = addPermRoot(*store, outPath, rootName, indirectRoot); + outPath = store->addPermRoot(outPath, rootName, indirectRoot); } outputs.insert(outPath); } @@ -98,10 +98,10 @@ static PathSet realisePath(Path path, bool build = true) else { Path rootName = gcRoot; rootNr++; - if (rootNr > 1) rootName += "-" + int2String(rootNr); - path = addPermRoot(*store, path, rootName, indirectRoot); + if (rootNr > 1) rootName += "-" + std::to_string(rootNr); + path = store->addPermRoot(path, rootName, indirectRoot); } - return singleton<PathSet>(path); + return {path}; } } @@ -113,34 +113,35 @@ static void opRealise(Strings opFlags, Strings opArgs) BuildMode buildMode = bmNormal; bool ignoreUnknown = false; - foreach (Strings::iterator, i, opFlags) - if (*i == "--dry-run") dryRun = true; - else if (*i == "--repair") buildMode = bmRepair; - else if (*i == "--check") buildMode = bmCheck; - else if (*i == "--ignore-unknown") ignoreUnknown = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--dry-run") dryRun = true; + else if (i == "--repair") buildMode = bmRepair; + else if (i == "--check") buildMode = bmCheck; + else if (i == "--hash") buildMode = bmHash; + else if (i == "--ignore-unknown") ignoreUnknown = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); Paths paths; - foreach (Strings::iterator, i, opArgs) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(*i); + for (auto & i : opArgs) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(i); paths.push_back(makeDrvPathWithOutputs(followLinksToStorePath(p.first), p.second)); } unsigned long long downloadSize, narSize; PathSet willBuild, willSubstitute, unknown; - queryMissing(*store, PathSet(paths.begin(), paths.end()), + store->queryMissing(PathSet(paths.begin(), paths.end()), willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { Paths paths2; - foreach (Paths::iterator, i, paths) - if (unknown.find(*i) == unknown.end()) paths2.push_back(*i); + for (auto & i : paths) + if (unknown.find(i) == unknown.end()) paths2.push_back(i); paths = paths2; unknown = PathSet(); } if (settings.get("print-missing", true)) - printMissing(willBuild, willSubstitute, unknown, downloadSize, narSize); + printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (dryRun) return; @@ -148,11 +149,11 @@ static void opRealise(Strings opFlags, Strings opArgs) store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); if (!ignoreUnknown) - foreach (Paths::iterator, i, paths) { - PathSet paths = realisePath(*i, false); + for (auto & i : paths) { + PathSet paths = realisePath(i, false); if (!noOutput) - foreach (PathSet::iterator, j, paths) - cout << format("%1%\n") % *j; + for (auto & j : paths) + cout << format("%1%\n") % j; } } @@ -173,10 +174,9 @@ static void opAddFixed(Strings opFlags, Strings opArgs) { bool recursive = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--recursive") recursive = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); @@ -194,10 +194,9 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { bool recursive = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto i : opFlags) + if (i == "--recursive") recursive = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); if (opArgs.size() != 3) throw UsageError(format("‘--print-fixed-path’ requires three arguments")); @@ -217,13 +216,13 @@ static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forc { if (forceRealise) realisePath(storePath); if (useOutput && isDerivation(storePath)) { - Derivation drv = derivationFromPath(*store, storePath); + Derivation drv = store->derivationFromPath(storePath); PathSet outputs; - foreach (DerivationOutputs::iterator, i, drv.outputs) - outputs.insert(i->second.path); + for (auto & i : drv.outputs) + outputs.insert(i.second.path); return outputs; } - else return singleton<PathSet>(storePath); + else return {storePath}; } @@ -247,18 +246,17 @@ static void printTree(const Path & path, cout << format("%1%%2%\n") % firstPad % path; - PathSet references; - store->queryReferences(path, references); + auto references = store->queryPathInfo(path)->references; /* Topologically sort under the relation A < B iff A \in closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - Paths sorted = topoSortPaths(*store, references); + Paths sorted = store->topoSortPaths(references); reverse(sorted.begin(), sorted.end()); - foreach (Paths::iterator, i, sorted) { - Paths::iterator j = i; ++j; + for (auto i = sorted.begin(); i != sorted.end(); ++i) { + auto j = i; ++j; printTree(*i, tailPad + treeConn, j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, done); @@ -279,34 +277,34 @@ static void opQuery(Strings opFlags, Strings opArgs) bool forceRealise = false; string bindingName; - foreach (Strings::iterator, i, opFlags) { + for (auto & i : opFlags) { QueryType prev = query; - if (*i == "--outputs") query = qOutputs; - else if (*i == "--requisites" || *i == "-R") query = qRequisites; - else if (*i == "--references") query = qReferences; - else if (*i == "--referrers" || *i == "--referers") query = qReferrers; - else if (*i == "--referrers-closure" || *i == "--referers-closure") query = qReferrersClosure; - else if (*i == "--deriver" || *i == "-d") query = qDeriver; - else if (*i == "--binding" || *i == "-b") { + if (i == "--outputs") query = qOutputs; + else if (i == "--requisites" || i == "-R") query = qRequisites; + else if (i == "--references") query = qReferences; + else if (i == "--referrers" || i == "--referers") query = qReferrers; + else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; + else if (i == "--deriver" || i == "-d") query = qDeriver; + else if (i == "--binding" || i == "-b") { if (opArgs.size() == 0) throw UsageError("expected binding name"); bindingName = opArgs.front(); opArgs.pop_front(); query = qBinding; } - else if (*i == "--hash") query = qHash; - else if (*i == "--size") query = qSize; - else if (*i == "--tree") query = qTree; - else if (*i == "--graph") query = qGraph; - else if (*i == "--xml") query = qXml; - else if (*i == "--resolve") query = qResolve; - else if (*i == "--roots") query = qRoots; - else if (*i == "--use-output" || *i == "-u") useOutput = true; - else if (*i == "--force-realise" || *i == "--force-realize" || *i == "-f") forceRealise = true; - else if (*i == "--include-outputs") includeOutputs = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + else if (i == "--hash") query = qHash; + else if (i == "--size") query = qSize; + else if (i == "--tree") query = qTree; + else if (i == "--graph") query = qGraph; + else if (i == "--xml") query = qXml; + else if (i == "--resolve") query = qResolve; + else if (i == "--roots") query = qRoots; + else if (i == "--use-output" || i == "-u") useOutput = true; + else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; + else if (i == "--include-outputs") includeOutputs = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); if (prev != qDefault && prev != query) - throw UsageError(format("query type ‘%1%’ conflicts with earlier flag") % *i); + throw UsageError(format("query type ‘%1%’ conflicts with earlier flag") % i); } if (query == qDefault) query = qOutputs; @@ -316,12 +314,12 @@ static void opQuery(Strings opFlags, Strings opArgs) switch (query) { case qOutputs: { - foreach (Strings::iterator, i, opArgs) { - *i = followLinksToStorePath(*i); - if (forceRealise) realisePath(*i); - Derivation drv = derivationFromPath(*store, *i); - foreach (DerivationOutputs::iterator, j, drv.outputs) - cout << format("%1%\n") % j->second.path; + for (auto & i : opArgs) { + i = followLinksToStorePath(i); + if (forceRealise) realisePath(i); + Derivation drv = store->derivationFromPath(i); + for (auto & j : drv.outputs) + cout << format("%1%\n") % j.second.path; } break; } @@ -331,16 +329,19 @@ static void opQuery(Strings opFlags, Strings opArgs) case qReferrers: case qReferrersClosure: { PathSet paths; - foreach (Strings::iterator, i, opArgs) { - PathSet ps = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - foreach (PathSet::iterator, j, ps) { - if (query == qRequisites) computeFSClosure(*store, *j, paths, false, includeOutputs); - else if (query == qReferences) store->queryReferences(*j, paths); - else if (query == qReferrers) store->queryReferrers(*j, paths); - else if (query == qReferrersClosure) computeFSClosure(*store, *j, paths, true); + for (auto & i : opArgs) { + PathSet ps = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); + for (auto & j : ps) { + if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); + else if (query == qReferences) { + for (auto & p : store->queryPathInfo(j)->references) + paths.insert(p); + } + else if (query == qReferrers) store->queryReferrers(j, paths); + else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); } } - Paths sorted = topoSortPaths(*store, paths); + Paths sorted = store->topoSortPaths(paths); for (Paths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) cout << format("%s\n") % *i; @@ -348,17 +349,17 @@ static void opQuery(Strings opFlags, Strings opArgs) } case qDeriver: - foreach (Strings::iterator, i, opArgs) { - Path deriver = store->queryDeriver(followLinksToStorePath(*i)); + for (auto & i : opArgs) { + Path deriver = store->queryPathInfo(followLinksToStorePath(i))->deriver; cout << format("%1%\n") % (deriver == "" ? "unknown-deriver" : deriver); } break; case qBinding: - foreach (Strings::iterator, i, opArgs) { - Path path = useDeriver(followLinksToStorePath(*i)); - Derivation drv = derivationFromPath(*store, path); + for (auto & i : opArgs) { + Path path = useDeriver(followLinksToStorePath(i)); + Derivation drv = store->derivationFromPath(path); StringPairs::iterator j = drv.env.find(bindingName); if (j == drv.env.end()) throw Error(format("derivation ‘%1%’ has no environment binding named ‘%2%’") @@ -369,64 +370,64 @@ static void opQuery(Strings opFlags, Strings opArgs) case qHash: case qSize: - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - foreach (PathSet::iterator, j, paths) { - ValidPathInfo info = store->queryPathInfo(*j); + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); + for (auto & j : paths) { + auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info.hash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(info.hash); + assert(info->narHash.type == htSHA256); + cout << format("sha256:%1%\n") % printHash32(info->narHash); } else if (query == qSize) - cout << format("%1%\n") % info.narSize; + cout << format("%1%\n") % info->narSize; } } break; case qTree: { PathSet done; - foreach (Strings::iterator, i, opArgs) - printTree(followLinksToStorePath(*i), "", "", done); + for (auto & i : opArgs) + printTree(followLinksToStorePath(i), "", "", done); break; } case qGraph: { PathSet roots; - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); roots.insert(paths.begin(), paths.end()); } - printDotGraph(roots); + printDotGraph(ref<Store>(store), roots); break; } case qXml: { PathSet roots; - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); roots.insert(paths.begin(), paths.end()); } - printXmlGraph(roots); + printXmlGraph(ref<Store>(store), roots); break; } case qResolve: { - foreach (Strings::iterator, i, opArgs) - cout << format("%1%\n") % followLinksToStorePath(*i); + for (auto & i : opArgs) + cout << format("%1%\n") % followLinksToStorePath(i); break; } case qRoots: { PathSet referrers; - foreach (Strings::iterator, i, opArgs) { - PathSet paths = maybeUseOutputs(followLinksToStorePath(*i), useOutput, forceRealise); - foreach (PathSet::iterator, j, paths) - computeFSClosure(*store, *j, referrers, true, + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); + for (auto & j : paths) + store->computeFSClosure(j, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - foreach (Roots::iterator, i, roots) - if (referrers.find(i->second) != referrers.end()) - cout << format("%1%\n") % i->first; + for (auto & i : roots) + if (referrers.find(i.second) != referrers.end()) + cout << format("%1%\n") % i.first; break; } @@ -439,8 +440,8 @@ static void opQuery(Strings opFlags, Strings opArgs) static string shellEscape(const string & s) { string r; - foreach (string::const_iterator, i, s) - if (*i == '\'') r += "'\\''"; else r += *i; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; return r; } @@ -451,19 +452,21 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) if (opArgs.size() != 1) throw UsageError("‘--print-env’ requires one derivation store path"); Path drvPath = opArgs.front(); - Derivation drv = derivationFromPath(*store, drvPath); + Derivation drv = store->derivationFromPath(drvPath); /* Print each environment variable in the derivation in a format that can be sourced by the shell. */ - foreach (StringPairs::iterator, i, drv.env) - cout << format("export %1%; %1%='%2%'\n") % i->first % shellEscape(i->second); + for (auto & i : drv.env) + cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ cout << "export _args; _args='"; - foreach (Strings::iterator, i, drv.args) { - if (i != drv.args.begin()) cout << ' '; - cout << shellEscape(*i); + bool first = true; + for (auto & i : drv.args) { + if (!first) cout << ' '; + first = false; + cout << shellEscape(i); } cout << "'\n"; } @@ -475,8 +478,8 @@ static void opReadLog(Strings opFlags, Strings opArgs) RunPager pager; - foreach (Strings::iterator, i, opArgs) { - Path path = useDeriver(followLinksToStorePath(*i)); + for (auto & i : opArgs) { + Path path = useDeriver(followLinksToStorePath(i)); string baseName = baseNameOf(path); bool found = false; @@ -498,21 +501,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) } else if (pathExists(logBz2Path)) { - AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); - FILE * f = 0; - if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) - throw SysError(format("opening file ‘%1%’") % logBz2Path); - int err; - BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); - if (!bz) throw Error(format("cannot open bzip2 file ‘%1%’") % logBz2Path); - unsigned char buf[128 * 1024]; - do { - int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); - if (err != BZ_OK && err != BZ_STREAM_END) - throw Error(format("error reading bzip2 file ‘%1%’") % logBz2Path); - writeFull(STDOUT_FILENO, buf, n); - } while (err != BZ_STREAM_END); - BZ2_bzReadClose(&err, bz); + std::cout << *decompress("bzip2", readFile(logBz2Path)); found = true; break; } @@ -547,8 +536,8 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); PathSet validPaths = store->queryAllValidPaths(); - foreach (PathSet::iterator, i, validPaths) - cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true); + for (auto & i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); } @@ -565,14 +554,14 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) canonicalisePathMetaData(info.path, -1); if (!hashGiven) { HashResult hash = hashPath(htSHA256, info.path); - info.hash = hash.first; + info.narHash = hash.first; info.narSize = hash.second; } infos.push_back(info); } } - ensureLocalStore().registerValidPaths(infos); + ensureLocalStore()->registerValidPaths(infos); } @@ -590,11 +579,10 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) bool reregister = false; // !!! maybe this should be the default bool hashGiven = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--reregister") reregister = true; - else if (*i == "--hash-given") hashGiven = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--reregister") reregister = true; + else if (i == "--hash-given") hashGiven = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -606,15 +594,12 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) { bool printInvalid = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--print-invalid") printInvalid = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--print-invalid") printInvalid = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); - for (Strings::iterator i = opArgs.begin(); - i != opArgs.end(); ++i) - { - Path path = followLinksToStorePath(*i); + for (auto & i : opArgs) { + Path path = followLinksToStorePath(i); if (!store->isValidPath(path)) { if (printInvalid) cout << format("%1%\n") % path; @@ -634,7 +619,7 @@ static void opGC(Strings opFlags, Strings opArgs) GCResults results; /* Do what? */ - foreach (Strings::iterator, i, opFlags) + for (auto i = opFlags.begin(); i != opFlags.end(); ++i) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; @@ -649,8 +634,8 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(); - foreach (Roots::iterator, i, roots) - cout << i->first << " -> " << i->second << std::endl; + for (auto & i : roots) + cout << i.first << " -> " << i.second << std::endl; } else { @@ -658,8 +643,8 @@ static void opGC(Strings opFlags, Strings opArgs) store->collectGarbage(options, results); if (options.action != GCOptions::gcDeleteDead) - foreach (PathSet::iterator, i, results.paths) - cout << *i << std::endl; + for (auto & i : results.paths) + cout << i << std::endl; } } @@ -672,12 +657,12 @@ static void opDelete(Strings opFlags, Strings opArgs) GCOptions options; options.action = GCOptions::gcDeleteSpecific; - foreach (Strings::iterator, i, opFlags) - if (*i == "--ignore-liveness") options.ignoreLiveness = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--ignore-liveness") options.ignoreLiveness = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); - foreach (Strings::iterator, i, opArgs) - options.pathsToDelete.insert(followLinksToStorePath(*i)); + for (auto & i : opArgs) + options.pathsToDelete.insert(followLinksToStorePath(i)); GCResults results; PrintFreed freed(true, results); @@ -712,33 +697,26 @@ static void opRestore(Strings opFlags, Strings opArgs) static void opExport(Strings opFlags, Strings opArgs) { - bool sign = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--sign") sign = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + throw UsageError(format("unknown flag ‘%1%’") % i); FdSink sink(STDOUT_FILENO); - Paths sorted = topoSortPaths(*store, PathSet(opArgs.begin(), opArgs.end())); - reverse(sorted.begin(), sorted.end()); - exportPaths(*store, sorted, sign, sink); + store->exportPaths(opArgs, sink); } static void opImport(Strings opFlags, Strings opArgs) { - bool requireSignature = false; - foreach (Strings::iterator, i, opFlags) - if (*i == "--require-signature") requireSignature = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + throw UsageError(format("unknown flag ‘%1%’") % i); if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(requireSignature, source); + Paths paths = store->importPaths(source, 0); - foreach (Paths::iterator, i, paths) - cout << format("%1%\n") % *i << std::flush; + for (auto & i : paths) + cout << format("%1%\n") % i << std::flush; } @@ -762,11 +740,10 @@ static void opVerify(Strings opFlags, Strings opArgs) bool checkContents = false; bool repair = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) - if (*i == "--check-contents") checkContents = true; - else if (*i == "--repair") repair = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--check-contents") checkContents = true; + else if (i == "--repair") repair = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); if (store->verifyStore(checkContents, repair)) { printMsg(lvlError, "warning: not all errors were fixed"); @@ -783,15 +760,17 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) int status = 0; - foreach (Strings::iterator, i, opArgs) { - Path path = followLinksToStorePath(*i); + for (auto & i : opArgs) { + Path path = followLinksToStorePath(i); printMsg(lvlTalkative, format("checking path ‘%1%’...") % path); - ValidPathInfo info = store->queryPathInfo(path); - HashResult current = hashPath(info.hash.type, path); - if (current.first != info.hash) { + auto info = store->queryPathInfo(path); + HashSink sink(info->narHash.type); + store->narFromPath(path, sink); + auto current = sink.finish(); + if (current.first != info->narHash) { printMsg(lvlError, format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’") - % path % printHash(info.hash) % printHash(current.first)); + % path % printHash(info->narHash) % printHash(current.first)); status = 1; } } @@ -807,9 +786,9 @@ static void opRepairPath(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("no flags expected"); - foreach (Strings::iterator, i, opArgs) { - Path path = followLinksToStorePath(*i); - ensureLocalStore().repairPath(path); + for (auto & i : opArgs) { + Path path = followLinksToStorePath(i); + ensureLocalStore()->repairPath(path); } } @@ -823,31 +802,13 @@ static void opOptimise(Strings opFlags, Strings opArgs) store->optimiseStore(); } -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); - PathSet failed = store->queryFailedPaths(); - foreach (PathSet::iterator, i, failed) - cout << format("%1%\n") % *i; -} - - -static void opClearFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); -} - - /* Serve the nix store in a way usable by a restricted ssh user. */ static void opServe(Strings opFlags, Strings opArgs) { bool writeAllowed = false; - foreach (Strings::iterator, i, opFlags) - if (*i == "--write") writeAllowed = true; - else throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + if (i == "--write") writeAllowed = true; + else throw UsageError(format("unknown flag ‘%1%’") % i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -857,10 +818,21 @@ static void opServe(Strings opFlags, Strings opArgs) /* Exchange the greeting. */ unsigned int magic = readInt(in); if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - writeInt(SERVE_MAGIC_2, out); - writeInt(SERVE_PROTOCOL_VERSION, out); + out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; out.flush(); - readInt(in); // Client version, unused for now + unsigned int clientVersion = readInt(in); + + auto getBuildSettings = [&]() { + // FIXME: changing options here doesn't work if we're + // building through the daemon. + verbosity = lvlError; + settings.keepLog = false; + settings.useSubstitutes = false; + settings.maxSilentTime = readInt(in); + settings.buildTimeout = readInt(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 2) + settings.maxLogSize = readInt(in); + }; while (true) { ServeCommand cmd; @@ -890,7 +862,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!isDerivation(path)) paths2.insert(path); unsigned long long downloadSize, narSize; PathSet willBuild, willSubstitute, unknown; - queryMissing(*store, PathSet(paths2.begin(), paths2.end()), + store->queryMissing(PathSet(paths2.begin(), paths2.end()), willBuild, willSubstitute, unknown, downloadSize, narSize); /* FIXME: should use ensurePath(), but it only does one path at a time. */ @@ -902,25 +874,24 @@ static void opServe(Strings opFlags, Strings opArgs) } } - writeStrings(store->queryValidPaths(paths), out); + out << store->queryValidPaths(paths); break; } case cmdQueryPathInfos: { PathSet paths = readStorePaths<PathSet>(in); // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - if (!store->isValidPath(*i)) - continue; - ValidPathInfo info = store->queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); + for (auto & i : paths) { + try { + auto info = store->queryPathInfo(i); + out << info->path << info->deriver << info->references; + // !!! Maybe we want compression? + out << info->narSize // downloadSize + << info->narSize; + } catch (InvalidPath &) { + } } - writeString("", out); + out << ""; break; } @@ -930,52 +901,62 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); - store->importPaths(false, in); - writeInt(1, out); // indicate success + store->importPaths(in, 0); + out << 1; // indicate success break; } case cmdExportPaths: { - bool sign = readInt(in); - Paths sorted = topoSortPaths(*store, readStorePaths<PathSet>(in)); + readInt(in); // obsolete + Paths sorted = store->topoSortPaths(readStorePaths<PathSet>(in)); reverse(sorted.begin(), sorted.end()); - exportPaths(*store, sorted, sign, out); + store->exportPaths(sorted, out); break; } - case cmdBuildPaths: { + case cmdBuildPaths: { /* Used by build-remote.pl. */ - /* Used by build-remote.pl. */ if (!writeAllowed) throw Error("building paths is not allowed"); PathSet paths = readStorePaths<PathSet>(in); - // FIXME: changing options here doesn't work if we're - // building through the daemon. - verbosity = lvlError; - settings.keepLog = false; - settings.useSubstitutes = false; - settings.maxSilentTime = readInt(in); - settings.buildTimeout = readInt(in); + getBuildSettings(); try { MonitorFdHup monitor(in.fd); store->buildPaths(paths); - writeInt(0, out); + out << 0; } catch (Error & e) { assert(e.status); - writeInt(e.status, out); - writeString(e.msg(), out); + out << e.status << e.msg(); } break; } + case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + + if (!writeAllowed) throw Error("building paths is not allowed"); + + Path drvPath = readStorePath(in); // informational only + BasicDerivation drv; + in >> drv; + + getBuildSettings(); + + MonitorFdHup monitor(in.fd); + auto status = store->buildDerivation(drvPath, drv); + + out << status.status << status.errorMsg; + + break; + } + case cmdQueryClosure: { bool includeOutputs = readInt(in); PathSet paths = readStorePaths<PathSet>(in); PathSet closure; for (auto & i : paths) - computeFSClosure(*store, i, closure, false, includeOutputs); - writeStrings(closure, out); + store->computeFSClosure(i, closure, false, includeOutputs); + out << closure; break; } @@ -990,8 +971,8 @@ static void opServe(Strings opFlags, Strings opArgs) static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) { - foreach (Strings::iterator, i, opFlags) - throw UsageError(format("unknown flag ‘%1%’") % *i); + for (auto & i : opFlags) + throw UsageError(format("unknown flag ‘%1%’") % i); if (opArgs.size() != 3) throw UsageError("three arguments expected"); auto i = opArgs.begin(); @@ -1000,7 +981,8 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string publicKeyFile = *i++; #if HAVE_SODIUM - sodium_init(); + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; @@ -1016,6 +998,12 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) } +static void opVersion(Strings opFlags, Strings opArgs) +{ + printVersion("nix-store"); +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -1033,7 +1021,7 @@ int main(int argc, char * * argv) if (*arg == "--help") showManPage("nix-store"); else if (*arg == "--version") - printVersion("nix-store"); + op = opVersion; else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") op = opRealise; else if (*arg == "--add" || *arg == "-A") @@ -1078,10 +1066,6 @@ int main(int argc, char * * argv) op = opRepairPath; else if (*arg == "--optimise" || *arg == "--optimize") op = opOptimise; - else if (*arg == "--query-failed-paths") - op = opQueryFailedPaths; - else if (*arg == "--clear-failed-paths") - op = opClearFailedPaths; else if (*arg == "--serve") op = opServe; else if (*arg == "--generate-binary-cache-key") @@ -1109,7 +1093,7 @@ int main(int argc, char * * argv) if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ - store = openStore(op != opGC); + store = openStore(); op(opFlags, opArgs); }); diff --git a/src/nix-store/serve-protocol.hh b/src/nix-store/serve-protocol.hh index 741b622beb17..c4e2a370300b 100644 --- a/src/nix-store/serve-protocol.hh +++ b/src/nix-store/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x200 +#define SERVE_PROTOCOL_VERSION 0x202 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -17,6 +17,7 @@ typedef enum { cmdExportPaths = 5, cmdBuildPaths = 6, cmdQueryClosure = 7, + cmdBuildDerivation = 8, } ServeCommand; } diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc index 1b3ad3d28ad4..0f7be7f7a02d 100644 --- a/src/nix-store/xmlgraph.cc +++ b/src/nix-store/xmlgraph.cc @@ -33,34 +33,29 @@ static string makeNode(const string & id) } -void printXmlGraph(const PathSet & roots) +void printXmlGraph(ref<Store> store, const PathSet & roots) { PathSet workList(roots); PathSet doneSet; cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<nix>\n"; + << "<nix>\n"; while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); + Path path = *(workList.begin()); + workList.erase(path); - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); + if (doneSet.find(path) != doneSet.end()) continue; + doneSet.insert(path); - cout << makeNode(path); + cout << makeNode(path); - PathSet references; - store->queryReferences(path, references); - - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) - { - if (*i != path) { - workList.insert(*i); - cout << makeEdge(*i, path); - } - } + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); + } + } } diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh index c2216c5a4627..a6e7d4e2805a 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/xmlgraph.hh @@ -4,6 +4,8 @@ namespace nix { -void printXmlGraph(const PathSet & roots); +class Store; + +void printXmlGraph(ref<Store> store, const PathSet & roots); } diff --git a/src/nix/build.cc b/src/nix/build.cc new file mode 100644 index 000000000000..812464d7582b --- /dev/null +++ b/src/nix/build.cc @@ -0,0 +1,46 @@ +#include "command.hh" +#include "common-args.hh" +#include "installables.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdBuild : StoreCommand, MixDryRun, MixInstallables +{ + CmdBuild() + { + } + + std::string name() override + { + return "build"; + } + + std::string description() override + { + return "build a derivation or fetch a store path"; + } + + void run(ref<Store> store) override + { + auto elems = evalInstallables(store); + + PathSet pathsToBuild; + + for (auto & elem : elems) { + if (elem.isDrv) + pathsToBuild.insert(elem.drvPath); + else + pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); + } + + printMissing(store, pathsToBuild); + + if (dryRun) return; + + store->buildPaths(pathsToBuild); + } +}; + +static RegisterCommand r1(make_ref<CmdBuild>()); diff --git a/src/nix/cat.cc b/src/nix/cat.cc new file mode 100644 index 000000000000..2405a8cb44ef --- /dev/null +++ b/src/nix/cat.cc @@ -0,0 +1,74 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixCat : virtual Args +{ + std::string path; + + void cat(ref<FSAccessor> accessor) + { + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path ‘%1%’ does not exist") % path); + if (st.type != FSAccessor::Type::tRegular) + throw Error(format("path ‘%1%’ is not a regular file") % path); + + std::cout << accessor->readFile(path); + } +}; + +struct CmdCatStore : StoreCommand, MixCat +{ + CmdCatStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "cat-store"; + } + + std::string description() override + { + return "print the contents of a store file on stdout"; + } + + void run(ref<Store> store) override + { + cat(store->getFSAccessor()); + } +}; + +struct CmdCatNar : StoreCommand, MixCat +{ + Path narPath; + + CmdCatNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "cat-nar"; + } + + std::string description() override + { + return "print the contents of a file inside a NAR file"; + } + + void run(ref<Store> store) override + { + cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref<CmdCatStore>()); +static RegisterCommand r2(make_ref<CmdCatNar>()); diff --git a/src/nix/command.cc b/src/nix/command.cc new file mode 100644 index 000000000000..253c9686ba75 --- /dev/null +++ b/src/nix/command.cc @@ -0,0 +1,118 @@ +#include "command.hh" +#include "store-api.hh" + +namespace nix { + +Commands * RegisterCommand::commands = 0; + +void Command::printHelp(const string & programName, std::ostream & out) +{ + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto & ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } +} + +MultiCommand::MultiCommand(const Commands & _commands) + : commands(_commands) +{ + expectedArgs.push_back(ExpectedArg{"command", 1, [=](Strings ss) { + assert(!command); + auto i = commands.find(ss.front()); + if (i == commands.end()) + throw UsageError(format("‘%1%’ is not a recognised command") % ss.front()); + command = i->second; + }}); +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto & command : commands) + table.push_back(std::make_pair(command.second->name(), command.second->description())); + printTable(out, table); + + out << "\n"; + out << "For full documentation, run ‘man " << programName << "’ or ‘man " << programName << "-<COMMAND>’.\n"; +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + +StoreCommand::StoreCommand() +{ + storeUri = getEnv("NIX_REMOTE"); + + mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); +} + +void StoreCommand::run() +{ + run(openStoreAt(storeUri)); +} + +StorePathsCommand::StorePathsCommand() +{ + expectArgs("paths", &storePaths); + mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); + mkFlag(0, "all", "apply operation to the entire store", &all); +} + +void StorePathsCommand::run(ref<Store> store) +{ + if (all) { + if (storePaths.size()) + throw UsageError("‘--all’ does not expect arguments"); + for (auto & p : store->queryAllValidPaths()) + storePaths.push_back(p); + } + + else { + for (auto & storePath : storePaths) + storePath = followLinksToStorePath(storePath); + + if (recursive) { + PathSet closure; + for (auto & storePath : storePaths) + store->computeFSClosure(storePath, closure, false, false); + storePaths = Paths(closure.begin(), closure.end()); + } + } + + run(store, storePaths); +} + +} diff --git a/src/nix/command.hh b/src/nix/command.hh new file mode 100644 index 000000000000..34affc43d96e --- /dev/null +++ b/src/nix/command.hh @@ -0,0 +1,89 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + virtual std::string name() = 0; + virtual void prepare() { }; + virtual void run() = 0; + + struct Example + { + std::string description; + std::string command; + }; + + typedef std::list<Example> Examples; + + virtual Examples examples() { return Examples(); } + + void printHelp(const string & programName, std::ostream & out) override; +}; + +class Store; + +/* A command that require a Nix store. */ +struct StoreCommand : virtual Command +{ + std::string storeUri; + StoreCommand(); + void run() override; + virtual void run(ref<Store>) = 0; +}; + +/* A command that operates on zero or more store paths. */ +struct StorePathsCommand : public StoreCommand +{ +private: + + Paths storePaths; + bool recursive = false; + bool all = false; + +public: + + StorePathsCommand(); + + virtual void run(ref<Store> store, Paths storePaths) = 0; + + void run(ref<Store> store) override; +}; + +typedef std::map<std::string, ref<Command>> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘<command> <subcommand>’. */ +class MultiCommand : virtual Args +{ +public: + Commands commands; + + std::shared_ptr<Command> command; + + MultiCommand(const Commands & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + +/* A helper class for registering commands globally. */ +struct RegisterCommand +{ + static Commands * commands; + + RegisterCommand(ref<Command> command) + { + if (!commands) commands = new Commands; + commands->emplace(command->name(), command); + } +}; + +} diff --git a/src/nix/copy.cc b/src/nix/copy.cc new file mode 100644 index 000000000000..de306cbf91d9 --- /dev/null +++ b/src/nix/copy.cc @@ -0,0 +1,79 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdCopy : StorePathsCommand +{ + std::string srcUri, dstUri; + + CmdCopy() + { + mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri); + mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri); + } + + std::string name() override + { + return "copy"; + } + + std::string description() override + { + return "copy paths between Nix stores"; + } + + Examples examples() override + { + return { + Example{ + "To copy Firefox to the local store to a binary cache in file:///tmp/cache:", + "nix copy --to file:///tmp/cache -r $(type -p firefox)" + }, + }; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (srcUri.empty() && dstUri.empty()) + throw UsageError("you must pass ‘--from’ and/or ‘--to’"); + + ref<Store> srcStore = srcUri.empty() ? store : openStoreAt(srcUri); + ref<Store> dstStore = dstUri.empty() ? store : openStoreAt(dstUri); + + std::string copiedLabel = "copied"; + + logger->setExpected(copiedLabel, storePaths.size()); + + ThreadPool pool; + + processGraph<Path>(pool, + PathSet(storePaths.begin(), storePaths.end()), + + [&](const Path & storePath) { + return srcStore->queryPathInfo(storePath)->references; + }, + + [&](const Path & storePath) { + checkInterrupt(); + + if (!dstStore->isValidPath(storePath)) { + Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); + + copyStorePath(srcStore, dstStore, storePath); + + logger->incProgress(copiedLabel); + } else + logger->incExpected(copiedLabel, -1); + }); + + pool.process(); + } +}; + +static RegisterCommand r1(make_ref<CmdCopy>()); diff --git a/src/nix/hash.cc b/src/nix/hash.cc new file mode 100644 index 000000000000..5dd891e8add3 --- /dev/null +++ b/src/nix/hash.cc @@ -0,0 +1,140 @@ +#include "command.hh" +#include "hash.hh" +#include "legacy.hh" +#include "shared.hh" + +using namespace nix; + +struct CmdHash : Command +{ + enum Mode { mFile, mPath }; + Mode mode; + bool base32 = false; + bool truncate = false; + HashType ht = htSHA512; + Strings paths; + + CmdHash(Mode mode) : mode(mode) + { + mkFlag(0, "base32", "print hash in base-32", &base32); + mkFlag(0, "base16", "print hash in base-16", &base32, false); + mkHashTypeFlag("type", &ht); + expectArgs("paths", &paths); + } + + std::string name() override + { + return mode == mFile ? "hash-file" : "hash-path"; + } + + std::string description() override + { + return mode == mFile + ? "print cryptographic hash of a regular file" + : "print cryptographic hash of the NAR serialisation of a path"; + } + + void run() override + { + for (auto path : paths) { + Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); + std::cout << format("%1%\n") % + (base32 ? printHash32(h) : printHash(h)); + } + } +}; + +static RegisterCommand r1(make_ref<CmdHash>(CmdHash::mFile)); +static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); + +struct CmdToBase : Command +{ + bool toBase32; + HashType ht = htSHA512; + Strings args; + + CmdToBase(bool toBase32) : toBase32(toBase32) + { + mkHashTypeFlag("type", &ht); + expectArgs("strings", &args); + } + + std::string name() override + { + return toBase32 ? "to-base32" : "to-base16"; + } + + std::string description() override + { + return toBase32 + ? "convert a hash to base-32 representation" + : "convert a hash to base-16 representation"; + } + + void run() override + { + for (auto s : args) { + Hash h = parseHash16or32(ht, s); + std::cout << format("%1%\n") % + (toBase32 ? printHash32(h) : printHash(h)); + } + } +}; + +static RegisterCommand r3(make_ref<CmdToBase>(false)); +static RegisterCommand r4(make_ref<CmdToBase>(true)); + +/* Legacy nix-hash command. */ +static int compatNixHash(int argc, char * * argv) +{ + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + Strings ss; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-hash"); + else if (*arg == "--version") + printVersion("nix-hash"); + else if (*arg == "--flat") flat = true; + else if (*arg == "--base32") base32 = true; + else if (*arg == "--truncate") truncate = true; + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type ‘%1%’") % s); + } + else if (*arg == "--to-base16") op = opTo16; + else if (*arg == "--to-base32") op = opTo32; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + ss.push_back(*arg); + return true; + }); + + if (op == opHash) { + CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + cmd.ht = ht; + cmd.base32 = base32; + cmd.truncate = truncate; + cmd.paths = ss; + cmd.run(); + } + + else { + CmdToBase cmd(op == opTo32); + cmd.args = ss; + cmd.ht = ht; + cmd.run(); + } + + return 0; +} + +static RegisterLegacyCommand s1("nix-hash", compatNixHash); diff --git a/src/nix/installables.cc b/src/nix/installables.cc new file mode 100644 index 000000000000..fb5a515825aa --- /dev/null +++ b/src/nix/installables.cc @@ -0,0 +1,75 @@ +#include "attr-path.hh" +#include "common-opts.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "installables.hh" +#include "store-api.hh" + +namespace nix { + +UserEnvElems MixInstallables::evalInstallables(ref<Store> store) +{ + UserEnvElems res; + + for (auto & installable : installables) { + + if (std::string(installable, 0, 1) == "/") { + + if (isStorePath(installable)) { + + if (isDerivation(installable)) { + UserEnvElem elem; + // FIXME: handle empty case, drop version + elem.attrPath = {storePathToName(installable)}; + elem.isDrv = true; + elem.drvPath = installable; + res.push_back(elem); + } + + else { + UserEnvElem elem; + // FIXME: handle empty case, drop version + elem.attrPath = {storePathToName(installable)}; + elem.isDrv = false; + elem.outPaths = {installable}; + res.push_back(elem); + } + } + + else + throw UsageError(format("don't know what to do with ‘%1%’") % installable); + } + + else { + + EvalState state({}, store); + + Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); + + Value vRoot; + state.eval(e, vRoot); + + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + + Value & v(*findAlongAttrPath(state, installable, autoArgs, vRoot)); + state.forceValue(v); + + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + + for (auto & i : drvs) { + UserEnvElem elem; + elem.isDrv = true; + elem.drvPath = i.queryDrvPath(); + res.push_back(elem); + } + } + } + + return res; +} + +} diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000000..5eb897d46148 --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct UserEnvElem +{ + Strings attrPath; + + // FIXME: should use boost::variant or so. + bool isDrv; + + // Derivation case: + Path drvPath; + StringSet outputNames; + + // Non-derivation case: + PathSet outPaths; +}; + +typedef std::vector<UserEnvElem> UserEnvElems; + +struct MixInstallables : virtual Args +{ + Strings installables; + Path file = "<nixpkgs>"; + + MixInstallables() + { + mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); + expectArgs("installables", &installables); + } + + UserEnvElems evalInstallables(ref<Store> store); +}; + +} diff --git a/src/nix/legacy.cc b/src/nix/legacy.cc new file mode 100644 index 000000000000..6df09ee37a5e --- /dev/null +++ b/src/nix/legacy.cc @@ -0,0 +1,7 @@ +#include "legacy.hh" + +namespace nix { + +RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; + +} diff --git a/src/nix/legacy.hh b/src/nix/legacy.hh new file mode 100644 index 000000000000..f503b0da3e1a --- /dev/null +++ b/src/nix/legacy.hh @@ -0,0 +1,23 @@ +#pragma once + +#include <functional> +#include <map> +#include <string> + +namespace nix { + +typedef std::function<void(int, char * *)> MainFunction; + +struct RegisterLegacyCommand +{ + typedef std::map<std::string, MainFunction> Commands; + static Commands * commands; + + RegisterLegacyCommand(const std::string & name, MainFunction fun) + { + if (!commands) commands = new Commands; + (*commands)[name] = fun; + } +}; + +} diff --git a/src/nix/local.mk b/src/nix/local.mk new file mode 100644 index 000000000000..f6e7073b6e7d --- /dev/null +++ b/src/nix/local.mk @@ -0,0 +1,9 @@ +programs += nix + +nix_DIR := $(d) + +nix_SOURCES := $(wildcard $(d)/*.cc) + +nix_LIBS = libexpr libmain libstore libutil libformat + +$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/ls.cc b/src/nix/ls.cc new file mode 100644 index 000000000000..3476dfb05287 --- /dev/null +++ b/src/nix/ls.cc @@ -0,0 +1,123 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixLs : virtual Args +{ + std::string path; + + bool recursive = false; + bool verbose = false; + bool showDirectory = false; + + MixLs() + { + mkFlag('R', "recursive", "list subdirectories recursively", &recursive); + mkFlag('l', "long", "show more file information", &verbose); + mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + } + + void list(ref<FSAccessor> accessor) + { + std::function<void(const FSAccessor::Stat &, const Path &, const std::string &, bool)> doPath; + + auto showFile = [&](const Path & curPath, const std::string & relPath) { + if (verbose) { + auto st = accessor->stat(curPath); + std::string tp = + st.type == FSAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + "dr-xr-xr-x"; + std::cout << + (format("%s %20d %s") % tp % st.fileSize % relPath); + if (st.type == FSAccessor::Type::tSymlink) + std::cout << " -> " << accessor->readLink(curPath) + ; + std::cout << "\n"; + if (recursive && st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } else { + std::cout << relPath << "\n"; + if (recursive) { + auto st = accessor->stat(curPath); + if (st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } + } + }; + + doPath = [&](const FSAccessor::Stat & st , const Path & curPath, + const std::string & relPath, bool showDirectory) + { + if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + auto names = accessor->readDirectory(curPath); + for (auto & name : names) + showFile(curPath + "/" + name, relPath + "/" + name); + } else + showFile(curPath, relPath); + }; + + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path ‘%1%’ does not exist") % path); + doPath(st, path, + st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + showDirectory); + } +}; + +struct CmdLsStore : StoreCommand, MixLs +{ + CmdLsStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "ls-store"; + } + + std::string description() override + { + return "show information about a store path"; + } + + void run(ref<Store> store) override + { + list(store->getFSAccessor()); + } +}; + +struct CmdLsNar : Command, MixLs +{ + Path narPath; + + CmdLsNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "ls-nar"; + } + + std::string description() override + { + return "show information about the contents of a NAR file"; + } + + void run() override + { + list(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref<CmdLsStore>()); +static RegisterCommand r2(make_ref<CmdLsNar>()); diff --git a/src/nix/main.cc b/src/nix/main.cc new file mode 100644 index 000000000000..440ced97dfcc --- /dev/null +++ b/src/nix/main.cc @@ -0,0 +1,61 @@ +#include <algorithm> + +#include "command.hh" +#include "common-args.hh" +#include "eval.hh" +#include "globals.hh" +#include "legacy.hh" +#include "shared.hh" +#include "store-api.hh" +#include "progress-bar.hh" + +namespace nix { + +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +{ + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + { + mkFlag('h', "help", "show usage information", [=]() { + printHelp(programName, std::cout); + std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + throw Exit(); + }); + + mkFlag(0, "version", "show version information", std::bind(printVersion, programName)); + } +}; + +void mainWrapped(int argc, char * * argv) +{ + settings.verboseBuild = false; + + initNix(); + initGC(); + + string programName = baseNameOf(argv[0]); + + { + auto legacy = (*RegisterLegacyCommand::commands)[programName]; + if (legacy) return legacy(argc, argv); + } + + NixArgs args; + + args.parseCmdline(argvToStrings(argc, argv)); + + assert(args.command); + + StartProgressBar bar; + + args.command->prepare(); + args.command->run(); +} + +} + +int main(int argc, char * * argv) +{ + return nix::handleExceptions(argv[0], [&]() { + nix::mainWrapped(argc, argv); + }); +} diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc new file mode 100644 index 000000000000..c61fe7ff1e00 --- /dev/null +++ b/src/nix/path-info.cc @@ -0,0 +1,85 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +#include <iomanip> +#include <algorithm> + +using namespace nix; + +struct CmdPathInfo : StorePathsCommand +{ + bool showSize = false; + bool showClosureSize = false; + bool showSigs = false; + + CmdPathInfo() + { + mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); + mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); + mkFlag(0, "sigs", "show signatures", &showSigs); + } + + std::string name() override + { + return "path-info"; + } + + std::string description() override + { + return "query information about store paths"; + } + + Examples examples() override + { + return { + Example{ + "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", + "nix path-info -rS /run/current-system | sort -nk2" + }, + Example{ + "To check the existence of a path in a binary cache:", + "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" + }, + }; + } + + void run(ref<Store> store, Paths storePaths) override + { + size_t pathLen = 0; + for (auto & storePath : storePaths) + pathLen = std::max(pathLen, storePath.size()); + + for (auto storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + storePath = info->path; // FIXME: screws up padding + + std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + + if (showSize) { + std::cout << '\t' << std::setw(11) << info->narSize; + } + + if (showClosureSize) { + size_t totalSize = 0; + PathSet closure; + store->computeFSClosure(storePath, closure, false, false); + for (auto & p : closure) + totalSize += store->queryPathInfo(p)->narSize; + std::cout << '\t' << std::setw(11) << totalSize; + } + + if (showSigs) { + std::cout << '\t'; + Strings ss; + if (info->ultimate) ss.push_back("ultimate"); + for (auto & sig : info->sigs) ss.push_back(sig); + std::cout << concatStringsSep(" ", ss); + } + + std::cout << std::endl; + } + } +}; + +static RegisterCommand r1(make_ref<CmdPathInfo>()); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc new file mode 100644 index 000000000000..659d6572ad93 --- /dev/null +++ b/src/nix/progress-bar.cc @@ -0,0 +1,157 @@ +#include "progress-bar.hh" +#include "util.hh" +#include "sync.hh" + +#include <map> + +namespace nix { + +class ProgressBar : public Logger +{ +private: + + struct ActInfo + { + Activity * activity; + Verbosity lvl; + std::string s; + }; + + struct Progress + { + uint64_t expected = 0, progress = 0; + }; + + struct State + { + std::list<ActInfo> activities; + std::map<Activity *, std::list<ActInfo>::iterator> its; + std::map<std::string, Progress> progress; + }; + + Sync<State> state_; + +public: + + ~ProgressBar() + { + auto state(state_.lock()); + assert(state->activities.empty()); + writeToStderr("\r\e[K"); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + auto state(state_.lock()); + log(*state, lvl, fs.s); + } + + void log(State & state, Verbosity lvl, const std::string & s) + { + writeToStderr("\r\e[K" + s + "\n"); + update(state); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + auto state(state_.lock()); + state->activities.emplace_back(ActInfo{&activity, lvl, fs.s}); + state->its.emplace(&activity, std::prev(state->activities.end())); + update(*state); + } + + void stopActivity(Activity & activity) override + { + auto state(state_.lock()); + auto i = state->its.find(&activity); + if (i == state->its.end()) return; + state->activities.erase(i->second); + state->its.erase(i); + update(*state); + } + + void setExpected(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].expected = value; + } + + void setProgress(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].progress = value; + } + + void incExpected(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].expected += value; + } + + void incProgress(const std::string & label, uint64_t value) + { + auto state(state_.lock()); + state->progress[label].progress += value; + } + + void update() + { + auto state(state_.lock()); + } + + void update(State & state) + { + std::string line = "\r"; + + std::string status = getStatus(state); + if (!status.empty()) { + line += '['; + line += status; + line += "]"; + } + + if (!state.activities.empty()) { + if (!status.empty()) line += " "; + line += state.activities.rbegin()->s; + } + + line += "\e[K"; + writeToStderr(line); + } + + std::string getStatus(State & state) + { + std::string res; + for (auto & p : state.progress) + if (p.second.expected || p.second.progress) { + if (!res.empty()) res += ", "; + res += std::to_string(p.second.progress); + if (p.second.expected) { + res += "/"; + res += std::to_string(p.second.expected); + } + res += " "; res += p.first; + } + return res; + } +}; + +StartProgressBar::StartProgressBar() +{ + if (isatty(STDERR_FILENO)) { + prev = logger; + logger = new ProgressBar(); + } +} + +StartProgressBar::~StartProgressBar() +{ + if (prev) { + auto bar = logger; + logger = prev; + delete bar; + } +} + +} diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh new file mode 100644 index 000000000000..d2e44f7c4fd9 --- /dev/null +++ b/src/nix/progress-bar.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "logging.hh" + +namespace nix { + +class StartProgressBar +{ + Logger * prev = 0; +public: + StartProgressBar(); + ~StartProgressBar(); +}; + +} diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc new file mode 100644 index 000000000000..9932aa4a9eb0 --- /dev/null +++ b/src/nix/sigs.cc @@ -0,0 +1,139 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdCopySigs : StorePathsCommand +{ + Strings substituterUris; + + CmdCopySigs() + { + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + } + + std::string name() override + { + return "copy-sigs"; + } + + std::string description() override + { + return "copy path signatures from substituters (like binary caches)"; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (substituterUris.empty()) + throw UsageError("you must specify at least one substituter using ‘-s’"); + + // FIXME: factor out commonality with MixVerify. + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + ThreadPool pool; + + std::string doneLabel = "done"; + std::atomic<size_t> added{0}; + + logger->setExpected(doneLabel, storePaths.size()); + + auto doPath = [&](const Path & storePath) { + Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); + + checkInterrupt(); + + auto info = store->queryPathInfo(storePath); + + StringSet newSigs; + + for (auto & store2 : substituters) { + try { + auto info2 = store2->queryPathInfo(storePath); + + /* Don't import signatures that don't match this + binary. */ + if (info->narHash != info2->narHash || + info->narSize != info2->narSize || + info->references != info2->references) + continue; + + for (auto & sig : info2->sigs) + if (!info->sigs.count(sig)) + newSigs.insert(sig); + } catch (InvalidPath &) { + } + } + + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } + + logger->incProgress(doneLabel); + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + printMsg(lvlInfo, format("imported %d signatures") % added); + } +}; + +static RegisterCommand r1(make_ref<CmdCopySigs>()); + +struct CmdSignPaths : StorePathsCommand +{ + Path secretKeyFile; + + CmdSignPaths() + { + mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile); + } + + std::string name() override + { + return "sign-paths"; + } + + std::string description() override + { + return "sign the specified paths"; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (secretKeyFile.empty()) + throw UsageError("you must specify a secret key file using ‘-k’"); + + SecretKey secretKey(readFile(secretKeyFile)); + + size_t added{0}; + + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + + auto info2(*info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); + + if (!info->sigs.count(*info2.sigs.begin())) { + store->addSignatures(storePath, info2.sigs); + added++; + } + } + + printMsg(lvlInfo, format("added %d signatures") % added); + } +}; + +static RegisterCommand r3(make_ref<CmdSignPaths>()); diff --git a/src/nix/verify.cc b/src/nix/verify.cc new file mode 100644 index 000000000000..fd904f465687 --- /dev/null +++ b/src/nix/verify.cc @@ -0,0 +1,168 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdVerify : StorePathsCommand +{ + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded; + + CmdVerify() + { + mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + } + + std::string name() override + { + return "verify"; + } + + std::string description() override + { + return "verify the integrity of store paths"; + } + + Examples examples() override + { + return { + Example{ + "To verify the entire Nix store:", + "nix verify --all" + }, + Example{ + "To check whether each path in the closure of Firefox has at least 2 signatures:", + "nix verify -r -n2 --no-contents $(type -p firefox)" + }, + }; + } + + void run(ref<Store> store, Paths storePaths) override + { + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + auto publicKeys = getDefaultPublicKeys(); + + std::atomic<size_t> done{0}; + std::atomic<size_t> untrusted{0}; + std::atomic<size_t> corrupted{0}; + std::atomic<size_t> failed{0}; + + std::string doneLabel("paths checked"); + std::string untrustedLabel("untrusted"); + std::string corruptedLabel("corrupted"); + std::string failedLabel("failed"); + logger->setExpected(doneLabel, storePaths.size()); + + ThreadPool pool; + + auto doPath = [&](const Path & storePath) { + try { + checkInterrupt(); + + Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + + HashSink sink(info->narHash.type); + store->narFromPath(info->path, sink); + + auto hash = sink.finish(); + + if (hash.first != info->narHash) { + logger->incProgress(corruptedLabel); + corrupted = 1; + printMsg(lvlError, + format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") + % info->path % printHash(info->narHash) % printHash(hash.first)); + } + + } + + if (!noTrust) { + + bool good = false; + + if (info->ultimate && !sigsNeeded) + good = true; + + else { + + StringSet sigsSeen; + size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t validSigs = 0; + + auto doSigs = [&](StringSet sigs) { + for (auto sig : sigs) { + if (sigsSeen.count(sig)) continue; + sigsSeen.insert(sig); + if (info->checkSignature(publicKeys, sig)) + validSigs++; + } + }; + + doSigs(info->sigs); + + for (auto & store2 : substituters) { + if (validSigs >= actualSigsNeeded) break; + try { + doSigs(store2->queryPathInfo(info->path)->sigs); + } catch (InvalidPath &) { + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + } + } + + if (validSigs >= actualSigsNeeded) + good = true; + } + + if (!good) { + logger->incProgress(untrustedLabel); + untrusted++; + printMsg(lvlError, format("path ‘%s’ is untrusted") % info->path); + } + + } + + logger->incProgress(doneLabel); + done++; + + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logger->incProgress(failedLabel); + failed++; + } + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + printMsg(lvlInfo, format("%d paths checked, %d untrusted, %d corrupted, %d failed") + % done % untrusted % corrupted % failed); + + throw Exit( + (corrupted ? 1 : 0) | + (untrusted ? 2 : 0) | + (failed ? 4 : 0)); + } +}; + +static RegisterCommand r1(make_ref<CmdVerify>()); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index c72d2defa5d0..48bc62d311de 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,7 +1,6 @@ source common.sh clearStore -clearManifests clearCache # Create the binary cache. @@ -13,7 +12,7 @@ nix-push --dest $cacheDir $outPath # By default, a binary cache doesn't support "nix-env -qas", but does # support installation. clearStore -rm -f $NIX_STATE_DIR/binary-cache* +clearCacheCache export _NIX_CACHE_FILE_URLS=1 @@ -26,7 +25,7 @@ nix-store --option binary-caches "file://$cacheDir" -r $outPath # But with the right configuration, "nix-env -qas" should also work. clearStore -rm -f $NIX_STATE_DIR/binary-cache* +clearCacheCache echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info nix-env --option binary-caches "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" @@ -49,15 +48,14 @@ mkdir -p $TEST_ROOT/empty nix-store --dump $TEST_ROOT/empty | xz > $nar nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "hash mismatch in downloaded path" $TEST_ROOT/log +grep -q "hash mismatch" $TEST_ROOT/log mv $nar.good $nar # Test whether this unsigned cache is rejected if the user requires signed caches. clearStore - -rm -f $NIX_STATE_DIR/binary-cache* +clearCacheCache if nix-store --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' -r $outPath; then echo "unsigned binary cache incorrectly accepted" @@ -86,7 +84,7 @@ clearStore rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "Downloading" $TEST_ROOT/log +grep -q "fetching path" $TEST_ROOT/log if [ -n "$HAVE_SODIUM" ]; then @@ -108,16 +106,14 @@ nix-push --dest $cacheDir --key-file $TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. clearStore - -rm -f $NIX_STATE_DIR/binary-cache* +clearCacheCache (! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' ) # And it should fail if we provide an incorrect key. clearStore - -rm -f $NIX_STATE_DIR/binary-cache* +clearCacheCache (! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$badKey") @@ -129,13 +125,21 @@ nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed- # It should fail if we corrupt the .narinfo. clearStore -for i in $cacheDir/*.narinfo; do +cacheDir2=$TEST_ROOT/binary-cache-2 +rm -rf $cacheDir2 +cp -r $cacheDir $cacheDir2 + +for i in $cacheDir2/*.narinfo; do grep -v References $i > $i.tmp mv $i.tmp $i done -rm -f $NIX_STATE_DIR/binary-cache* +clearCacheCache + +(! nix-store -r $outPath --option binary-caches "file://$cacheDir2" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey") + +# If we provide a bad and a good binary cache, it should succeed. -(! nix-store -r $outPath --option binary-caches "file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey") +nix-store -r $outPath --option binary-caches "file://$cacheDir2 file://$cacheDir" --option signed-binary-caches '*' --option binary-cache-public-keys "$publicKey" fi # HAVE_LIBSODIUM diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix deleted file mode 100644 index 8ed474d1f27f..000000000000 --- a/tests/binary-patching.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ version }: - -with import ./config.nix; - -mkDerivation { - name = "foo-${toString version}"; - builder = builtins.toFile "builder.sh" - '' - mkdir $out - (for ((n = 1; n < 100000; n++)); do echo $n; done) > $out/foo - ${if version != 1 then '' - (for ((n = 100000; n < 110000; n++)); do echo $n; done) >> $out/foo - '' else ""} - ${if version == 3 then '' - echo foobar >> $out/foo - '' else ""} - ''; -} diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh deleted file mode 100644 index 188be109a0b5..000000000000 --- a/tests/binary-patching.sh +++ /dev/null @@ -1,61 +0,0 @@ -source common.sh - -clearManifests - -mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches - -RESULT=$TEST_ROOT/result - -# Build version 1 and 2 of the "foo" package. -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \ - $(nix-build -o $RESULT binary-patching.nix --arg version 1) -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1 - -out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2) -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2 -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2 - -out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3) -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3 -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3 - -rm $RESULT - -# Generate binary patches. -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2 - -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3 - -grep -q "patch {" $TEST_ROOT/manifest3 - -# Get rid of versions 2 and 3. -nix-store --delete $out2 $out3 - -# Pull the manifest containing the patches. -clearManifests -nix-pull file://$TEST_ROOT/manifest3 - -# Make sure that the download size prediction uses the patches rather -# than the full download. -nix-build -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB" - -# Now rebuild it. This should use the two patches generated above. -rm -f $TEST_ROOT/var/log/nix/downloads -nix-build -o $RESULT binary-patching.nix --arg version 3 -rm $RESULT -[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ] - -# Add a patch from version 1 directly to version 3. -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3 - -# Rebuild version 3. This should use the direct patch rather than the -# sequence of two patches. -nix-store --delete $out2 $out3 -clearManifests -rm $TEST_ROOT/var/log/nix/downloads -nix-pull file://$TEST_ROOT/manifest3 -nix-build -o $RESULT binary-patching.nix --arg version 3 -[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ] diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh index 8b2454915a52..77689215def1 100644 --- a/tests/check-reqs.sh +++ b/tests/check-reqs.sh @@ -6,7 +6,8 @@ nix-build -o $RESULT check-reqs.nix -A test1 (! nix-build -o $RESULT check-reqs.nix -A test2) (! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep1' +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep2' (! nix-build -o $RESULT check-reqs.nix -A test5) (! nix-build -o $RESULT check-reqs.nix -A test6) diff --git a/tests/common.sh.in b/tests/common.sh.in index eb9798a27b45..756c20c9d9d6 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -2,7 +2,7 @@ set -e datadir="@datadir@" -export TEST_ROOT=$(pwd)/test-tmp +export TEST_ROOT=${TMPDIR:-/tmp}/nix-test export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. @@ -17,6 +17,9 @@ export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests export _NIX_TEST_SHARED=$TEST_ROOT/shared export NIX_REMOTE=$NIX_REMOTE_ +export TEST_HOME=$TEST_ROOT/test-home +export HOME=$TEST_HOME +mkdir -p $TEST_HOME export PATH=@bindir@:$PATH @@ -54,14 +57,14 @@ clearStore() { rm -f "$NIX_STATE_DIR"/gcroots/ref } -clearManifests() { - rm -f $NIX_STATE_DIR/manifests/* -} - clearCache() { rm -rf "$cacheDir" } +clearCacheCache() { + rm -f $TEST_HOME/.cache/nix/binary-cache* +} + startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. diff --git a/tests/dump-db.sh b/tests/dump-db.sh index 234b7ac02680..57c8c401600d 100644 --- a/tests/dump-db.sh +++ b/tests/dump-db.sh @@ -9,6 +9,7 @@ deps="$(nix-store -qR $TEST_ROOT/result)" nix-store --dump-db > $TEST_ROOT/dump rm -rf $NIX_DB_DIR +mkdir $NIX_DB_DIR nix-store --load-db < $TEST_ROOT/dump diff --git a/tests/fallback.sh b/tests/fallback.sh deleted file mode 100644 index f3a6b50515bf..000000000000 --- a/tests/fallback.sh +++ /dev/null @@ -1,20 +0,0 @@ -source common.sh - -clearStore - -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -outPath=$(nix-store -q --fallback "$drvPath") -echo "output path is $outPath" - -# Build with a substitute that fails. This should fail. -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh -if nix-store -r "$drvPath"; then echo unexpected fallback; exit 1; fi - -# Build with a substitute that fails. This should fall back to a source build. -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh -nix-store -r --fallback "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hello World!"; then exit 1; fi diff --git a/tests/fetchurl.nix b/tests/fetchurl.nix deleted file mode 100644 index 2abcc039a832..000000000000 --- a/tests/fetchurl.nix +++ /dev/null @@ -1,6 +0,0 @@ -{ filename, sha256 }: - -import <nix/fetchurl.nix> { - url = "file://${filename}"; - inherit sha256; -} diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index 6acc87eafca8..b6fa3a27edd8 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -2,8 +2,40 @@ source common.sh clearStore -hash=$(nix-hash --flat --type sha256 ./fetchurl.nix) +# Test fetching a flat file. +hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build ./fetchurl.nix --argstr filename $(pwd)/fetchurl.nix --argstr sha256 $hash --no-out-link) +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) -cmp $outPath fetchurl.nix +cmp $outPath fetchurl.sh + +# Test unpacking a NAR. +rm -rf $TEST_ROOT/archive +mkdir -p $TEST_ROOT/archive +cp ./fetchurl.sh $TEST_ROOT/archive +chmod +x $TEST_ROOT/archive/fetchurl.sh +ln -s foo $TEST_ROOT/archive/symlink +nar=$TEST_ROOT/archive.nar +nix-store --dump $TEST_ROOT/archive > $nar + +hash=$(nix-hash --flat --type sha256 $nar) + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \ + --arg unpack true --argstr name xyzzy --no-out-link) + +echo $outPath | grep -q 'xyzzy' + +test -x $outPath/fetchurl.sh +test -L $outPath/symlink + +nix-store --delete $outPath + +# Test unpacking a compressed NAR. +narxz=$TEST_ROOT/archive.nar.xz +rm -f $narxz +xz --keep $nar +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$narxz --argstr sha256 $hash \ + --arg unpack true --argstr name xyzzy --no-out-link) + +test -x $outPath/fetchurl.sh +test -L $outPath/symlink diff --git a/tests/filter-source.nix b/tests/filter-source.nix index a620f0fda5c4..9071636394af 100644 --- a/tests/filter-source.nix +++ b/tests/filter-source.nix @@ -8,5 +8,5 @@ mkDerivation { type != "symlink" && baseNameOf path != "foo" && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); - in builtins.filterSource filter ./test-tmp/filterin; + in builtins.filterSource filter ((builtins.getEnv "TEST_ROOT") + "/filterin"); } diff --git a/tests/fixed.sh b/tests/fixed.sh index ed0d06dd29cb..cac3f0be91b0 100644 --- a/tests/fixed.sh +++ b/tests/fixed.sh @@ -40,13 +40,10 @@ echo "Hello World!" > $TEST_ROOT/fixed/foo ln -s foo $TEST_ROOT/fixed/bar out2=$(nix-store --add $TEST_ROOT/fixed) -echo $out2 -test "$out" = "$out2" || exit 1 +[ "$out" = "$out2" ] out3=$(nix-store --add-fixed --recursive sha256 $TEST_ROOT/fixed) -echo $out3 -test "$out" = "$out3" || exit 1 +[ "$out" = "$out3" ] out4=$(nix-store --print-fixed-path --recursive sha256 "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik" fixed) -echo $out4 -test "$out" = "$out4" || exit 1 +[ "$out" = "$out4" ] diff --git a/tests/hash.sh b/tests/hash.sh index d659bbe34e8f..a95c68683f84 100644 --- a/tests/hash.sh +++ b/tests/hash.sh @@ -17,12 +17,18 @@ try md5 "abcdefghijklmnopqrstuvwxyz" "c3fcd3d76192e4007dfb496cca67e13b" try md5 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "d174ab98d277d9f5a5611c2c9f419d9f" try md5 "12345678901234567890123456789012345678901234567890123456789012345678901234567890" "57edf4a22be3c955ac49da2e2107b67a" +try sha1 "" "da39a3ee5e6b4b0d3255bfef95601890afd80709" try sha1 "abc" "a9993e364706816aba3e25717850c26c9cd0d89d" try sha1 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "84983e441c3bd26ebaae4aa1f95129e5e54670f1" +try sha256 "" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" try sha256 "abc" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" +try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" +try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" + EXTRA=--base32 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" EXTRA= @@ -56,7 +62,12 @@ ln -s x $TEST_ROOT/hash-path/hello try2 md5 "f78b733a68f5edbdf9413899339eaa4a" # Conversion. -test $(nix-hash --type sha256 --to-base32 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") = "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" -test $(nix-hash --type sha256 --to-base16 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s") = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" -test $(nix-hash --type sha1 --to-base32 "800d59cfcd3c05e900cb4e214be48f6b886a08df") = "vw46m23bizj4n8afrc0fj19wrp7mj3c0" -test $(nix-hash --type sha1 --to-base16 "vw46m23bizj4n8afrc0fj19wrp7mj3c0") = "800d59cfcd3c05e900cb4e214be48f6b886a08df" +try3() { + h32=$(nix-hash --type "$1" --to-base32 "$2") + [ "$h32" = "$3" ] + h16=$(nix-hash --type "$1" --to-base16 "$h32") + [ "$h16" = "$2" ] +} +try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" +try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" +try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" diff --git a/tests/install-package.sh b/tests/install-package.sh index 653dfee4c8d1..1916f72713e2 100644 --- a/tests/install-package.sh +++ b/tests/install-package.sh @@ -1,15 +1,14 @@ source common.sh -# Note: this test expects to be run *after* nix-push.sh. - drvPath=$(nix-instantiate ./dependencies.nix) -outPath=$(nix-store -q $drvPath) +outPath=$(nix-store -r $drvPath) +nix-push --dest $cacheDir $outPath clearStore clearProfiles cat > $TEST_ROOT/foo.nixpkg <<EOF -NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath +NIXPKG1 - simple $system $drvPath $outPath file://$cacheDir EOF nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg diff --git a/tests/lang/eval-okay-any-all.exp b/tests/lang/eval-okay-any-all.exp new file mode 100644 index 000000000000..eb273f45b2a6 --- /dev/null +++ b/tests/lang/eval-okay-any-all.exp @@ -0,0 +1 @@ +[ false false true true true true false true ] diff --git a/tests/lang/eval-okay-any-all.nix b/tests/lang/eval-okay-any-all.nix new file mode 100644 index 000000000000..a3f26ea2aa83 --- /dev/null +++ b/tests/lang/eval-okay-any-all.nix @@ -0,0 +1,11 @@ +with builtins; + +[ (any (x: x == 1) []) + (any (x: x == 1) [2 3 4]) + (any (x: x == 1) [1 2 3 4]) + (any (x: x == 1) [4 3 2 1]) + (all (x: x == 1) []) + (all (x: x == 1) [1]) + (all (x: x == 1) [1 2 3]) + (all (x: x == 1) [1 1 1]) +] diff --git a/tests/lang/eval-okay-attrs5.nix b/tests/lang/eval-okay-attrs5.nix index 0a98b8fdffa6..a4584cd3b398 100644 --- a/tests/lang/eval-okay-attrs5.nix +++ b/tests/lang/eval-okay-attrs5.nix @@ -15,7 +15,7 @@ in as.a.b.c or as.x.y.z as.x.y.bla or bs.f-o-o.bar or "xyzzy" as.x.y.bla or bs.bar.foo or "xyzzy" - 123.bla or null.foo or "xyzzy" + (123).bla or null.foo or "xyzzy" # Backwards compatibility test. (fold or [] [true false false]) ] diff --git a/tests/lang/eval-okay-concatstringssep.exp b/tests/lang/eval-okay-concatstringssep.exp new file mode 100644 index 000000000000..93987647ffe6 --- /dev/null +++ b/tests/lang/eval-okay-concatstringssep.exp @@ -0,0 +1 @@ +[ "" "foobarxyzzy" "foo, bar, xyzzy" "foo" "" ] diff --git a/tests/lang/eval-okay-concatstringssep.nix b/tests/lang/eval-okay-concatstringssep.nix new file mode 100644 index 000000000000..adc4c41bd551 --- /dev/null +++ b/tests/lang/eval-okay-concatstringssep.nix @@ -0,0 +1,8 @@ +with builtins; + +[ (concatStringsSep "" []) + (concatStringsSep "" ["foo" "bar" "xyzzy"]) + (concatStringsSep ", " ["foo" "bar" "xyzzy"]) + (concatStringsSep ", " ["foo"]) + (concatStringsSep ", " []) +] diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/lang/eval-okay-fromjson.nix index 5ed0c1c4395d..102ee82b5e6b 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/lang/eval-okay-fromjson.nix @@ -12,7 +12,9 @@ builtins.fromJSON "Width": 100 }, "Animated" : false, - "IDs": [116, 943, 234, 38793, true ,false,null, -100] + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,5 +30,7 @@ builtins.fromJSON }; Animated = false; IDs = [ 116 943 234 38793 true false null (0-100) ]; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-hash.exp b/tests/lang/eval-okay-hash.exp index 7bbe452bcc01..d720a082ddb3 100644 --- a/tests/lang/eval-okay-hash.exp +++ b/tests/lang/eval-okay-hash.exp @@ -1 +1 @@ -[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" ] +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hash.nix b/tests/lang/eval-okay-hash.nix index 2fff17f849bb..b0f62b245ca8 100644 --- a/tests/lang/eval-okay-hash.nix +++ b/tests/lang/eval-okay-hash.nix @@ -1,7 +1,4 @@ let - md5 = builtins.hashString "md5"; - sha1 = builtins.hashString "sha1"; - sha256 = builtins.hashString "sha256"; strings = [ "" "text 1" "text 2" ]; in - (builtins.map md5 strings) ++ (builtins.map sha1 strings) ++ (builtins.map sha256 strings) + builtins.concatLists (map (hash: map (builtins.hashString hash) strings) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp new file mode 100644 index 000000000000..a2add1b7b140 --- /dev/null +++ b/tests/lang/eval-okay-replacestrings.exp @@ -0,0 +1 @@ +[ "faabar" "fbar" "fubar" "faboor" "fubar" ] diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix new file mode 100644 index 000000000000..6284a0e660ae --- /dev/null +++ b/tests/lang/eval-okay-replacestrings.nix @@ -0,0 +1,8 @@ +with builtins; + +[ (replaceStrings ["o"] ["a"] "foobar") + (replaceStrings ["o"] [""] "foobar") + (replaceStrings ["oo"] ["u"] "foobar") + (replaceStrings ["oo" "a"] ["a" "oo"] "foobar") + (replaceStrings ["oo" "oo"] ["u" "i"] "foobar") +] diff --git a/tests/lang/eval-okay-sort.exp b/tests/lang/eval-okay-sort.exp new file mode 100644 index 000000000000..148b93516394 --- /dev/null +++ b/tests/lang/eval-okay-sort.exp @@ -0,0 +1 @@ +[ [ 42 77 147 249 483 526 ] [ 526 483 249 147 77 42 ] [ "bar" "fnord" "foo" "xyzzy" ] [ { key = 1; value = "foo"; } { key = 1; value = "fnord"; } { key = 2; value = "bar"; } ] ] diff --git a/tests/lang/eval-okay-sort.nix b/tests/lang/eval-okay-sort.nix new file mode 100644 index 000000000000..8299c3a4a3aa --- /dev/null +++ b/tests/lang/eval-okay-sort.nix @@ -0,0 +1,8 @@ +with builtins; + +[ (sort lessThan [ 483 249 526 147 42 77 ]) + (sort (x: y: y < x) [ 483 249 526 147 42 77 ]) + (sort lessThan [ "foo" "bar" "xyzzy" "fnord" ]) + (sort (x: y: x.key < y.key) + [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) +] diff --git a/tests/lang/eval-okay-string.exp b/tests/lang/eval-okay-string.exp index 6079202470e3..63f650f73a3a 100644 --- a/tests/lang/eval-okay-string.exp +++ b/tests/lang/eval-okay-string.exp @@ -1 +1 @@ -"foobar/a/b/c/d/foo/xyzzy/foo.txt/../foo/x/yescape: \"quote\" \n \\end\nof\nlinefoobarblaatfoo$bar" +"foobar/a/b/c/d/foo/xyzzy/foo.txt/../foo/x/yescape: \"quote\" \n \\end\nof\nlinefoobarblaatfoo$bar$\"$\"$" diff --git a/tests/lang/eval-okay-string.nix b/tests/lang/eval-okay-string.nix index 839539e6c171..47cc989ad46a 100644 --- a/tests/lang/eval-okay-string.nix +++ b/tests/lang/eval-okay-string.nix @@ -8,3 +8,5 @@ of line" + "foo${if true then "b${"a" + "r"}" else "xyzzy"}blaat" + "foo$bar" + + "$\"$\"" + + "$" diff --git a/tests/lang/eval-okay-tojson.exp b/tests/lang/eval-okay-tojson.exp index e8164af2b66e..33588493f75c 100644 --- a/tests/lang/eval-okay-tojson.exp +++ b/tests/lang/eval-okay-tojson.exp @@ -1 +1 @@ -"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3}" +"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44}" diff --git a/tests/lang/eval-okay-tojson.nix b/tests/lang/eval-okay-tojson.nix index 0d4e55b3d367..c046ba4ae59b 100644 --- a/tests/lang/eval-okay-tojson.nix +++ b/tests/lang/eval-okay-tojson.nix @@ -8,4 +8,5 @@ builtins.toJSON g = [ 1 2 3 ]; h = [ "a" [ "b" { "foo\nbar" = {}; } ] ]; i = 1 + 2; + j = 1.44; } diff --git a/tests/lang/eval-okay-types.exp b/tests/lang/eval-okay-types.exp index 82487f7100e2..9a8ea0bcbd8a 100644 --- a/tests/lang/eval-okay-types.exp +++ b/tests/lang/eval-okay-types.exp @@ -1 +1 @@ -[ true false true false true false true false true false true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] +[ true false true false true false true false true true true true true true true true true true true false true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] diff --git a/tests/lang/eval-okay-types.nix b/tests/lang/eval-okay-types.nix index 8cb225e247fb..a34775f5e602 100644 --- a/tests/lang/eval-okay-types.nix +++ b/tests/lang/eval-okay-types.nix @@ -8,6 +8,16 @@ with builtins; (isString [ "x" ]) (isInt (1 + 2)) (isInt { x = 123; }) + (isInt (1 / 2)) + (isInt (1 + 1)) + (isInt (1 / 2)) + (isInt (1 * 2)) + (isInt (1 - 2)) + (isFloat (1.2)) + (isFloat (1 + 1.0)) + (isFloat (1 / 2.0)) + (isFloat (1 * 2.0)) + (isFloat (1 - 2.0)) (isBool (true && false)) (isBool null) (isAttrs { x = 123; }) diff --git a/tests/lang/eval-okay-xml.exp.xml b/tests/lang/eval-okay-xml.exp.xml index f124f939ed48..92b75e0b8b17 100644 --- a/tests/lang/eval-okay-xml.exp.xml +++ b/tests/lang/eval-okay-xml.exp.xml @@ -45,5 +45,8 @@ <attr name="x"> <int value="123" /> </attr> + <attr name="y"> + <float value="567.89" /> + </attr> </attrs> </expr> diff --git a/tests/lang/eval-okay-xml.nix b/tests/lang/eval-okay-xml.nix index b9389bfae759..9ee9f8a0b4f5 100644 --- a/tests/lang/eval-okay-xml.nix +++ b/tests/lang/eval-okay-xml.nix @@ -2,6 +2,8 @@ rec { x = 123; + y = 567.890; + a = "foo"; b = "bar"; diff --git a/tests/lang/lib.nix b/tests/lang/lib.nix index 882005dc1b5c..028a538314b7 100644 --- a/tests/lang/lib.nix +++ b/tests/lang/lib.nix @@ -17,7 +17,7 @@ rec { then fold (x: y: (flatten x) ++ y) [] x else [x]; - sum = fold (x: y: add x y) 0; + sum = foldl' (x: y: add x y) 0; hasSuffix = ext: fileName: let lenFileName = stringLength fileName; @@ -54,8 +54,8 @@ rec { const = x: y: x; range = first: last: - if builtins.lessThan last first - then [] - else [first] ++ range (builtins.add first 1) last; + if first > last + then [] + else genList (n: first + n) (last - first + 1); } diff --git a/tests/lang/parse-okay-regression-751.nix b/tests/lang/parse-okay-regression-751.nix new file mode 100644 index 000000000000..05c78b3016d3 --- /dev/null +++ b/tests/lang/parse-okay-regression-751.nix @@ -0,0 +1,2 @@ +let const = a: "const"; in +''${ const { x = "q"; }}'' diff --git a/tests/local.mk b/tests/local.mk index 03f53b44c275..7c5a553d39e0 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,15 +3,14 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - build-hook.sh substitutes.sh substitutes2.sh \ - fallback.sh nix-push.sh gc.sh gc-concurrent.sh nix-pull.sh \ + build-hook.sh nix-push.sh gc.sh gc-concurrent.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh negative-caching.sh \ - binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ + remote-store.sh export.sh export-graph.sh \ + timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ - check-reqs.sh pass-as-file.sh tarball.sh + check-reqs.sh pass-as-file.sh tarball.sh restricted.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/logging.sh b/tests/logging.sh index 77b2337a9d00..86f32bade941 100644 --- a/tests/logging.sh +++ b/tests/logging.sh @@ -2,16 +2,7 @@ source common.sh clearStore -# Produce an escaped log file. -path=$(nix-build --log-type escapes -vv dependencies.nix --no-out-link 2> $TEST_ROOT/log.esc) - -# Convert it to an XML representation. -nix-log2xml < $TEST_ROOT/log.esc > $TEST_ROOT/log.xml - -# Is this well-formed XML? -if test "$xmllint" != "false"; then - $xmllint --noout $TEST_ROOT/log.xml || fail "malformed XML" -fi +path=$(nix-build dependencies.nix --no-out-link) # Test nix-store -l. [ "$(nix-store -l $path)" = FOO ] diff --git a/tests/misc.sh b/tests/misc.sh index 1b4d8f2cfc8e..6d0ab3adcec8 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -14,3 +14,6 @@ nix-env --version | grep "$version" # Usage errors. nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" + +# Eval Errors. +nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$" diff --git a/tests/negative-caching.nix b/tests/negative-caching.nix deleted file mode 100644 index 10df67a748fc..000000000000 --- a/tests/negative-caching.nix +++ /dev/null @@ -1,21 +0,0 @@ -with import ./config.nix; - -rec { - - fail = mkDerivation { - name = "fail"; - builder = builtins.toFile "builder.sh" "echo FAIL; exit 1"; - }; - - succeed = mkDerivation { - name = "succeed"; - builder = builtins.toFile "builder.sh" "echo SUCCEED; mkdir $out"; - }; - - depOnFail = mkDerivation { - name = "dep-on-fail"; - builder = builtins.toFile "builder.sh" "echo URGH; mkdir $out"; - inputs = [fail succeed]; - }; - -} diff --git a/tests/negative-caching.sh b/tests/negative-caching.sh deleted file mode 100644 index 4217bc38e121..000000000000 --- a/tests/negative-caching.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore - -set +e - -opts="--option build-cache-failure true --print-build-trace" - -# This build should fail, and the failure should be cached. -log=$(nix-build $opts negative-caching.nix -A fail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "@ build-failed" || fail "no build-failed trace" - -# Do it again. The build shouldn't be tried again. -log=$(nix-build $opts negative-caching.nix -A fail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "FAIL" && fail "failed build not cached" -echo "$log" | grep -q "@ build-failed .* cached" || fail "trace doesn't say cached" - -# Check that --keep-going works properly with cached failures. -log=$(nix-build $opts --keep-going negative-caching.nix -A depOnFail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "FAIL" && fail "failed build not cached (2)" -echo "$log" | grep -q "@ build-failed .* cached" || fail "trace doesn't say cached (2)" -echo "$log" | grep -q "@ build-succeeded .*-succeed" || fail "didn't keep going" diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index b3442f6a8471..2ec986dd415b 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -1,20 +1,16 @@ source common.sh clearProfiles -clearManifests -rm -f $TEST_ROOT/.nix-channels - -# Override location of ~/.nix-channels. -export HOME=$TEST_ROOT +rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile # Test add/list/remove. nix-channel --add http://foo/bar xyzzy nix-channel --list | grep -q http://foo/bar nix-channel --remove xyzzy -[ -e $TEST_ROOT/.nix-channels ] -[ "$(cat $TEST_ROOT/.nix-channels)" = '' ] +[ -e $TEST_HOME/.nix-channels ] +[ "$(cat $TEST_HOME/.nix-channels)" = '' ] # Create a channel. rm -rf $TEST_ROOT/foo @@ -42,11 +38,8 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml nix-env -i dependencies [ -e $TEST_ROOT/var/nix/profiles/default/foobar ] - - clearProfiles -clearManifests -rm -f $TEST_ROOT/.nix-channels +rm -f $TEST_HOME/.nix-channels # Test updating from a tarball nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 foo diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 1418c65897d3..0e42cc0a3d72 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -4,7 +4,7 @@ with import <nixpkgs/nixos/lib/testing.nix> { inherit system; }; -makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; pkgC = pkgs.hello; in { +makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { nodes = { client = diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 3586a7efc3c8..2cc3910ba12d 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -1,10 +1,10 @@ source common.sh -home=$TEST_ROOT/home -rm -rf $home -mkdir -p $home -HOME=$home $SHELL -e -c ". ../scripts/nix-profile.sh" -HOME=$home $SHELL -e -c ". ../scripts/nix-profile.sh" # test idempotency +user=$(whoami) +rm -rf $TEST_HOME +mkdir -p $TEST_HOME +USER=$user $SHELL -e -c ". ../scripts/nix-profile.sh" +USER=$user $SHELL -e -c ". ../scripts/nix-profile.sh" # test idempotency -[ -L $home/.nix-profile ] -[ -e $home/.nix-channels ] +[ -L $TEST_HOME/.nix-profile ] +[ -e $TEST_HOME/.nix-channels ] diff --git a/tests/nix-pull.sh b/tests/nix-pull.sh deleted file mode 100644 index 87239948c481..000000000000 --- a/tests/nix-pull.sh +++ /dev/null @@ -1,33 +0,0 @@ -source common.sh - -pullCache () { - echo "pulling cache..." - nix-pull file://$TEST_ROOT/cache/MANIFEST -} - -clearStore -clearManifests -pullCache - -drvPath=$(nix-instantiate dependencies.nix) -outPath=$(nix-store -q $drvPath) - -echo "building $outPath using substitutes..." -nix-store -r $outPath - -cat $outPath/input-2/bar - -clearStore -clearManifests -pullCache - -echo "building $drvPath using substitutes..." -nix-store -r $drvPath - -cat $outPath/input-2/bar - -# Check that the derivers are set properly. -test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver $(readLink $outPath/input-2) | grep -q -- "-input-2.drv" - -clearManifests diff --git a/tests/referrers.sh b/tests/referrers.sh index 0a1c86e0ab19..5c1ef20cfeb4 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -4,7 +4,7 @@ clearStore max=500 -reference=$NIX_STORE_DIR/abcdef +reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa touch $reference (echo $reference && echo && echo 0) | nix-store --register-validity @@ -12,9 +12,9 @@ echo "making registration..." set +x for ((n = 0; n < $max; n++)); do - storePath=$NIX_STORE_DIR/$n + storePath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$n echo -n > $storePath - ref2=$NIX_STORE_DIR/$((n+1)) + ref2=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$((n+1)) if test $((n+1)) = $max; then ref2=$reference fi @@ -30,7 +30,7 @@ echo "collecting garbage..." ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref nix-store --gc -if [ -n "$(type -p sqlite3)" -a "$(sqlite3 ./test-tmp/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then +if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_DB_DIR/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then echo "referrers not cleaned up" exit 1 fi diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 8312424f0ac6..b3908717a40e 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -1,7 +1,6 @@ source common.sh clearStore -clearManifests startDaemon diff --git a/tests/repair.sh b/tests/repair.sh index ae82b649c6ac..92f2f8fe60a5 100644 --- a/tests/repair.sh +++ b/tests/repair.sh @@ -18,6 +18,18 @@ if nix-store --verify --check-contents -v; then exit 1 fi +# The path can be repaired by rebuilding the derivation. +nix-store --verify --check-contents --repair + +nix-store --verify-path $path2 + +# Re-corrupt and delete the deriver. Now --verify --repair should +# not work. +chmod u+w $path2 +touch $path2/bad + +nix-store --delete $(nix-store -qd $path2) + if nix-store --verify --check-contents --repair; then echo "nix-store --verify --repair succeeded unexpectedly" >&2 exit 1 diff --git a/tests/restricted.sh b/tests/restricted.sh new file mode 100644 index 000000000000..19096a9f8dd2 --- /dev/null +++ b/tests/restricted.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore + +nix-instantiate --option restrict-eval true --eval -E '1 + 2' +(! nix-instantiate --option restrict-eval true ./simple.nix) +nix-instantiate --option restrict-eval true ./simple.nix -I src=. +nix-instantiate --option restrict-eval true ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh + +(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix') +nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix' -I src=.. + +(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost') +nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost' -I src=../src + +(! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') +nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. + diff --git a/tests/secure-drv-outputs.sh b/tests/secure-drv-outputs.sh index 4888123da910..50a9c4428d30 100644 --- a/tests/secure-drv-outputs.sh +++ b/tests/secure-drv-outputs.sh @@ -5,7 +5,6 @@ source common.sh clearStore -clearManifests startDaemon diff --git a/tests/substituter.sh b/tests/substituter.sh deleted file mode 100755 index 9aab295de87b..000000000000 --- a/tests/substituter.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/sh -e -echo -echo substituter args: $* >&2 - -if test $1 = "--query"; then - while read cmd args; do - echo "CMD = $cmd, ARGS = $args" >&2 - if test "$cmd" = "have"; then - for path in $args; do - read path - if grep -q "$path" $TEST_ROOT/sub-paths; then - echo $path - fi - done - echo - elif test "$cmd" = "info"; then - for path in $args; do - echo $path - echo "" # deriver - echo 0 # nr of refs - echo $((1 * 1024 * 1024)) # download size - echo $((2 * 1024 * 1024)) # nar size - done - echo - else - echo "bad command $cmd" - exit 1 - fi - done -elif test $1 = "--substitute"; then - mkdir $2 - echo "Hallo Wereld" > $2/hello - echo # no expected hash -else - echo "unknown substituter operation" - exit 1 -fi diff --git a/tests/substituter2.sh b/tests/substituter2.sh deleted file mode 100755 index 5d1763599c25..000000000000 --- a/tests/substituter2.sh +++ /dev/null @@ -1,33 +0,0 @@ -#! /bin/sh -e -echo -echo substituter2 args: $* >&2 - -if test $1 = "--query"; then - while read cmd args; do - if test "$cmd" = have; then - for path in $args; do - if grep -q "$path" $TEST_ROOT/sub-paths; then - echo $path - fi - done - echo - elif test "$cmd" = info; then - for path in $args; do - echo $path - echo "" # deriver - echo 0 # nr of refs - echo 0 # download size - echo 0 # nar size - done - echo - else - echo "bad command $cmd" - exit 1 - fi - done -elif test $1 = "--substitute"; then - exit 1 -else - echo "unknown substituter operation" - exit 1 -fi diff --git a/tests/substitutes.sh b/tests/substitutes.sh deleted file mode 100644 index 0c6adf2601fa..000000000000 --- a/tests/substitutes.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore - -# Instantiate. -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -# Find the output path. -outPath=$(nix-store -qvv "$drvPath") -echo "output path is $outPath" - -echo $outPath > $TEST_ROOT/sub-paths - -export NIX_SUBSTITUTERS=$(pwd)/substituter.sh - -nix-store -r "$drvPath" --dry-run 2>&1 | grep -q "1.00 MiB.*2.00 MiB" - -nix-store -rvv "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi diff --git a/tests/substitutes2.sh b/tests/substitutes2.sh deleted file mode 100644 index bd914575cca8..000000000000 --- a/tests/substitutes2.sh +++ /dev/null @@ -1,21 +0,0 @@ -source common.sh - -clearStore - -# Instantiate. -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -# Find the output path. -outPath=$(nix-store -qvvvvv "$drvPath") -echo "output path is $outPath" - -echo $outPath > $TEST_ROOT/sub-paths - -# First try a substituter that fails, then one that succeeds -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh:$(pwd)/substituter.sh - -nix-store -j0 -rvv "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi diff --git a/tests/tarball.sh b/tests/tarball.sh index cb5258a9e548..329e73b91696 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -2,8 +2,7 @@ source common.sh clearStore -export HOME=$TEST_ROOT/home -rm -rf $TEST_ROOT/home +rm -rf $TEST_HOME tarroot=$TEST_ROOT/tarball rm -rf $tarroot @@ -16,8 +15,14 @@ tarball=$TEST_ROOT/tarball.tar.xz nix-env -f file://$tarball -qa --out-path | grep -q dependencies -nix-build file://$tarball +nix-build -o $TMPDIR/result file://$tarball -nix-build '<foo>' -I foo=file://$tarball +nix-build -o $TMPDIR/result '<foo>' -I foo=file://$tarball -nix-build -E "import (fetchTarball file://$tarball)" +nix-build -o $TMPDIR/result -E "import (fetchTarball file://$tarball)" + +nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz +nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz +(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz) + +nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=. diff --git a/tests/user-envs.sh b/tests/user-envs.sh index 5e4fcb7bc7f5..526c1267cefe 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -6,8 +6,7 @@ clearProfiles # Query installed: should be empty. test "$(nix-env -p $profiles/test -q '*' | wc -l)" -eq 0 -export HOME=$TEST_ROOT/home -mkdir -p $HOME +mkdir -p $TEST_HOME nix-env --switch-profile $profiles/test # Query available: should contain several. diff --git a/version b/version index 578c71bd5582..35d51f33b34f 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.10 \ No newline at end of file +1.12 \ No newline at end of file |