diff options
82 files changed, 2264 insertions, 1018 deletions
diff --git a/.gitignore b/.gitignore index ce22fa007dc7..0a9599378567 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ perl/Makefile.config /scripts/nix-copy-closure /scripts/nix-reduce-build /scripts/nix-http-export.cgi +/scripts/nix-profile-daemon.sh # /src/libexpr/ /src/libexpr/lexer-tab.cc diff --git a/Makefile b/Makefile index 5d8e990cc5c0..c867823fc485 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,8 @@ makefiles = \ misc/launchd/local.mk \ misc/upstart/local.mk \ doc/manual/local.mk \ - tests/local.mk + tests/local.mk \ + tests/plugins/local.mk GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h diff --git a/Makefile.config.in b/Makefile.config.in index fab82194656e..a9785dc73955 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -7,6 +7,7 @@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ HAVE_READLINE = @HAVE_READLINE@ HAVE_BROTLI = @HAVE_BROTLI@ +HAVE_SECCOMP = @HAVE_SECCOMP@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ diff --git a/configure.ac b/configure.ac index 83b2346d065c..54322d463ae7 100644 --- a/configure.ac +++ b/configure.ac @@ -175,6 +175,8 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium]) # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) +AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], + [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) # Look for libbrotli{enc,dec}, optional dependencies @@ -186,9 +188,22 @@ AC_SUBST(HAVE_BROTLI, [$have_brotli]) # Look for libseccomp, required for Linux sandboxing. if test "$sys_name" = linux; then - PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], - [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + AC_ARG_ENABLE([seccomp-sandboxing], + AC_HELP_STRING([--disable-seccomp-sandboxing], + [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] + )) + if test "x$enable_seccomp_sandboxing" != "xno"; then + PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + have_seccomp=1 + AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) + else + have_seccomp= + fi +else + have_seccomp= fi +AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) # Look for aws-cpp-sdk-s3. diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index 1957e1105e68..20fd6a0cfb0d 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -4,71 +4,109 @@ version="5.0" xml:id='chap-distributed-builds'> -<title>Distributed Builds</title> - -<para>Nix supports distributed builds, where a local Nix installation can -forward Nix builds to other machines over the network. This allows -multiple builds to be performed in parallel (thus improving -performance) and allows Nix to perform multi-platform builds in a -semi-transparent way. For instance, if you perform a build for a -<literal>x86_64-darwin</literal> on an <literal>i686-linux</literal> -machine, Nix can automatically forward the build to a -<literal>x86_64-darwin</literal> machine, if available.</para> - -<para>You can enable distributed builds by setting the environment -variable <envar>NIX_BUILD_HOOK</envar> to point to a program that Nix -will call whenever it wants to build a derivation. The build hook -(typically a shell or Perl script) can decline the build, in which Nix -will perform it in the usual way if possible, or it can accept it, in -which case it is responsible for somehow getting the inputs of the -build to another machine, doing the build there, and getting the -results back.</para> - -<example xml:id='ex-remote-systems'><title>Remote machine configuration: -<filename>remote-systems.conf</filename></title> -<programlisting> -nix@mcflurry.labs.cs.uu.nl x86_64-darwin /home/nix/.ssh/id_quarterpounder_auto 2 -nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm -nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 -nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 kvm perf -</programlisting> -</example> - -<para>Nix ships with a build hook that should be suitable for most -purposes. It uses <command>ssh</command> and -<command>nix-copy-closure</command> to copy the build inputs and -outputs and perform the remote build. To use it, you should set -<envar>NIX_BUILD_HOOK</envar> to -<filename><replaceable>prefix</replaceable>/libexec/nix/build-remote</filename>. -You should also define a list of available build machines and point -the environment variable <envar>NIX_REMOTE_SYSTEMS</envar> to -it. <envar>NIX_REMOTE_SYSTEMS</envar> must be an absolute path. An -example configuration is shown in <xref linkend='ex-remote-systems' -/>. Each line in the file specifies a machine, with the following -bits of information: +<title>Remote Builds</title> + +<para>Nix supports remote builds, where a local Nix installation can +forward Nix builds to other machines. This allows multiple builds to +be performed in parallel and allows Nix to perform multi-platform +builds in a semi-transparent way. For instance, if you perform a +build for a <literal>x86_64-darwin</literal> on an +<literal>i686-linux</literal> machine, Nix can automatically forward +the build to a <literal>x86_64-darwin</literal> machine, if +available.</para> + +<para>To forward a build to a remote machine, it’s required that the +remote machine is accessible via SSH and that it has Nix +installed. You can test whether connecting to the remote Nix instance +works, e.g. + +<screen> +$ nix ping-store --store ssh://mac +</screen> + +will try to connect to the machine named <literal>mac</literal>. It is +possible to specify an SSH identity file as part of the remote store +URI, e.g. + +<screen> +$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key +</screen> + +Since builds should be non-interactive, the key should not have a +passphrase. Alternatively, you can load identities ahead of time into +<command>ssh-agent</command> or <command>gpg-agent</command>.</para> + +<para>If you get the error + +<screen> +bash: nix-store: command not found +error: cannot connect to 'mac' +</screen> + +then you need to ensure that the <envar>PATH</envar> of +non-interactive login shells contains Nix.</para> + +<warning><para>If you are building via the Nix daemon, it is the Nix +daemon user account (that is, <literal>root</literal>) that should +have SSH access to the remote machine. If you can’t or don’t want to +configure <literal>root</literal> to be able to access to remote +machine, you can use a private Nix store instead by passing +e.g. <literal>--store ~/my-nix</literal>.</para></warning> + +<para>The list of remote machines can be specified on the command line +or in the Nix configuration file. The former is convenient for +testing. For example, the following command allows you to build a +derivation for <literal>x86_64-darwin</literal> on a Linux machine: + +<screen> +$ uname +Linux + +$ nix build \ + '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ + --builders 'ssh://mac x86_64-darwin' +[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac + +$ cat ./result +Darwin +</screen> + +It is possible to specify multiple builders separated by a semicolon +or a newline, e.g. + +<screen> + --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' +</screen> +</para> + +<para>Each machine specification consists of the following elements, +separated by spaces. Only the first element is required. <orderedlist> - <listitem><para>The name of the remote machine, with optionally the - user under which the remote build should be performed. This is - actually passed as an argument to <command>ssh</command>, so it can - be an alias defined in your + <listitem><para>The URI of the remote store in the format + <literal>ssh://[<replaceable>username</replaceable>@]<replaceable>hostname</replaceable></literal>, + e.g. <literal>ssh://nix@mac</literal> or + <literal>ssh://mac</literal>. For backward compatibility, + <literal>ssh://</literal> may be omitted. The hostname may be an + alias defined in your <filename>~/.ssh/config</filename>.</para></listitem> <listitem><para>A comma-separated list of Nix platform type identifiers, such as <literal>x86_64-darwin</literal>. It is possible for a machine to support multiple platform types, e.g., - <literal>i686-linux,x86_64-linux</literal>.</para></listitem> + <literal>i686-linux,x86_64-linux</literal>. If omitted, this + defaults to the local platform type.</para></listitem> - <listitem><para>The SSH private key to be used to log in to the - remote machine. Since builds should be non-interactive, this key - should not have a passphrase!</para></listitem> + <listitem><para>The SSH identity file to be used to log in to the + remote machine. If omitted, SSH will use its regular + identities.</para></listitem> - <listitem><para>The maximum number of builds that - <filename>build-remote</filename> will execute in parallel on the - machine. Typically this should be equal to the number of CPU cores. - For instance, the machine <literal>itchy</literal> in the example - will execute up to 8 builds in parallel.</para></listitem> + <listitem><para>The maximum number of builds that Nix will execute + in parallel on the machine. Typically this should be equal to the + number of CPU cores. For instance, the machine + <literal>itchy</literal> in the example will execute up to 8 builds + in parallel.</para></listitem> <listitem><para>The “speed factor”, indicating the relative speed of the machine. If there are multiple machines of the right type, Nix @@ -76,30 +114,69 @@ bits of information: <listitem><para>A comma-separated list of <emphasis>supported features</emphasis>. If a derivation has the - <varname>requiredSystemFeatures</varname> attribute, then - <filename>build-remote</filename> will only perform the - derivation on a machine that has the specified features. For - instance, the attribute + <varname>requiredSystemFeatures</varname> attribute, then Nix will + only perform the derivation on a machine that has the specified + features. For instance, the attribute <programlisting> requiredSystemFeatures = [ "kvm" ]; </programlisting> will cause the build to be performed on a machine that has the - <literal>kvm</literal> feature (i.e., <literal>scratchy</literal> in - the example above).</para></listitem> + <literal>kvm</literal> feature.</para></listitem> <listitem><para>A comma-separated list of <emphasis>mandatory features</emphasis>. A machine will only be used to build a derivation if all of the machine’s mandatory features appear in the - derivation’s <varname>requiredSystemFeatures</varname> attribute. - Thus, in the example, the machine <literal>poochie</literal> will - only do derivations that have - <varname>requiredSystemFeatures</varname> set to <literal>["kvm" - "perf"]</literal> or <literal>["perf"]</literal>.</para></listitem> + derivation’s <varname>requiredSystemFeatures</varname> + attribute..</para></listitem> </orderedlist> -</para> +For example, the machine specification + +<programlisting> +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark +</programlisting> + +specifies several machines that can perform +<literal>i686-linux</literal> builds. However, +<literal>poochie</literal> will only do builds that have the attribute + +<programlisting> +requiredSystemFeatures = [ "benchmark" ]; +</programlisting> + +or + +<programlisting> +requiredSystemFeatures = [ "benchmark" "kvm" ]; +</programlisting> + +<literal>itchy</literal> cannot do builds that require +<literal>kvm</literal>, but <literal>scratchy</literal> does support +such builds. For regular builds, <literal>itchy</literal> will be +preferred over <literal>scratchy</literal> because it has a higher +speed factor.</para> + +<para>Remote builders can also be configured in +<filename>nix.conf</filename>, e.g. + +<programlisting> +builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd +</programlisting> + +Finally, remote builders can be configured in a separate configuration +file included in <option>builders</option> via the syntax +<literal>@<replaceable>file</replaceable></literal>. For example, + +<programlisting> +builders = @/etc/nix/machines +</programlisting> + +causes the list of machines in <filename>/etc/nix/machines</filename> +to be included. (This is the default.)</para> </chapter> diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index fff7994f28df..c76640c97e7e 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -40,7 +40,12 @@ <para>The configuration files consist of <literal><replaceable>name</replaceable> = -<replaceable>value</replaceable></literal> pairs, one per line. +<replaceable>value</replaceable></literal> pairs, one per line. Other +files can be included with a line like <literal>include +<replaceable>path</replaceable></literal>, where +<replaceable>path</replaceable> is interpreted relative to the current +conf file and a missing file is an error unless +<literal>!include</literal> is used instead. Comments start with a <literal>#</literal> character. Here is an example configuration file:</para> @@ -58,147 +63,99 @@ false</literal>.</para> <variablelist> - <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term> - - <listitem><para>If <literal>true</literal>, the garbage collector - will keep the outputs of non-garbage derivations. If - <literal>false</literal> (default), outputs will be deleted unless - they are GC roots themselves (or reachable from other roots).</para> - - <para>In general, outputs must be registered as roots separately. - However, even if the output of a derivation is registered as a - root, the collector will still delete store paths that are used - only at build time (e.g., the C compiler, or source tarballs - downloaded from the network). To prevent it from doing so, set - this option to <literal>true</literal>.</para></listitem> - - </varlistentry> - - - <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term> - - <listitem><para>If <literal>true</literal> (default), the garbage - collector will keep the derivations from which non-garbage store - paths were built. If <literal>false</literal>, they will be - deleted unless explicitly registered as a root (or reachable from - other roots).</para> - - <para>Keeping derivation around is useful for querying and - traceability (e.g., it allows you to ask with what dependencies or - options a store path was built), so by default this option is on. - Turn it off to save a bit of disk space (or a lot if - <literal>keep-outputs</literal> is also turned on).</para></listitem> - - </varlistentry> - - - <varlistentry><term><literal>keep-env-derivations</literal></term> + <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term> - <listitem><para>If <literal>false</literal> (default), derivations - are not stored in Nix user environments. That is, the derivation - any build-time-only dependencies may be garbage-collected.</para> + <listitem> - <para>If <literal>true</literal>, when you add a Nix derivation to - a user environment, the path of the derivation is stored in the - user environment. Thus, the derivation will not be - garbage-collected until the user environment generation is deleted - (<command>nix-env --delete-generations</command>). To prevent - build-time-only dependencies from being collected, you should also - turn on <literal>keep-outputs</literal>.</para> + <para>A list of URI prefixes to which access is allowed in + restricted evaluation mode. For example, when set to + <literal>https://github.com/NixOS</literal>, builtin functions + such as <function>fetchGit</function> are allowed to access + <literal>https://github.com/NixOS/patchelf.git</literal>.</para> - <para>The difference between this option and - <literal>keep-derivations</literal> is that this one is - “sticky”: it applies to any user environment created while this - option was enabled, while <literal>keep-derivations</literal> - only applies at the moment the garbage collector is - run.</para></listitem> + </listitem> </varlistentry> - <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term> + <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term> - <listitem><para>This option defines the maximum number of jobs - that Nix will try to build in parallel. The default is - <literal>1</literal>. The special value <literal>auto</literal> - causes Nix to use the number of CPUs in your system. It can be - overridden using the <option - linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>) - command line switch.</para></listitem> + <listitem><para>By default, Nix allows you to <function>import</function> from a derivation, + allowing building at evaluation time. With this option set to false, Nix will throw an error + when evaluating an expression that uses this feature, allowing users to ensure their evaluation + will not require any builds to take place.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term> + <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term> - <listitem><para>Sets the value of the - <envar>NIX_BUILD_CORES</envar> environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - <varname>enableParallelBuilding</varname> is set to - <literal>true</literal>, the builder passes the - <option>-j<replaceable>N</replaceable></option> flag to GNU Make. - It can be overridden using the <option - linkend='opt-cores'>--cores</option> command line switch and - defaults to <literal>1</literal>. The value <literal>0</literal> - means that the builder should use all available CPU cores in the - system.</para></listitem> + <listitem><para>(Linux-specific.) By default, builders on Linux + cannot acquire new privileges by calling setuid/setgid programs or + programs that have file capabilities. For example, programs such + as <command>sudo</command> or <command>ping</command> will + fail. (Note that in sandbox builds, no such programs are available + unless you bind-mount them into the sandbox via the + <option>sandbox-paths</option> option.) You can allow the + use of such programs by enabling this option. This is impure and + usually undesirable, but may be useful in certain scenarios + (e.g. to spin up containers or set up userspace network interfaces + in tests).</para></listitem> </varlistentry> - <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term> + <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term> <listitem> - <para>This option defines the maximum number of seconds that a - builder can go without producing any data on standard output or - standard error. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite - loop, or to catch remote builds that are hanging due to network - problems. It can be overridden using the <option - linkend="opt-max-silent-time">--max-silent-time</option> command - line switch.</para> + <para>A list of names of users (separated by whitespace) that + are allowed to connect to the Nix daemon. As with the + <option>trusted-users</option> option, you can specify groups by + prefixing them with <literal>@</literal>. Also, you can allow + all users by specifying <literal>*</literal>. The default is + <literal>*</literal>.</para> - <para>The value <literal>0</literal> means that there is no - timeout. This is also the default.</para> + <para>Note that trusted users are always allowed to connect.</para> </listitem> </varlistentry> - <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term> + <varlistentry><term><literal>auto-optimise-store</literal></term> - <listitem> + <listitem><para>If set to <literal>true</literal>, Nix + automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to <literal>false</literal> (the + default), you can still run <command>nix-store + --optimise</command> to get rid of duplicate + files.</para></listitem> - <para>This option defines the maximum number of seconds that a - builder can run. This is useful (for instance in an automated - build system) to catch builds that are stuck in an infinite loop - but keep writing to their standard output or standard error. It - can be overridden using the <option - linkend="opt-timeout">--timeout</option> command line - switch.</para> + </varlistentry> - <para>The value <literal>0</literal> means that there is no - timeout. This is also the default.</para> + <varlistentry xml:id="conf-builders"> + <term><literal>builders</literal></term> + <listitem> + <para>A list of machines on which to perform builds. <phrase + condition="manual">See <xref linkend="chap-distributed-builds" + /> for details.</phrase></para> </listitem> - </varlistentry> - <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term> - - <listitem> - - <para>This option defines the maximum number of bytes that a - builder can write to its stdout/stderr. If the builder exceeds - this limit, it’s killed. A value of <literal>0</literal> (the - default) means that there is no limit.</para> + <varlistentry><term><literal>builders-use-substitutes</literal></term> - </listitem> + <listitem><para>If set to <literal>true</literal>, Nix will instruct + remote build machines to use their own binary substitutes if available. In + practical terms, this means that remote hosts will fetch as many build + dependencies as possible from their own substitutes (e.g, from + <literal>cache.nixos.org</literal>), instead of waiting for this host to + upload them all. This can drastically reduce build times if the network + connection between this computer and the remote build host is slow. Defaults + to <literal>false</literal>.</para></listitem> </varlistentry> @@ -244,66 +201,51 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>sandbox</literal></term> + <varlistentry><term><literal>compress-build-log</literal></term> - <listitem><para>If set to <literal>true</literal>, builds will be - performed in a <emphasis>sandboxed environment</emphasis>, i.e., - they’re isolated from the normal file system hierarchy and will - only see their dependencies in the Nix store, the temporary build - directory, private versions of <filename>/proc</filename>, - <filename>/dev</filename>, <filename>/dev/shm</filename> and - <filename>/dev/pts</filename> (on Linux), and the paths configured with the - <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal> - option</link>. This is useful to prevent undeclared dependencies - on files in directories such as <filename>/usr/bin</filename>. In - addition, on Linux, builds run in private PID, mount, network, IPC - and UTS namespaces to isolate them from other processes in the - system (except that fixed-output derivations do not run in private - network namespace to ensure they can access the network).</para> + <listitem><para>If set to <literal>true</literal> (the default), + build logs written to <filename>/nix/var/log/nix/drvs</filename> + will be compressed on the fly using bzip2. Otherwise, they will + not be compressed.</para></listitem> - <para>Currently, sandboxing only work on Linux and macOS. The use - of a sandbox requires that Nix is run as root (so you should use - the <link linkend='conf-build-users-group'>“build users” - feature</link> to perform the actual builds under different users - than root).</para> + </varlistentry> - <para>If this option is set to <literal>relaxed</literal>, then - fixed-output derivations and derivations that have the - <varname>__noChroot</varname> attribute set to - <literal>true</literal> do not run in sandboxes.</para> - <para>The default is <literal>false</literal>.</para> + <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term> + + <listitem> + + <para>The timeout (in seconds) for establishing connections in + the binary cache substituter. It corresponds to + <command>curl</command>’s <option>--connect-timeout</option> + option.</para> </listitem> </varlistentry> - <varlistentry xml:id="conf-sandbox-paths"> - <term><literal>sandbox-paths</literal></term> - - <listitem><para>A list of paths bind-mounted into Nix sandbox - environments. You can use the syntax - <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal> - to mount a path in a different location in the sandbox; for - instance, <literal>/bin=/nix-bin</literal> will mount the path - <literal>/nix-bin</literal> as <literal>/bin</literal> inside the - sandbox. If <replaceable>source</replaceable> is followed by - <literal>?</literal>, then it is not an error if - <replaceable>source</replaceable> does not exist; for example, - <literal>/dev/nvidiactl?</literal> specifies that - <filename>/dev/nvidiactl</filename> will only be mounted in the - sandbox if it exists in the host filesystem.</para> + <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term> - <para>Depending on how Nix was built, the default value for this option - may be empty or provide <filename>/bin/sh</filename> as a - bind-mount of <command>bash</command>.</para></listitem> + <listitem><para>Sets the value of the + <envar>NIX_BUILD_CORES</envar> environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + <varname>enableParallelBuilding</varname> is set to + <literal>true</literal>, the builder passes the + <option>-j<replaceable>N</replaceable></option> flag to GNU Make. + It can be overridden using the <option + linkend='opt-cores'>--cores</option> command line switch and + defaults to <literal>1</literal>. The value <literal>0</literal> + means that the builder should use all available CPU cores in the + system.</para></listitem> </varlistentry> <varlistentry xml:id="conf-extra-sandbox-paths"> - <term><literal>build-extra-sandbox-paths</literal></term> + <term><literal>extra-sandbox-paths</literal></term> <listitem><para>A list of additional paths appended to <option>sandbox-paths</option>. Useful if you want to extend @@ -312,25 +254,13 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>substitute</literal></term> - - <listitem><para>If set to <literal>true</literal> (default), Nix - will use binary substitutes if available. This option can be - disabled to force building from source.</para></listitem> - - </varlistentry> - - - <varlistentry><term><literal>builders-use-substitutes</literal></term> + <varlistentry><term><literal>extra-substituters</literal></term> - <listitem><para>If set to <literal>true</literal>, Nix will instruct - remote build machines to use their own binary substitutes if available. In - practical terms, this means that remote hosts will fetch as many build - dependencies as possible from their own substitutes (e.g, from - <literal>cache.nixos.org</literal>), instead of waiting for this host to - upload them all. This can drastically reduce build times if the network - connection between this computer and the remote build host is slow. Defaults - to <literal>false</literal>.</para></listitem> + <listitem><para>Additional binary caches appended to those + specified in <option>substituters</option>. When used by + unprivileged users, untrusted substituters (i.e. those not listed + in <option>trusted-substituters</option>) are silently + ignored.</para></listitem> </varlistentry> @@ -345,119 +275,168 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>keep-build-log</literal></term> + <varlistentry><term><literal>fsync-metadata</literal></term> - <listitem><para>If set to <literal>true</literal> (the default), - Nix will write the build log of a derivation (i.e. the standard - output and error of its builder) to the directory - <filename>/nix/var/log/nix/drvs</filename>. The build log can be - retrieved using the command <command>nix-store -l - <replaceable>path</replaceable></command>.</para></listitem> + <listitem><para>If set to <literal>true</literal>, changes to the + Nix store metadata (in <filename>/nix/var/nix/db</filename>) are + synchronously flushed to disk. This improves robustness in case + of system crashes, but reduces performance. The default is + <literal>true</literal>.</para></listitem> </varlistentry> - <varlistentry><term><literal>compress-build-log</literal></term> + <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term> - <listitem><para>If set to <literal>true</literal> (the default), - build logs written to <filename>/nix/var/log/nix/drvs</filename> - will be compressed on the fly using bzip2. Otherwise, they will - not be compressed.</para></listitem> + <listitem><para>A list of web servers used by + <function>builtins.fetchurl</function> to obtain files by + hash. The default is + <literal>http://tarballs.nixos.org/</literal>. Given a hash type + <replaceable>ht</replaceable> and a base-16 hash + <replaceable>h</replaceable>, Nix will try to download the file + from + <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>. + This allows files to be downloaded even if they have disappeared + from their original URI. For example, given the default mirror + <literal>http://tarballs.nixos.org/</literal>, when building the derivation + +<programlisting> +builtins.fetchurl { + url = https://example.org/foo-1.2.3.tar.xz; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; +} +</programlisting> + + Nix will attempt to download this file from + <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal> + first. If it is not available there, if will try the original URI.</para></listitem> </varlistentry> - <varlistentry><term><literal>substituters</literal></term> + <varlistentry><term><literal>http-connections</literal></term> - <listitem><para>A list of URLs of substituters, separated by - whitespace. The default is - <literal>https://cache.nixos.org</literal>.</para></listitem> + <listitem><para>The maximum number of parallel TCP connections + used to fetch files from binary caches and by other downloads. It + defaults to 25. 0 means no limit.</para></listitem> </varlistentry> - <!-- - <varlistentry><term><literal>binary-caches-files</literal></term> + <varlistentry><term><literal>keep-build-log</literal></term> - <listitem><para>A list of names of files that will be read to - obtain additional binary cache URLs. The default is - <literal>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels/binary-caches/*</literal>. - Note that when you’re using the Nix daemon, - <replaceable>username</replaceable> is always equal to - <literal>root</literal>, so Nix will only use the binary caches - provided by the channels installed by root. Do not set this - option to read files created by untrusted users!</para></listitem> + <listitem><para>If set to <literal>true</literal> (the default), + Nix will write the build log of a derivation (i.e. the standard + output and error of its builder) to the directory + <filename>/nix/var/log/nix/drvs</filename>. The build log can be + retrieved using the command <command>nix-store -l + <replaceable>path</replaceable></command>.</para></listitem> </varlistentry> - --> - <varlistentry><term><literal>trusted-substituters</literal></term> + <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term> - <listitem><para>A list of URLs of substituters, separated by - whitespace. These are not used by default, but can be enabled by - users of the Nix daemon by specifying <literal>--option - substituters <replaceable>urls</replaceable></literal> on the - command line. Unprivileged users are only allowed to pass a - subset of the URLs listed in <literal>substituters</literal> and - <literal>trusted-substituters</literal>.</para></listitem> + <listitem><para>If <literal>true</literal> (default), the garbage + collector will keep the derivations from which non-garbage store + paths were built. If <literal>false</literal>, they will be + deleted unless explicitly registered as a root (or reachable from + other roots).</para> + + <para>Keeping derivation around is useful for querying and + traceability (e.g., it allows you to ask with what dependencies or + options a store path was built), so by default this option is on. + Turn it off to save a bit of disk space (or a lot if + <literal>keep-outputs</literal> is also turned on).</para></listitem> </varlistentry> - <varlistentry><term><literal>extra-substituters</literal></term> + <varlistentry><term><literal>keep-env-derivations</literal></term> - <listitem><para>Additional binary caches appended to those - specified in <option>substituters</option>. When used by - unprivileged users, untrusted substituters (i.e. those not listed - in <option>trusted-substituters</option>) are silently - ignored.</para></listitem> + <listitem><para>If <literal>false</literal> (default), derivations + are not stored in Nix user environments. That is, the derivation + any build-time-only dependencies may be garbage-collected.</para> + + <para>If <literal>true</literal>, when you add a Nix derivation to + a user environment, the path of the derivation is stored in the + user environment. Thus, the derivation will not be + garbage-collected until the user environment generation is deleted + (<command>nix-env --delete-generations</command>). To prevent + build-time-only dependencies from being collected, you should also + turn on <literal>keep-outputs</literal>.</para> + + <para>The difference between this option and + <literal>keep-derivations</literal> is that this one is + “sticky”: it applies to any user environment created while this + option was enabled, while <literal>keep-derivations</literal> + only applies at the moment the garbage collector is + run.</para></listitem> </varlistentry> - <varlistentry><term><literal>require-sigs</literal></term> + <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term> - <listitem><para>If set to <literal>true</literal> (the default), - any non-content-addressed path added or copied to the Nix store - (e.g. when substituting from a binary cache) must have a valid - signature, that is, be signed using one of the keys listed in - <option>trusted-public-keys</option> or - <option>secret-key-files</option>. Set to <literal>false</literal> - to disable signature checking.</para></listitem> + <listitem><para>If <literal>true</literal>, the garbage collector + will keep the outputs of non-garbage derivations. If + <literal>false</literal> (default), outputs will be deleted unless + they are GC roots themselves (or reachable from other roots).</para> + + <para>In general, outputs must be registered as roots separately. + However, even if the output of a derivation is registered as a + root, the collector will still delete store paths that are used + only at build time (e.g., the C compiler, or source tarballs + downloaded from the network). To prevent it from doing so, set + this option to <literal>true</literal>.</para></listitem> </varlistentry> - <varlistentry><term><literal>trusted-public-keys</literal></term> + <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term> - <listitem><para>A whitespace-separated list of public keys. When - paths are copied from another Nix store (such as a binary cache), - they must be signed with one of these keys. For example: - <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem> + <listitem> + + <para>This option defines the maximum number of bytes that a + builder can write to its stdout/stderr. If the builder exceeds + this limit, it’s killed. A value of <literal>0</literal> (the + default) means that there is no limit.</para> + + </listitem> </varlistentry> - <varlistentry><term><literal>secret-key-files</literal></term> + <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term> - <listitem><para>A whitespace-separated list of files containing - secret (private) keys. These are used to sign locally-built - paths. They can be generated using <command>nix-store - --generate-binary-cache-key</command>. The corresponding public - key can be distributed to other users, who can add it to - <option>trusted-public-keys</option> in their - <filename>nix.conf</filename>.</para></listitem> + <listitem><para>This option defines the maximum number of jobs + that Nix will try to build in parallel. The default is + <literal>1</literal>. The special value <literal>auto</literal> + causes Nix to use the number of CPUs in your system. It can be + overridden using the <option + linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>) + command line switch.</para></listitem> </varlistentry> - <varlistentry><term><literal>http-connections</literal></term> + <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term> - <listitem><para>The maximum number of parallel TCP connections - used to fetch files from binary caches and by other downloads. It - defaults to 25. 0 means no limit.</para></listitem> + <listitem> + + <para>This option defines the maximum number of seconds that a + builder can go without producing any data on standard output or + standard error. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite + loop, or to catch remote builds that are hanging due to network + problems. It can be overridden using the <option + linkend="opt-max-silent-time">--max-silent-time</option> command + line switch.</para> + + <para>The value <literal>0</literal> means that there is no + timeout. This is also the default.</para> + + </listitem> </varlistentry> @@ -485,104 +464,95 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>system</literal></term> - - <listitem><para>This option specifies the canonical Nix system - name of the current installation, such as - <literal>i686-linux</literal> or - <literal>x86_64-darwin</literal>. Nix can only build derivations - whose <literal>system</literal> attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only - makes sense if the Nix binaries can run on multiple platforms, - e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and - <literal>i686-linux</literal>.</para> - - <para>It defaults to the canonical Nix system name detected by - <filename>configure</filename> at build time.</para></listitem> + <varlistentry xml:id="conf-plugin-files"> + <term><literal>plugin-files</literal></term> + <listitem> + <para> + A list of plugin files to be loaded by Nix. Each of these + files will be dlopened by Nix, allowing them to affect + execution through static initialization. In particular, these + plugins may construct static instances of RegisterPrimOp to + add new primops or constants to the expression language, + RegisterStoreImplementation to add new store implementations, + RegisterCommand to add new subcommands to the + <literal>nix</literal> command, and RegisterSetting to add new + nix config settings. See the constructors for those types for + more details. + </para> + <para> + Since these files are loaded into the same address space as + Nix itself, they must be DSOs compatible with the instance of + Nix running at the time (i.e. compiled against the same + headers, not linked to any incompatible libraries). They + should not be linked to any Nix libs directly, as those will + be available already at load time. + </para> + <para> + If an entry in the list is a directory, all files in the + directory are loaded as plugins (non-recursively). + </para> + </listitem> </varlistentry> - <varlistentry><term><literal>fsync-metadata</literal></term> - - <listitem><para>If set to <literal>true</literal>, changes to the - Nix store metadata (in <filename>/nix/var/nix/db</filename>) are - synchronously flushed to disk. This improves robustness in case - of system crashes, but reduces performance. The default is - <literal>true</literal>.</para></listitem> - - </varlistentry> + <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term> + <listitem> - <varlistentry><term><literal>auto-optimise-store</literal></term> - <listitem><para>If set to <literal>true</literal>, Nix - automatically detects files in the store that have identical - contents, and replaces them with hard links to a single copy. - This saves disk space. If set to <literal>false</literal> (the - default), you can still run <command>nix-store - --optimise</command> to get rid of duplicate - files.</para></listitem> + <para>If set, the path to a program that can set extra + derivation-specific settings for this system. This is used for settings + that can't be captured by the derivation model itself and are too variable + between different versions of the same system to be hard-coded into nix. + </para> - </varlistentry> + <para>The hook is passed the derivation path and, if sandboxes are enabled, + the sandbox directory. It can then modify the sandbox and send a series of + commands to modify various settings to stdout. The currently recognized + commands are:</para> + <variablelist> + <varlistentry xml:id="extra-sandbox-paths"> + <term><literal>extra-sandbox-paths</literal></term> - <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term> + <listitem> - <listitem> + <para>Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an empty + line. Entries have the same format as + <literal>sandbox-paths</literal>.</para> - <para>The timeout (in seconds) for establishing connections in - the binary cache substituter. It corresponds to - <command>curl</command>’s <option>--connect-timeout</option> - option.</para> + </listitem> + </varlistentry> + </variablelist> </listitem> </varlistentry> - <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term> - - <listitem> - - <para>A list of names of users (separated by whitespace) that - have additional rights when connecting to the Nix daemon, such - as the ability to specify additional binary caches, or to import - unsigned NARs. You can also specify groups by prefixing them - with <literal>@</literal>; for instance, - <literal>@wheel</literal> means all users in the - <literal>wheel</literal> group. The default is - <literal>root</literal>.</para> - - <warning><para>Adding a user to <option>trusted-users</option> - is essentially equivalent to giving that user root access to the - system. For example, the user can set - <option>sandbox-paths</option> and thereby obtain read access to - directories that are otherwise inacessible to - them.</para></warning> + <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term> - </listitem> + <listitem><para>How many times to repeat builds to check whether + they are deterministic. The default value is 0. If the value is + non-zero, every build is repeated the specified number of + times. If the contents of any of the runs differs from the + previous ones, the build is rejected and the resulting store paths + are not registered as “valid” in Nix’s database.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term> - - <listitem> - - <para>A list of names of users (separated by whitespace) that - are allowed to connect to the Nix daemon. As with the - <option>trusted-users</option> option, you can specify groups by - prefixing them with <literal>@</literal>. Also, you can allow - all users by specifying <literal>*</literal>. The default is - <literal>*</literal>.</para> - - <para>Note that trusted users are always allowed to connect.</para> + <varlistentry><term><literal>require-sigs</literal></term> - </listitem> + <listitem><para>If set to <literal>true</literal> (the default), + any non-content-addressed path added or copied to the Nix store + (e.g. when substituting from a binary cache) must have a valid + signature, that is, be signed using one of the keys listed in + <option>trusted-public-keys</option> or + <option>secret-key-files</option>. Set to <literal>false</literal> + to disable signature checking.</para></listitem> </varlistentry> @@ -603,141 +573,202 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term> + <varlistentry><term><literal>sandbox</literal></term> - <listitem> + <listitem><para>If set to <literal>true</literal>, builds will be + performed in a <emphasis>sandboxed environment</emphasis>, i.e., + they’re isolated from the normal file system hierarchy and will + only see their dependencies in the Nix store, the temporary build + directory, private versions of <filename>/proc</filename>, + <filename>/dev</filename>, <filename>/dev/shm</filename> and + <filename>/dev/pts</filename> (on Linux), and the paths configured with the + <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal> + option</link>. This is useful to prevent undeclared dependencies + on files in directories such as <filename>/usr/bin</filename>. In + addition, on Linux, builds run in private PID, mount, network, IPC + and UTS namespaces to isolate them from other processes in the + system (except that fixed-output derivations do not run in private + network namespace to ensure they can access the network).</para> - <para>A list of URI prefixes to which access is allowed in - restricted evaluation mode. For example, when set to - <literal>https://github.com/NixOS</literal>, builtin functions - such as <function>fetchGit</function> are allowed to access - <literal>https://github.com/NixOS/patchelf.git</literal>.</para> + <para>Currently, sandboxing only work on Linux and macOS. The use + of a sandbox requires that Nix is run as root (so you should use + the <link linkend='conf-build-users-group'>“build users” + feature</link> to perform the actual builds under different users + than root).</para> + + <para>If this option is set to <literal>relaxed</literal>, then + fixed-output derivations and derivations that have the + <varname>__noChroot</varname> attribute set to + <literal>true</literal> do not run in sandboxes.</para> + + <para>The default is <literal>false</literal>.</para> </listitem> </varlistentry> - <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term> + <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term> - <listitem> + <listitem><para>This option determines the maximum size of the + <literal>tmpfs</literal> filesystem mounted on + <filename>/dev/shm</filename> in Linux sandboxes. For the format, + see the description of the <option>size</option> option of + <literal>tmpfs</literal> in + <citerefentry><refentrytitle>mount</refentrytitle><manvolnum>8</manvolnum></citerefentry>. The + default is <literal>50%</literal>.</para></listitem> + </varlistentry> - <para>If set, the path to a program that can set extra - derivation-specific settings for this system. This is used for settings - that can't be captured by the derivation model itself and are too variable - between different versions of the same system to be hard-coded into nix. - </para> - <para>The hook is passed the derivation path and, if sandboxes are enabled, - the sandbox directory. It can then modify the sandbox and send a series of - commands to modify various settings to stdout. The currently recognized - commands are:</para> + <varlistentry xml:id="conf-sandbox-paths"> + <term><literal>sandbox-paths</literal></term> - <variablelist> - <varlistentry xml:id="extra-sandbox-paths"> - <term><literal>extra-sandbox-paths</literal></term> + <listitem><para>A list of paths bind-mounted into Nix sandbox + environments. You can use the syntax + <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal> + to mount a path in a different location in the sandbox; for + instance, <literal>/bin=/nix-bin</literal> will mount the path + <literal>/nix-bin</literal> as <literal>/bin</literal> inside the + sandbox. If <replaceable>source</replaceable> is followed by + <literal>?</literal>, then it is not an error if + <replaceable>source</replaceable> does not exist; for example, + <literal>/dev/nvidiactl?</literal> specifies that + <filename>/dev/nvidiactl</filename> will only be mounted in the + sandbox if it exists in the host filesystem.</para> - <listitem> + <para>Depending on how Nix was built, the default value for this option + may be empty or provide <filename>/bin/sh</filename> as a + bind-mount of <command>bash</command>.</para></listitem> - <para>Pass a list of files and directories to be included in the - sandbox for this build. One entry per line, terminated by an empty - line. Entries have the same format as - <literal>sandbox-paths</literal>.</para> + </varlistentry> - </listitem> - </varlistentry> - </variablelist> - </listitem> + <varlistentry><term><literal>secret-key-files</literal></term> + + <listitem><para>A whitespace-separated list of files containing + secret (private) keys. These are used to sign locally-built + paths. They can be generated using <command>nix-store + --generate-binary-cache-key</command>. The corresponding public + key can be distributed to other users, who can add it to + <option>trusted-public-keys</option> in their + <filename>nix.conf</filename>.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term> + <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term> - <listitem><para>How many times to repeat builds to check whether - they are deterministic. The default value is 0. If the value is - non-zero, every build is repeated the specified number of - times. If the contents of any of the runs differs from the - previous ones, the build is rejected and the resulting store paths - are not registered as “valid” in Nix’s database.</para></listitem> + <listitem><para>Causes Nix to print out a stack trace in case of Nix + expression evaluation errors.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term> + <varlistentry><term><literal>substitute</literal></term> - <listitem><para>This option determines the maximum size of the - <literal>tmpfs</literal> filesystem mounted on - <filename>/dev/shm</filename> in Linux sandboxes. For the format, - see the description of the <option>size</option> option of - <literal>tmpfs</literal> in - <citerefentry><refentrytitle>mount</refentrytitle><manvolnum>8</manvolnum></citerefentry>. The - default is <literal>50%</literal>.</para></listitem> + <listitem><para>If set to <literal>true</literal> (default), Nix + will use binary substitutes if available. This option can be + disabled to force building from source.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term> + <varlistentry><term><literal>substituters</literal></term> - <listitem><para>By default, Nix allows you to <function>import</function> from a derivation, - allowing building at evaluation time. With this option set to false, Nix will throw an error - when evaluating an expression that uses this feature, allowing users to ensure their evaluation - will not require any builds to take place.</para></listitem> + <listitem><para>A list of URLs of substituters, separated by + whitespace. The default is + <literal>https://cache.nixos.org</literal>.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term> + <varlistentry><term><literal>system</literal></term> - <listitem><para>(Linux-specific.) By default, builders on Linux - cannot acquire new privileges by calling setuid/setgid programs or - programs that have file capabilities. For example, programs such - as <command>sudo</command> or <command>ping</command> will - fail. (Note that in sandbox builds, no such programs are available - unless you bind-mount them into the sandbox via the - <option>sandbox-paths</option> option.) You can allow the - use of such programs by enabling this option. This is impure and - usually undesirable, but may be useful in certain scenarios - (e.g. to spin up containers or set up userspace network interfaces - in tests).</para></listitem> + <listitem><para>This option specifies the canonical Nix system + name of the current installation, such as + <literal>i686-linux</literal> or + <literal>x86_64-darwin</literal>. Nix can only build derivations + whose <literal>system</literal> attribute equals the value + specified here. In general, it never makes sense to modify this + value from its default, since you can use it to ‘lie’ about the + platform you are building on (e.g., perform a Mac OS build on a + Linux machine; the result would obviously be wrong). It only + makes sense if the Nix binaries can run on multiple platforms, + e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and + <literal>i686-linux</literal>.</para> + + <para>It defaults to the canonical Nix system name detected by + <filename>configure</filename> at build time.</para></listitem> </varlistentry> - <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term> + <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term> - <listitem><para>A list of web servers used by - <function>builtins.fetchurl</function> to obtain files by - hash. The default is - <literal>http://tarballs.nixos.org/</literal>. Given a hash type - <replaceable>ht</replaceable> and a base-16 hash - <replaceable>h</replaceable>, Nix will try to download the file - from - <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>. - This allows files to be downloaded even if they have disappeared - from their original URI. For example, given the default mirror - <literal>http://tarballs.nixos.org/</literal>, when building the derivation + <listitem> -<programlisting> -builtins.fetchurl { - url = https://example.org/foo-1.2.3.tar.xz; - sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; -} -</programlisting> + <para>This option defines the maximum number of seconds that a + builder can run. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite loop + but keep writing to their standard output or standard error. It + can be overridden using the <option + linkend="opt-timeout">--timeout</option> command line + switch.</para> - Nix will attempt to download this file from - <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal> - first. If it is not available there, if will try the original URI.</para></listitem> + <para>The value <literal>0</literal> means that there is no + timeout. This is also the default.</para> + + </listitem> </varlistentry> - <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term> + <varlistentry><term><literal>trusted-public-keys</literal></term> - <listitem><para>Causes Nix to print out a stack trace in case of Nix - expression evaluation errors.</para></listitem> + <listitem><para>A whitespace-separated list of public keys. When + paths are copied from another Nix store (such as a binary cache), + they must be signed with one of these keys. For example: + <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry><term><literal>trusted-substituters</literal></term> + + <listitem><para>A list of URLs of substituters, separated by + whitespace. These are not used by default, but can be enabled by + users of the Nix daemon by specifying <literal>--option + substituters <replaceable>urls</replaceable></literal> on the + command line. Unprivileged users are only allowed to pass a + subset of the URLs listed in <literal>substituters</literal> and + <literal>trusted-substituters</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term> + + <listitem> + + <para>A list of names of users (separated by whitespace) that + have additional rights when connecting to the Nix daemon, such + as the ability to specify additional binary caches, or to import + unsigned NARs. You can also specify groups by prefixing them + with <literal>@</literal>; for instance, + <literal>@wheel</literal> means all users in the + <literal>wheel</literal> group. The default is + <literal>root</literal>.</para> + + <warning><para>Adding a user to <option>trusted-users</option> + is essentially equivalent to giving that user root access to the + system. For example, the user can set + <option>sandbox-paths</option> and thereby obtain read access to + directories that are otherwise inacessible to + them.</para></warning> + + </listitem> </varlistentry> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 5a3a8645c1d9..8a32ed8b5c99 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -126,6 +126,17 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> + <varlistentry><term><function>builtins.splitVersion</function> + <replaceable>s</replaceable></term> + + <listitem><para>Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in <link linkend="ssec-version-comparisons"> + <command>nix-env -u</command></link>.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.concatLists</function> <replaceable>lists</replaceable></term> @@ -308,8 +319,9 @@ stdenv.mkDerivation { … } </varlistentry> - <varlistentry><term><function>builtins.filterSource</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-filterSource'> + <term><function>builtins.filterSource</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem> @@ -768,6 +780,75 @@ Evaluates to <literal>[ "foo" ]</literal>. </varlistentry> + <varlistentry> + <term> + <function>builtins.path</function> + <replaceable>args</replaceable> + </term> + + <listitem> + <para> + An enrichment of the built-in path type, based on the attributes + present in <replaceable>args</replaceable>. All are optional + except <varname>path</varname>: + </para> + + <variablelist> + <varlistentry> + <term>path</term> + <listitem> + <para>The underlying path.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>name</term> + <listitem> + <para> + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like <literal>@</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>filter</term> + <listitem> + <para> + A function of the type expected by + <link linkend="builtin-filterSource">builtins.filterSource</link>, + with the same semantics. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>recursive</term> + <listitem> + <para> + When <literal>false</literal>, when + <varname>path</varname> is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, <varname>path</varname> must refer to a + regular file, not a directory. This allows similar + behavior to <literal>fetchurl</literal>. Defaults to + <literal>true</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>sha256</term> + <listitem> + <para> + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + <literal>builtins.path</literal> to be used even when the + <literal>pure-eval</literal> nix config option is on. + </para> + </listitem> + </varlistentry> + </variablelist> + </listitem> + </varlistentry> <varlistentry><term><function>builtins.pathExists</function> <replaceable>path</replaceable></term> diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml index a13e3c89be78..69ae1ef27041 100644 --- a/doc/manual/installation/multi-user.xml +++ b/doc/manual/installation/multi-user.xml @@ -52,34 +52,6 @@ This creates 10 build users. There can never be more concurrent builds than the number of build users, so you may want to increase this if you expect to do many builds at the same time.</para> -<para>On macOS, you can create the required group and users by -running the following script: - -<programlisting> -#! /bin/bash -e - -dseditgroup -o create nixbld -q - -gid=$(dscl . -read /Groups/nixbld | awk '($1 == "PrimaryGroupID:") {print $2 }') - -echo "created nixbld group with gid $gid" - -for i in $(seq 1 10); do - user=/Users/nixbld$i - uid="$((30000 + $i))" - dscl . create $user - dscl . create $user RealName "Nix build user $i" - dscl . create $user PrimaryGroupID "$gid" - dscl . create $user UserShell /usr/bin/false - dscl . create $user NFSHomeDirectory /var/empty - dscl . create $user UniqueID "$uid" - dseditgroup -o edit -a nixbld$i -t user nixbld - echo "created nixbld$i user with uid $uid" -done -</programlisting> - -</para> - </simplesect> diff --git a/doc/manual/release-notes/rl-2.0.xml b/doc/manual/release-notes/rl-2.0.xml index 32cdb1d0cefc..0d5296cc9007 100644 --- a/doc/manual/release-notes/rl-2.0.xml +++ b/doc/manual/release-notes/rl-2.0.xml @@ -6,165 +6,572 @@ <title>Release 2.0 (2018-02-??)</title> -<para>This release has the following new features:</para> +<para>The following incompatible changes have been made:</para> <itemizedlist> <listitem> - <para>Start of new <command>nix</command> command line - interface. This is a work in progress and the interface is subject - to change.</para> + <para>The manifest-based substituter mechanism + (<command>download-using-manifests</command>) has been <link + xlink:href="https://github.com/NixOS/nix/commit/867967265b80946dfe1db72d40324b4f9af988ed">removed</link>. It + has been superseded by the binary cache substituter mechanism + since several years. As a result, the following programs have been + removed: <itemizedlist> + <listitem><para><command>nix-pull</command></para></listitem> + <listitem><para><command>nix-generate-patches</command></para></listitem> + <listitem><para><command>bsdiff</command></para></listitem> + <listitem><para><command>bspatch</command></para></listitem> + </itemizedlist> + </para> + </listitem> - <listitem><para>Self-documenting: <option>--help</option> shows - all available command-line arguments.</para></listitem> - - <listitem><para><option>--help-config</option> shows all - configuration options.</para></listitem> + <listitem> + <para>The “copy from other stores” substituter mechanism + (<command>copy-from-other-stores</command> and the + <envar>NIX_OTHER_STORES</envar> environment variable) has been + removed. It was primarily used by the NixOS installer to copy + available paths from the installation medium. The replacement is + to use a chroot store as a substituter + (e.g. <literal>--substituters /mnt</literal>), or to build into a + chroot store (e.g. <literal>--store /mnt --substituter /</literal>).</para> + </listitem> - <listitem><para><command>nix build</command>: Replacement for - <command>nix-build</command>.</para></listitem> + <listitem> + <para>The command <command>nix-push</command> has been removed as + part of the effort to eliminate Nix's dependency on Perl. You can + use <command>nix copy</command> instead, e.g. <literal>nix copy + --to /tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para> + </listitem> - <listitem><para><command>nix ls-store</command> and <command>nix - ls-nar</command> allow listing the contents of a store path or - NAR file.</para></listitem> + <listitem> + <para>The “nested” log output feature (<option>--log-type + pretty</option>) has been removed. As a result, + <command>nix-log2xml</command> was also removed.</para> + </listitem> - <listitem><para><command>nix cat-store</command> and - <command>nix cat-nar</command> allow extracting a file from a - store path or NAR file.</para></listitem> + <listitem> + <para>OpenSSL-based signing has been <link + xlink:href="https://github.com/NixOS/nix/commit/f435f8247553656774dd1b2c88e9de5d59cab203">removed</link>. This + feature was never well-supported. A better alternative is provided + by the <option>secret-key-files</option> and + <option>trusted-public-keys</option> options.</para> + </listitem> - <listitem><para><command>nix verify</command> checks whether a - store path is unmodified and/or is trusted.</para></listitem> + <listitem> + <para>Failed build caching has been <link + xlink:href="https://github.com/NixOS/nix/commit/8cffec84859cec8b610a2a22ab0c4d462a9351ff">removed</link>. This + feature was introduced to support the Hydra continuous build + system, but Hydra no longer uses it.</para> + </listitem> - <listitem><para><command>nix copy-sigs</command> copies - signatures from one store to another.</para></listitem> + <listitem> + <para><filename>nix-mode.el</filename> has been removed from + Nix. It is now <link + xlink:href="https://github.com/NixOS/nix-mode">a separate + repository</link> and can be installed through the MELPA package + repository.</para> + </listitem> - <listitem><para><command>nix sign-paths</command> signs store - paths.</para></listitem> +</itemizedlist> - <listitem><para><command>nix copy</command> copies paths between - arbitrary Nix stores, generalising - <command>nix-copy-closure</command> and - <command>nix-push</command>.</para></listitem> +<para>This release has the following new features:</para> - <listitem><para><command>nix path-info</command> shows - information about store paths.</para></listitem> +<itemizedlist> - <listitem><para><command>nix run</command> starts a shell in - which the specified packages are available.</para></listitem> + <listitem> + <para>It introduces a new command named <command>nix</command>, + which is intended to eventually replace all + <command>nix-*</command> commands with a more consistent and + better designed user interface. It currently provides replacements + for some (but not all) of the functionality provided by + <command>nix-store</command>, <command>nix-build</command>, + <command>nix-shell -p</command>, <command>nix-env -qa</command>, + <command>nix-instantiate --eval</command>, + <command>nix-push</command> and + <command>nix-copy-closure</command>. It has the following major + features:</para> - <listitem><para><command>nix log</command> shows the build log - of a package or path. If the build log is not available locally, - it will try to obtain it from a binary cache.</para></listitem> + <itemizedlist> - <listitem><para><command>nix eval</command> replaces - <command>nix-instantiate --eval</command>.</para></listitem> + <listitem> + <para>Unlike the legacy commands, it has a consistent way to + refer to packages and package-like arguments (like store + paths). For example, the following commands all copy the GNU + Hello package to a remote machine: + + <screen>nix copy --to ssh://machine nixpkgs.hello</screen> + <screen>nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen> + <screen>nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)'</screen> + + By contrast, <command>nix-copy-closure</command> only accepted + store paths as arguments.</para> + </listitem> + + <listitem> + <para>It is self-documenting: <option>--help</option> shows + all available command-line arguments. If + <option>--help</option> is given after a subcommand, it shows + examples for that subcommand. <command>nix + --help-config</command> shows all configuration + options.</para> + </listitem> + + <listitem> + <para>It is much less verbose. By default, it displays a + single-line progress indicator that shows how many packages + are left to be built or downloaded, and (if there are running + builds) the most recent line of builder output. If a build + fails, it shows the last few lines of builder output. The full + build log can be retrieved using <command>nix + log</command>.</para> + </listitem> + + <listitem> + <para>It <link + xlink:href="https://github.com/NixOS/nix/commit/b8283773bd64d7da6859ed520ee19867742a03ba">provides</link> + all <filename>nix.conf</filename> configuration options as + command line flags. For example, instead of <literal>--option + http-connections 100</literal> you can write + <literal>--http-connections 100</literal>. Boolean options can + be written as + <literal>--<replaceable>foo</replaceable></literal> or + <literal>--no-<replaceable>foo</replaceable></literal> + (e.g. <option>--no-auto-optimise-store</option>).</para> + </listitem> + + <listitem> + <para>Many subcommands have a <option>--json</option> flag to + write results to stdout in JSON format.</para> + </listitem> - <listitem><para><command>nix dump-path</command> to get a NAR - from a store path.</para></listitem> + </itemizedlist> - <listitem><para><command>nix edit</command> opens the source - code of a package in an editor.</para></listitem> + <warning><para>Please note that the <command>nix</command> command + is a work in progress and the interface is subject to + change.</para></warning> - <listitem><para><command>nix search</command> replaces - <command>nix-env -qa</command>. It searches the available - packages for occurences of a search string in the attribute - name, package name or description. It caches available packages - to speed up searches.</para></listitem> + <para>It provides the following high-level (“porcelain”) + subcommands:</para> - <listitem><para><command>nix why-depends</command> (d41c5eb13f4f3a37d80dbc6d3888644170c3b44a).</para></listitem> + <itemizedlist> - <listitem><para><command>nix show-derivation</command> (e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2).</para></listitem> + <listitem> + <para><command>nix build</command> is a replacement for + <command>nix-build</command>.</para> + </listitem> + + <listitem> + <para><command>nix run</command> executes a command in an + environment in which the specified packages are available. It + is (roughly) a replacement for <command>nix-shell + -p</command>. Unlike that command, it does not execute the + command in a shell, and has a flag (<command>-c</command>) + that specifies the unquoted command line to be + executed.</para> + + <para>It is particularly useful in conjunction with chroot + stores, allowing Linux users who do not have permission to + install Nix in <command>/nix/store</command> to still use + binary substitutes that assume + <command>/nix/store</command>. For example, + + <screen>nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!'</screen> + + downloads (or if not substitutes are available, builds) the + GNU Hello package into + <filename>~/my-nix/nix/store</filename>, then runs + <command>hello</command> in a mount namespace where + <filename>~/my-nix/nix/store</filename> is mounted onto + <command>/nix/store</command>.</para> + </listitem> + + <listitem> + <para><command>nix search</command> replaces <command>nix-env + -qa</command>. It searches the available packages for + occurrences of a search string in the attribute name, package + name or description. Unlike <command>nix-env -qa</command>, it + has a cache to speed up subsequent searches.</para> + </listitem> + + <listitem> + <para><command>nix copy</command> copies paths between + arbitrary Nix stores, generalising + <command>nix-copy-closure</command> and + <command>nix-push</command>.</para> + </listitem> + + <listitem> + <para><command>nix repl</command> replaces the external + program <command>nix-repl</command>. It provides an + interactive environment for evaluating and building Nix + expressions. Note that it uses <literal>linenoise-ng</literal> + instead of GNU Readline.</para> + </listitem> + + <listitem> + <para><command>nix upgrade-nix</command> upgrades Nix to the + latest stable version. This requires that Nix is installed in + a profile. (Thus it won’t work on NixOS, or if it’s installed + outside of the Nix store.)</para> + </listitem> + + <listitem> + <para><command>nix verify</command> checks whether store paths + are unmodified and/or “trusted” (see below). It replaces + <command>nix-store --verify</command> and <command>nix-store + --verify-path</command>.</para> + </listitem> + + <listitem> + <para><command>nix log</command> shows the build log of a + package or path. If the build log is not available locally, it + will try to obtain it from the configured substituters (such + as <uri>cache.nixos.org</uri>, which now provides build + logs).</para> + </listitem> + + <listitem> + <para><command>nix edit</command> opens the source code of a + package in your editor.</para> + </listitem> + + <listitem> + <para><command>nix eval</command> replaces + <command>nix-instantiate --eval</command>.</para> + </listitem> + + <listitem> + <para><command + xlink:href="https://github.com/NixOS/nix/commit/d41c5eb13f4f3a37d80dbc6d3888644170c3b44a">nix + why-depends</command> shows why one store path has another in + its closure. This is primarily useful to finding the causes of + closure bloat. For example, + + <screen>nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev</screen> + + shows a chain of files and fragments of file contents that + cause the VLC package to have the “dev” output of + <literal>libdrm</literal> in its closure — an undesirable + situation.</para> + </listitem> + + <listitem> + <para><command>nix path-info</command> shows information about + store paths, replacing <command>nix-store -q</command>. A + useful feature is the option <option>--closure-size</option> + (<option>-S</option>). For example, the following command show + the closure sizes of every path in the current NixOS system + closure, sorted by size: + + <screen>nix path-info -rS /run/current-system | sort -nk2</screen> + + </para> + </listitem> + + <listitem> + <para><command>nix optimise-store</command> replaces + <command>nix-store --optimise</command>. The main difference + is that it has a progress indicator.</para> + </listitem> - <listitem><para><command>nix add-to-store</command> (970366266b8df712f5f9cedb45af183ef5a8357f).</para></listitem> + </itemizedlist> - <listitem><para><command>nix upgrade-nix</command> upgrades Nix - to the latest stable version. This requires that Nix is - installed in a profile. (Thus it won’t work on NixOS, or if it’s - installed outside of the Nix store.)</para></listitem> + <para>A number of low-level (“plumbing”) commands are also + available:</para> - <listitem><para>Progress indicator.</para></listitem> + <itemizedlist> - <listitem><para>All options are available as flags now - (b8283773bd64d7da6859ed520ee19867742a03ba).</para></listitem> + <listitem> + <para><command>nix ls-store</command> and <command>nix + ls-nar</command> list the contents of a store path or NAR + file. The former is primarily useful in conjunction with + remote stores, e.g. + + <screen>nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen> + + lists the contents of path in a binary cache.</para> + </listitem> + + <listitem> + <para><command>nix cat-store</command> and <command>nix + cat-nar</command> allow extracting a file from a store path or + NAR file.</para> + </listitem> + + <listitem> + <para><command>nix dump-path</command> writes the contents of + a store path to stdout in NAR format. This replaces + <command>nix-store --dump</command>.</para> + </listitem> + + <listitem> + <para><command + xlink:href="https://github.com/NixOS/nix/commit/e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2">nix + show-derivation</command> displays a store derivation in JSON + format. This is an alternative to + <command>pp-aterm</command>.</para> + </listitem> + + <listitem> + <para><command + xlink:href="https://github.com/NixOS/nix/commit/970366266b8df712f5f9cedb45af183ef5a8357f">nix + add-to-store</command> replaces <command>nix-store + --add</command>.</para> + </listitem> + + <listitem> + <para><command>nix sign-paths</command> signs store + paths. (TODO: add examples)</para> + </listitem> + + <listitem> + <para><command>nix copy-sigs</command> copies signatures from + one store to another. (TODO: add examples and + tests)</para> + </listitem> + + <listitem> + <para><command>nix show-config</command> shows all + configuration options and their current values.</para> + </listitem> </itemizedlist> </listitem> <listitem> - <para>The external program <command>nix-repl</command> has been - integrated into Nix as <command>nix repl</command>.</para> - </listitem> + <para>The store abstraction that Nix has had for a long time to + support store access via the Nix daemon has been extended + significantly. In particular, substituters (which used to be + external programs such as + <command>download-from-binary-cache</command>) are now subclasses + of the abstract <classname>Store</classname> class. This allows + many Nix commands to operate on such store types. For example, + <command>nix path-info</command> shows information about paths in + your local Nix store, while <command>nix path-info --store + https://cache.nixos.org/</command> shows information about paths + in the specified binary cache. Similarly, + <command>nix-copy-closure</command>, <command>nix-push</command> + and substitution are all instances of the general notion of + copying paths between different kinds of Nix stores.</para> - <listitem> - <para>If a fixed-output derivation produces a result with an - incorrect hash, the output path will be moved to the location - corresponding to the actual hash and registered as valid. Thus, a - subsequent build of the fixed-output derivation with the correct - hash is unnecessary.</para> - </listitem> + <para>Stores are specified using an URI-like syntax, + e.g. <uri>https://cache.nixos.org/</uri> or + <uri>ssh://machine</uri>. The following store types are supported: + + <itemizedlist> + + <listitem> + + <para><classname>LocalStore</classname> (stori URI + <literal>local</literal> or an absolute path) and the misnamed + <classname>RemoteStore</classname> (<literal>daemon</literal>) + provide access to a local Nix store, the latter via the Nix + daemon. You can use <literal>auto</literal> or the empty + string to auto-select a local or daemon store depending on + whether you have write permission to the Nix store. It is no + longer necessary to set the <envar>NIX_REMOTE</envar> + environment variable to use the Nix daemon.</para> + + <para>As noted above, <classname>LocalStore</classname> now + supports chroot builds, allowing the “physical” location of + the Nix store + (e.g. <filename>/home/alice/nix/store</filename>) to differ + from its “logical” location (typically + <filename>/nix/store</filename>). This allows non-root users + to use Nix while still getting the benefits from prebuilt + binaries from <uri>cache.nixos.org</uri>.</para> + + </listitem> + + <listitem> + + <para><classname>BinaryCacheStore</classname> is the abstract + superclass of all binary cache stores. It supports writing + build logs and NAR content listings in JSON format.</para> + + </listitem> + + <listitem> + + <para><classname>HttpBinaryCacheStore</classname> + (<literal>http://</literal>, <literal>https://</literal>) + supports binary caches via HTTP or HTTPS. If the server + supports <literal>PUT</literal> requests, it supports + uploading store paths via commands such as <command>nix + copy</command>.</para> + + </listitem> + + <listitem> + + <para><classname>LocalBinaryCacheStore</classname> + (<literal>file://</literal>) supports binary caches in the + local filesystem.</para> + + </listitem> + + <listitem> + + <para><classname>S3BinaryCacheStore</classname> + (<literal>s3://</literal>) supports binary caches stored in + Amazon S3, if enabled at compile time.</para> + + </listitem> + + <listitem> + + <para><classname>LegacySSHStore</classname> (<literal>ssh://</literal>) + is used to implement remote builds and + <command>nix-copy-closure</command>.</para> + + </listitem> + + <listitem> + + <para><classname>SSHStore</classname> + (<literal>ssh-ng://</literal>) supports arbitrary Nix + operations on a remote machine via the same protocol used by + <command>nix-daemon</command>.</para> + + </listitem> + + </itemizedlist> + + </para> - <listitem> - <para>It is no longer necessary to set the - <envar>NIX_REMOTE</envar> environment variable if you need to use - the Nix daemon. Nix will use the daemon automatically if you don’t - have write access to the Nix database.</para> </listitem> <listitem> - <para>The Nix language now supports floating point numbers. They are - based on regular C++ <literal>float</literal> and compatible with - existing integers and number-related operations. Export and import to and - from JSON and XML works, too.</para> + + <para>Security has been improved in various ways: + + <itemizedlist> + + <listitem> + <para>Nix now stores signatures for local store + paths. When paths are copied between stores (e.g., copied from + a binary cache to a local store), signatures are + propagated.</para> + + <para>Locally-built paths are signed automatically using the + secret keys specified by the <option>secret-key-files</option> + store option. Secret/public key pairs can be generated using + <command>nix-store + --generate-binary-cache-key</command>. (TODO: rename)</para> + + <para>In addition, locally-built store paths are marked as + “ultimately trusted”, but this bit is not propagated when + paths are copied between stores.</para> + </listitem> + + <listitem> + <para>Content-addressable store paths no longer require + signatures — they can be imported into a store by unprivileged + users even if they lack signatures.</para> + </listitem> + + <listitem> + <para>The command <command>nix verify</command> checks whether + the specified paths are trusted, i.e., have a certain number + of trusted signatures, are ultimately trusted, or are + content-addressed.</para> + </listitem> + + <listitem> + <para>Substitutions from binary caches <link + xlink:href="https://github.com/NixOS/nix/commit/ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b">now</link> + require signatures by default. This was already the case on + NixOS.</para> + </listitem> + + <listitem> + <para>In Linux sandbox builds, we <link + xlink:href="https://github.com/NixOS/nix/commit/eba840c8a13b465ace90172ff76a0db2899ab11b">now</link> + use <filename>/build</filename> instead of + <filename>/tmp</filename> as the temporary build + directory. This fixes potential security problems when a build + accidentally stores its <envar>TMPDIR</envar> in some + security-sensitive place, such as an RPATH.</para> + </listitem> + + </itemizedlist> + + </para> + </listitem> <listitem> - <para><command>nix-shell</command> now sets the - <varname>IN_NIX_SHELL</varname> environment variable during - evaluation and in the shell itself. This can be used to perform - different actions depending on whether you’re in a Nix shell or in - a regular build. Nixpkgs provides - <varname>lib.inNixShell</varname> to check this variable during - evaluation. (bb36a1a3cf3fbe6bc9d0afcc5fa0f928bed03170)</para> + <para><emphasis>Pure evaluation mode</emphasis>. This is a variant + of the existing restricted evaluation mode. In pure mode, the Nix + evaluator forbids access to anything that could cause different + evaluations of the same command line arguments to produce a + different result. This includes builtin functions such as + <function>builtins.getEnv</function>, but more importantly, + <emphasis>all</emphasis> filesystem or network access unless a + content hash or commit hash is specified. For example, calls to + <function>builtins.fetchGit</function> are only allowed if a + <varname>rev</varname> attribute is specified.</para> + + <para>The goal of this feature is to enable true reproducibility + and traceability of builds (including NixOS system configurations) + at the evaluation level. For example, in the future, + <command>nixos-rebuild</command> might build configurations from a + Nix expression in a Git repository in pure mode. That expression + might fetch other repositories such as Nixpkgs via + <function>builtins.fetchGit</function>. The commit hash of the + top-level repository then uniquely identifies a running system, + and, in conjunction with that repository, allows it to be + reproduced or modified.</para> + </listitem> <listitem> - <para>Internal: all <classname>Store</classname> classes are now - thread-safe. <classname>RemoteStore</classname> supports multiple - concurrent connections to the daemon. This is primarily useful in - multi-threaded programs such as - <command>hydra-queue-runner</command>.</para> + <para>There are several new features to support binary + reproducibility (i.e. to help ensure that multiple builds of the + same derivation produce exactly the same output). When + <option>enforce-determinism</option> is set to + <literal>false</literal>, it’s <link + xlink:href="https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051">no + longer</link> a fatal error if build rounds produce different + output. Also, a hook named <option>diff-hook</option> is <link + xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169w">provided</link> + to allow you to run tools such as <command>diffoscope</command> + when build rounds produce different output.</para> </listitem> <listitem> - <para>The dependency on Perl has been removed. As a result, some - (obsolete) programs have been removed: <command>nix-push</command> - (replaced by <command>nix copy</command>), - <command>nix-pull</command> (obsoleted by binary caches), - <command>nix-generate-patches</command>, - <command>bsdiff</command>, <command>bspatch</command>.</para> + <para>Configuring remote builds is a lot easier now. Provided you + are not using the Nix daemon, you can now just specify a remote + build machine on the command line, e.g. <literal>--option builders + 'ssh://my-mac x86_64-darwin'</literal>. The environment variable + <envar>NIX_BUILD_HOOK</envar> has been removed and is no longer + needed. The environment variable <envar>NIX_REMOTE_SYSTEMS</envar> + is still supported for compatibility, but it is also possible to + specify builders in <command>nix.conf</command> by setting the + option <literal>builders = + @<replaceable>path</replaceable></literal>.</para> </listitem> <listitem> - <para>Improved store abstraction. Substituters - eliminated. BinaryCacheStore, LocalBinaryCacheStore, - HttpBinaryCacheStore, S3BinaryCacheStore (compile-time - optional), SSHStore. Add docs + examples? - </para> + <para>If a fixed-output derivation produces a result with an + incorrect hash, the output path is moved to the location + corresponding to the actual hash and registered as valid. Thus, a + subsequent build of the fixed-output derivation with the correct + hash is unnecessary.</para> </listitem> <listitem> - <para>Nix now stores signatures for local store - paths. Locally-built paths are now signed automatically using the - secret keys specified by the <option>secret-key-files</option> - store option.</para> - - <para>In addition, store paths that have been built locally are - marked as “ultimately trusted”, and content-addressable store - paths carry a “content-addressability assertion” that allow them - to be trusted without any signatures.</para> + <para><command>nix-shell</command> <link + xlink:href="https://github.com/NixOS/nix/commit/ea59f39326c8e9dc42dfed4bcbf597fbce58797c">now</link> + sets the <varname>IN_NIX_SHELL</varname> environment variable + during evaluation and in the shell itself. This can be used to + perform different actions depending on whether you’re in a Nix + shell or in a regular build. Nixpkgs provides + <varname>lib.inNixShell</varname> to check this variable during + evaluation.</para> </listitem> <listitem> @@ -179,7 +586,8 @@ <uri>https://nixos.org/channels/<replaceable>channel-name</replaceable>/nixexprs.tar.xz</uri>. For example, <literal>nix-build channel:nixos-15.09 -A hello</literal> will build the GNU Hello package from the - <literal>nixos-15.09</literal> channel.</para> + <literal>nixos-15.09</literal> channel. In the future, this may + use Git to fetch updates more efficiently.</para> </listitem> <listitem> @@ -189,45 +597,119 @@ </listitem> <listitem> - <para><function>builtins.fetchGit</function>. - (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para> + <para>Networking has been improved: + + <itemizedlist> + + <listitem> + <para>HTTP/2 is now supported. This makes binary cache lookups + <link + xlink:href="https://github.com/NixOS/nix/commit/90ad02bf626b885a5dd8967894e2eafc953bdf92">much + more efficient</link>.</para> + </listitem> + + <listitem> + <para>We now retry downloads on many HTTP errors, making + binary caches substituters more resilient to temporary + failures.</para> + </listitem> + + <listitem> + <para>HTTP credentials can now be configured via the standard + <filename>netrc</filename> mechanism.</para> + </listitem> + + <listitem> + <para>If S3 support is enabled at compile time, + <uri>s3://</uri> URIs are <link + xlink:href="https://github.com/NixOS/nix/commit/9ff9c3f2f80ba4108e9c945bbfda2c64735f987b">supported</link> + in all places where Nix allows URIs.</para> + </listitem> + + <listitem> + <para>Brotli compression is now supported. In particular, + <uri>cache.nixos.org</uri> build logs are now compressed using + Brotli.</para> + </listitem> + + </itemizedlist> + + </para> + </listitem> <listitem> - <para><literal><nix/fetchurl.nix></literal> now uses the - content-addressable tarball cache at - <uri>http://tarballs.nixos.org/</uri>, just like - <function>fetchurl</function> in - Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para> + <para><command>nix-env</command> <link + xlink:href="https://github.com/NixOS/nix/commit/b0cb11722626e906a73f10dd9a0c9eea29faf43a">now</link> + ignores packages with bad derivation names (in particular those + starting with a digit or containing a dot).</para> </listitem> <listitem> - <para>Chroot Nix stores: allow the “physical” location of the Nix - store (e.g. <filename>/home/alice/nix/store</filename>) to differ - from its “logical” location (typically - <filename>/nix/store</filename>). This allows non-root users to - use Nix while still getting the benefits from prebuilt binaries - from - <uri>cache.nixos.org</uri>. (4494000e04122f24558e1436e66d20d89028b4bd, - 3eb621750848e0e6b30e5a79f76afbb096bb6c8a)</para> + <para>Many configuration options have been renamed, either because + they were unnecessarily verbose + (e.g. <option>build-use-sandbox</option> is now just + <option>sandbox</option>) or to reflect generalised behaviour + (e.g. <option>binary-caches</option> is now + <option>substituters</option> because it allows arbitrary store + URIs). The old names are still supported for compatibility.</para> </listitem> <listitem> - <para>On Linux, builds are now executed in a user - namespace with uid 1000 and gid 100.</para> + <para>The <option>max-jobs</option> option can <link + xlink:href="https://github.com/NixOS/nix/commit/7251d048fa812d2551b7003bc9f13a8f5d4c95a5">now</link> + be set to <literal>auto</literal> to use the number of CPUs in the + system.</para> </listitem> <listitem> - <para><function>builtins.fetchurl</function> and - <function>builtins.fetchTarball</function> now support - <varname>sha256</varname> and <varname>name</varname> - attributes.</para> + <para>Hashes can <link + xlink:href="https://github.com/NixOS/nix/commit/c0015e87af70f539f24d2aa2bc224a9d8b84276b">now</link> + be specified in base-64 format, in addition to base-16 and the + non-standard base-32.</para> + </listitem> + + <listitem> + <para><command>nix-shell</command> now uses + <varname>bashInteractive</varname> from Nixpkgs, rather than the + <command>bash</command> command that happens to be in the caller’s + <envar>PATH</envar>. This is especially important on macOS where + the <command>bash</command> provided by the system is seriously + outdated and cannot execute <literal>stdenv</literal>’s setup + script.</para> </listitem> <listitem> - <para><literal>HttpBinaryCacheStore</literal> (the replacement of - <command>download-from-binary-cache</command>) now retries - automatically on certain HTTP error codes.</para> + <para>Nix can now automatically trigger a garbage collection if + free disk space drops below a certain level during a build. This + is configured using the <option>min-free</option> and + <option>max-free</option> options.</para> + </listitem> + + <listitem> + <para><command>nix-store -q --roots</command> and + <command>nix-store --gc --print-roots</command> now show temporary + and in-memory roots.</para> + </listitem> + + <listitem> + <para> + Nix can now be extended with plugins. See the documentation of + the <option>plugin-files</option> option for more details. + </para> + </listitem> + +</itemizedlist> + +<para>The Nix language has the following new features: + +<itemizedlist> + + <listitem> + <para>It supports floating point numbers. They are based on the + C++ <literal>float</literal> type and are supported by the + existing numerical operators. Export and import to and from JSON + and XML works, too.</para> </listitem> <listitem> @@ -245,187 +727,288 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev" add docs.</para> </listitem> - <listitem> - <para>Support for HTTP/2. This makes binary cache lookups much - more efficient. (90ad02bf626b885a5dd8967894e2eafc953bdf92)</para> - </listitem> +</itemizedlist> - <listitem> - <para>The <option>build-sandbox-paths</option> configuration - option can now specify optional paths by appending a - <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will - bind-mount <varname>/dev/nvidiactl</varname> only if it - exists.</para> - </listitem> +</para> - <listitem> - <para>More support for testing build reproducibility: when - <option>enforce-determinism</option> is set to - <literal>false</literal>, it’s no longer a fatal error build - rounds produce different output - (8bdf83f936adae6f2c907a6d2541e80d4120f051); add a hook to run - diffoscope when build rounds produce different output - (9a313469a4bdea2d1e8df24d16289dc2a172a169w).</para> - </listitem> +<para>The following builtin functions are new or extended: - <listitem> - <para>Kill builds as soon as stdout/stderr is closed. This fixes a - bug that allowed builds to hang Nix indefinitely (regardless of - timeouts). (21948deed99a3295e4d5666e027a6ca42dc00b40)</para> - </listitem> +<itemizedlist> <listitem> - <para>Add support for passing structured data to builders. TODO: - document. (6de33a9c675b187437a2e1abbcb290981a89ecb1)</para> - </listitem> + <para><function + xlink:href="https://github.com/NixOS/nix/commit/38539b943a060d9cdfc24d6e5d997c0885b8aa2f">builtins.fetchGit</function> + allows Git repositories to be fetched at evaluation time. Thus it + differs from the <function>fetchgit</function> function in + Nixpkgs, which fetches at build time and cannot be used to fetch + Nix expressions during evaluation. A typical use case is to import + external NixOS modules from your configuration, e.g. - <listitem> - <para><varname>exportReferencesGraph</varname>: Export more - complete info in JSON - format. (c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a)</para> - </listitem> + <programlisting>imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ];</programlisting> - <listitem> - <para>Support for - netrc. (e6e74f987f0fa284d220432d426eb965269a97d6, - 302386f775eea309679654e5ea7c972fb6e7b9af)</para> + </para> </listitem> <listitem> - <para>Support <uri>s3://</uri> URIs in all places where Nix allows - URIs. (9ff9c3f2f80ba4108e9c945bbfda2c64735f987b)</para> + <para>Similarly, <function>builtins.fetchMercurial</function> + allows you to fetch Mercurial repositories.</para> </listitem> <listitem> - <para>The <option>build-max-jobs</option> option can be set to - <literal>auto</literal> to use the number of CPUs in the - system. (7251d048fa812d2551b7003bc9f13a8f5d4c95a5)</para> + <para><function>builtins.path</function> generalises + <function>builtins.filterSource</function> and path literals + (e.g. <literal>./foo</literal>). It allows specifying a store path + name that differs from the source path name + (e.g. <literal>builtins.path { path = ./foo; name = "bar"; + }</literal>) and also supports filtering out unwanted + files.</para> </listitem> <listitem> - <para>Add support for Brotli compression. - <uri>cache.nixos.org</uri> compresses build logs using - Brotli.</para> + <para><function>builtins.fetchurl</function> and + <function>builtins.fetchTarball</function> now support + <varname>sha256</varname> and <varname>name</varname> + attributes.</para> </listitem> <listitem> - <para>Substitutions from binary caches now require signatures by - default. This was already the case on - NixOS. (ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b)</para> + <para><function + xlink:href="https://github.com/NixOS/nix/commit/b8867a0239b1930a16f9ef3f7f3e864b01416dff">builtins.split</function> + splits a string using a POSIX extended regular expression as the + separator.</para> </listitem> <listitem> - <para><command>nix-env</command> now ignores packages with bad - derivation names (in particular those starting with a digit or - containing a - dot). (b0cb11722626e906a73f10dd9a0c9eea29faf43a)</para> + <para><function + xlink:href="https://github.com/NixOS/nix/commit/26d92017d3b36cff940dcb7d1611c42232edb81a">builtins.partition</function> + partitions the elements of a list into two lists, depending on a + Boolean predicate.</para> </listitem> <listitem> - <para>Renamed various configuration options. (TODO: in progress)</para> + <para><literal><nix/fetchurl.nix></literal> now uses the + content-addressable tarball cache at + <uri>http://tarballs.nixos.org/</uri>, just like + <function>fetchurl</function> in + Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para> </listitem> <listitem> - <para>Remote machines can now be specified on the command - line. TODO: - document. (1a68710d4dff609bbaf61db3e17a2573f0aadf17)</para> + <para>In restricted and pure evaluation mode, builtin functions + that download from the network (such as + <function>fetchGit</function>) are permitted to fetch underneath a + list of URI prefixes specified in the option + <option>allowed-uris</option>.</para> </listitem> - <listitem> - <para>In Linux sandbox builds, we now use - <filename>/build</filename> instead of <filename>/tmp</filename> - as the temporary build directory. This fixes potential security - problems when a build accidentally stores its - <envar>TMPDIR</envar> in some critical place, such as an - RPATH. (eba840c8a13b465ace90172ff76a0db2899ab11b)</para> - </listitem> +</itemizedlist> - <listitem> - <para>In Linux sandbox builds, we now provide a default - <filename>/bin/sh</filename> (namely <filename>ash</filename> from - BusyBox). (a2d92bb20e82a0957067ede60e91fab256948b41)</para> - </listitem> +</para> - <listitem> - <para>Make all configuration options available as command line - flags (b8283773bd64d7da6859ed520ee19867742a03ba).</para> - </listitem> +<para>The Nix build environment has the following changes: + +<itemizedlist> <listitem> - <para>Support base-64 - hashes. (c0015e87af70f539f24d2aa2bc224a9d8b84276b)</para> + <para>Values such as Booleans, integers, (nested) lists and + attribute sets can <link + xlink:href="https://github.com/NixOS/nix/commit/6de33a9c675b187437a2e1abbcb290981a89ecb1">now</link> + be passed to builders in a non-lossy way. If the special attribute + <varname>__structuredAttrs</varname> is set to + <literal>true</literal>, the other derivation attributes are + serialised in JSON format and made available to the builder via + the file <envar>.attrs.json</envar> in the builder’s temporary + directory. This obviates the need for + <varname>passAsFile</varname> since JSON files have no size + restrictions, unlike process environments.</para> + + <para><link + xlink:href="https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699">As + a convenience to Bash builders</link>, Nix writes a script named + <envar>.attrs.sh</envar> to the builder’s directory that + initialises shell variables corresponding to all attributes that + are representable in Bash. This includes non-nested (associative) + arrays. For example, the attribute <literal>hardening.format = + true</literal> ends up as the Bash associative array element + <literal>${hardening[format]}</literal>.</para> + </listitem> + + <listitem> + <para>Builders can <link + xlink:href="https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3">now</link> + communicate what build phase they are in by writing messages to + the file descriptor specified in <envar>NIX_LOG_FD</envar>. The + current phase is shown by the <command>nix</command> progress + indicator. + </para> </listitem> <listitem> - <para><command>nix-shell</command> now uses - <varname>bashInteractive</varname> from Nixpkgs, rather than the - <command>bash</command> command that happens to be in the caller’s - <envar>PATH</envar>. This is especially important on macOS where - the <command>bash</command> provided by the system is seriously - outdated and cannot execute <literal>stdenv</literal>’s setup - script.</para> + <para>In Linux sandbox builds, we <link + xlink:href="https://github.com/NixOS/nix/commit/a2d92bb20e82a0957067ede60e91fab256948b41">now</link> + provide a default <filename>/bin/sh</filename> (namely + <filename>ash</filename> from BusyBox).</para> </listitem> <listitem> - <para>New builtin functions: <function>builtins.split</function> - (b8867a0239b1930a16f9ef3f7f3e864b01416dff), - <function>builtins.partition</function>.</para> + <para>In structured attribute mode, + <varname>exportReferencesGraph</varname> <link + xlink:href="https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a">exports</link> + extended information about closures in JSON format. In particular, + it includes the sizes and hashes of paths. This is primarily + useful for NixOS image builders.</para> </listitem> <listitem> - <para>Automatic garbage collection.</para> + <para>Builds are <link + xlink:href="https://github.com/NixOS/nix/commit/21948deed99a3295e4d5666e027a6ca42dc00b40">now</link> + killed as soon as Nix receives EOF on the builder’s stdout or + stderr. This fixes a bug that allowed builds to hang Nix + indefinitely, regardless of + timeouts.</para> </listitem> <listitem> - <para><command>nix-store -q --roots</command> and - <command>nix-store --gc --print-roots</command> now show temporary - and in-memory roots.</para> + <para>The <option>sandbox-paths</option> configuration + option can now specify optional paths by appending a + <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will + bind-mount <varname>/dev/nvidiactl</varname> only if it + exists.</para> </listitem> <listitem> - <para>Builders can now communicate what build phase they are in by - writing messages to the file descriptor specified in - <envar>NIX_LOG_FD</envar>. (88e6bb76de5564b3217be9688677d1c89101b2a3) - </para> + <para>On Linux, builds are now executed in a user + namespace with UID 1000 and GID 100.</para> </listitem> </itemizedlist> -<para>Some features were removed:</para> +</para> -<itemizedlist> +<para>A number of significant internal changes were made: - <listitem> - <para>“Nested” log output. As a result, - <command>nix-log2xml</command> was also removed.</para> - </listitem> - - <listitem> - <para>OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para> - </listitem> - - <listitem> - <para>Caching of failed - builds. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para> - </listitem> +<itemizedlist> <listitem> - <para><filename>nix-mode.el</filename> has been removed from - Nix. It is now a separate repository in - <uri>https://github.com/NixOS/nix-mode</uri> and can be installed - through the MELPA package repository.</para> + <para>Nix no longer depends on Perl and all Perl components have + been rewritten in C++ or removed. The Perl bindings that used to + be part of Nix have been moved to a separate package, + <literal>nix-perl</literal>.</para> </listitem> <listitem> - <para>In restricted evaluation mode - (<option>--restrict-eval</option>), builtin functions that - download from the network (such as <function>fetchGit</function>) - are permitted to fetch underneath the list of URI prefixes - specified in the option <option>allowed-uris</option>.</para> + <para>All <classname>Store</classname> classes are now + thread-safe. <classname>RemoteStore</classname> supports multiple + concurrent connections to the daemon. This is primarily useful in + multi-threaded programs such as + <command>hydra-queue-runner</command>.</para> </listitem> </itemizedlist> -<para>This release has contributions from TBD.</para> +</para> + +<para>This release has contributions from + +Adrien Devresse, +Alexander Ried, +Alex Cruice, +Alexey Shmalko, +AmineChikhaoui, +Andy Wingo, +Aneesh Agrawal, +Anthony Cowley, +Armijn Hemel, +aszlig, +Ben Gamari, +Benjamin Hipple, +Benjamin Staffin, +Benno Fünfstück, +Bjørn Forsman, +Brian McKenna, +Charles Strahan, +Chase Adams, +Chris Martin, +Christian Theune, +Chris Warburton, +Daiderd Jordan, +Dan Connolly, +Daniel Peebles, +Dan Peebles, +davidak, +David McFarland, +Dmitry Kalinkin, +Domen Kožar, +Eelco Dolstra, +Emery Hemingway, +Eric Litak, +Eric Wolf, +Fabian Schmitthenner, +Frederik Rietdijk, +Gabriel Gonzalez, +Giorgio Gallo, +Graham Christensen, +Guillaume Maudoux, +Harmen, +Iavael, +James Broadhead, +James Earl Douglas, +Janus Troelsen, +Jeremy Shaw, +Joachim Schiele, +Joe Hermaszewski, +Joel Moberg, +Johannes 'fish' Ziemke, +Jörg Thalheim, +Jude Taylor, +kballou, +Keshav Kini, +Kjetil Orbekk, +Langston Barrett, +Linus Heckemann, +Ludovic Courtès, +Manav Rathi, +Marc Scholten, +Markus Hauck, +Matt Audesse, +Matthew Bauer, +Matthias Beyer, +Matthieu Coudron, +N1X, +Nathan Zadoks, +Neil Mayhew, +Nicolas B. Pierron, +Niklas Hambüchen, +Nikolay Amiantov, +Ole Jørgen Brønner, +Orivej Desh, +Peter Simons, +Peter Stuart, +Pyry Jahkola, +regnat, +Renzo Carbonara, +Rhys, +Robert Vollmert, +Scott Olson, +Scott R. Parish, +Sergei Trofimovich, +Shea Levy, +Sheena Artrip, +Spencer Baugh, +Stefan Junker, +Susan Potter, +Thomas Tuegel, +Timothy Allen, +Tristan Hume, +Tuomas Tynkkynen, +tv, +Tyson Whitehead, +Vladimír Čunát, +Will Dietz, +wmertens, +Wout Mertens, +zimbatm and +Zoran Plesivčak. +</para> </section> diff --git a/doc/manual/style.css b/doc/manual/style.css index 53fd9d5709c3..592583ab086a 100644 --- a/doc/manual/style.css +++ b/doc/manual/style.css @@ -96,7 +96,6 @@ div.example margin-right: 1.5em; background: #f4f4f8; border-radius: 0.4em; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; } div.example p.title @@ -106,7 +105,6 @@ div.example p.title div.example pre { - box-shadow: none; } @@ -116,15 +114,12 @@ div.example pre pre.screen, pre.programlisting { - border: 1px solid #b0b0b0; - padding: 3px 3px; + padding: 6px 6px; margin-left: 1.5em; margin-right: 1.5em; color: #600000; background: #f4f4f8; font-family: monospace; - border-radius: 0.4em; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; } div.example pre.programlisting @@ -149,7 +144,6 @@ div.example pre.programlisting padding: 0.3em 0.3em 0.3em 0.3em; background: #fffff5; border-radius: 0.4em; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; } div.note, div.warning @@ -256,16 +250,14 @@ span.command strong div.calloutlist table { - box-shadow: none; } table { border-collapse: collapse; - box-shadow: 0.4em 0.4em 0.5em #e0e0e0; } div.affiliation { font-style: italic; -} \ No newline at end of file +} diff --git a/mk/libraries.mk b/mk/libraries.mk index 3cd7a53107bd..14c95fa91cf6 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -45,6 +45,11 @@ endif # - $(1)_INSTALL_DIR: the directory where the library will be # installed. Defaults to $(libdir). # +# - $(1)_EXCLUDE_FROM_LIBRARY_LIST: if defined, the library will not +# be automatically marked as a dependency of the top-level all +# target andwill not be listed in the make help output. This is +# useful for libraries built solely for testing, for example. +# # - BUILD_SHARED_LIBS: if equal to ‘1’, a dynamic library will be # built, otherwise a static library. define build-library @@ -149,7 +154,9 @@ define build-library $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn))) -include $$($(1)_DEPS) + ifndef $(1)_EXCLUDE_FROM_LIBRARY_LIST libs-list += $$($(1)_PATH) + endif clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) dist-files += $$(_srcs) endef diff --git a/release-common.nix b/release-common.nix index de5af452c0de..f9df6fab9ff6 100644 --- a/release-common.nix +++ b/release-common.nix @@ -1,7 +1,9 @@ { pkgs }: rec { - sh = pkgs.busybox.override { + # Use "busybox-sandbox-shell" if present, + # if not (legacy) fallback and hope it's sufficient. + sh = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { useMusl = true; enableStatic = true; enableMinimal = true; @@ -23,7 +25,7 @@ rec { CONFIG_ASH_PRINTF y CONFIG_ASH_TEST y ''; - }; + }); configureFlags = [ "--disable-init-state" diff --git a/release.nix b/release.nix index bdac283cfe3b..3f8d5da4721e 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? fetchTarball channel:nixos-17.09 +, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "nix-2.0"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: @@ -127,7 +127,6 @@ let binaryTarball = pkgs.lib.genAttrs systems (system: - # FIXME: temporarily use a different branch for the Darwin build. with import nixpkgs { inherit system; }; let @@ -137,7 +136,7 @@ let runCommand "nix-binary-tarball-${version}" { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ]; - buildInputs = [ perl shellcheck ]; + buildInputs = [ perl ] ++ lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' @@ -150,8 +149,10 @@ let --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} - shellcheck -e SC1090 $TMPDIR/install - shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user + if type -p shellcheck; then + shellcheck -e SC1090 $TMPDIR/install + shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user + fi chmod +x $TMPDIR/install chmod +x $TMPDIR/install-darwin-multi-user @@ -224,11 +225,13 @@ let nix = build.x86_64-linux; system = "x86_64-linux"; }); - tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system: - import ./tests/setuid.nix rec { - inherit nixpkgs; - nix = build.${system}; inherit system; - }); + tests.setuid = pkgs.lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: + import ./tests/setuid.nix rec { + inherit nixpkgs; + nix = build.${system}; inherit system; + }); tests.binaryTarball = with import nixpkgs { system = "x86_64-linux"; }; diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 8d59c1c2b9aa..716b6e9bc9a3 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -24,7 +24,7 @@ readonly YELLOW='\033[38;33m' readonly YELLOW_UL='\033[38;4;33m' readonly CORES=$(sysctl -n hw.ncpu) -readonly NIX_USER_COUNT="$CORES" +readonly NIX_USER_COUNT="32" readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_NAME="nixbld" readonly NIX_FIRST_BUILD_UID="30001" diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index df579729af29..dbf8fe1b8f8a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -64,6 +64,8 @@ int main (int argc, char * * argv) settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + initPlugins(); + auto store = openStore().cast<LocalStore>(); /* It would be more appropriate to use $XDG_RUNTIME_DIR, since @@ -241,7 +243,7 @@ connected: if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); - setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ + store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute); } diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 910428c02686..b284daa3c2f7 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -7,13 +7,14 @@ namespace nix { +/* Note: Various places expect the allocated memory to be zeroed. */ static void * allocBytes(size_t n) { void * p; #if HAVE_BOEHMGC p = GC_malloc(n); #else - p = malloc(n); + p = calloc(n, 1); #endif if (!p) throw std::bad_alloc(); return p; diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index e1fc2bf6d796..3119a1848af2 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -83,7 +83,7 @@ public: for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]); std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { - return (string) a->name < (string) b->name; + return (const string &) a->name < (const string &) b->name; }); return res; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 33a9bc614285..2144d3452579 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -43,13 +43,14 @@ static char * dupString(const char * s) } +/* Note: Various places expect the allocated memory to be zeroed. */ static void * allocBytes(size_t n) { void * p; #if HAVE_BOEHMGC p = GC_malloc(n); #else - p = malloc(n); + p = calloc(n, 1); #endif if (!p) throw std::bad_alloc(); return p; @@ -293,6 +294,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) + , sArgs(symbols.create("args")) + , sOutputHash(symbols.create("outputHash")) + , sOutputHashAlgo(symbols.create("outputHashAlgo")) + , sOutputHashMode(symbols.create("outputHashMode")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -378,6 +383,18 @@ void EvalState::checkURI(const std::string & uri) && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) return; + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (hasPrefix(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (hasPrefix(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); } @@ -392,7 +409,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context) }; -void EvalState::addConstant(const string & name, Value & v) +Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; @@ -400,12 +417,18 @@ void EvalState::addConstant(const string & name, Value & v) baseEnv.values[baseEnvDispl++] = v2; string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + return v2; } Value * EvalState::addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } Value * v = allocValue(); string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); @@ -564,9 +587,7 @@ Env & EvalState::allocEnv(unsigned int size) Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); env->size = size; - /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */ - for (unsigned i = 0; i < size; ++i) - env->values[i] = 0; + /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; } @@ -1566,7 +1587,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) dstPath = srcToStore[path]; else { dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(checkSourcePath(path)).first + ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") @@ -1688,10 +1709,13 @@ void EvalState::printStats() printMsg(v, format(" time elapsed: %1%") % cpuTime); printMsg(v, format(" size of a value: %1%") % sizeof(Value)); printMsg(v, format(" size of an attr: %1%") % sizeof(Attr)); - printMsg(v, format(" environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs); - printMsg(v, format(" list elements: %1% (%2% bytes)") % nrListElems % bLists); + printMsg(v, format(" environments allocated count: %1%") % nrEnvs); + printMsg(v, format(" environments allocated bytes: %1%") % bEnvs); + printMsg(v, format(" list elements count: %1%") % nrListElems); + printMsg(v, format(" list elements bytes: %1%") % bLists); printMsg(v, format(" list concatenations: %1%") % nrListConcats); - printMsg(v, format(" values allocated: %1% (%2% bytes)") % nrValues % bValues); + printMsg(v, format(" values allocated count: %1%") % nrValues); + printMsg(v, format(" values allocated bytes: %1%") % bValues); printMsg(v, format(" sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets); printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9e3d30d95f49..9d8799b7906b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -69,7 +69,8 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder; + sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sOutputHash, sOutputHashAlgo, sOutputHashMode; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -210,7 +211,7 @@ private: void createBaseEnv(); - void addConstant(const string & name, Value & v); + Value * addConstant(const string & name, Value & v); Value * addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 28a0a6a87896..e5e01fb5831a 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -49,9 +49,10 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } -static Expr * unescapeStr(SymbolTable & symbols, const char * s) +static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; + t.reserve(length); char c; while ((c = *s++)) { if (c == '\\') { @@ -150,7 +151,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); + yylval->e = unescapeStr(data->symbols, yytext, yyleng); return STR; } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } @@ -178,7 +179,7 @@ or { return OR_KW; } return IND_STR; } <IND_STRING>\'\'\\. { - yylval->e = unescapeStr(data->symbols, yytext + 2); + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } <IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc index 6d78d2116121..382088c78872 100644 --- a/src/libexpr/names.cc +++ b/src/libexpr/names.cc @@ -41,7 +41,7 @@ bool DrvName::matches(DrvName & n) } -static string nextComponent(string::const_iterator & p, +string nextComponent(string::const_iterator & p, const string::const_iterator end) { /* Skip any dots and dashes (component separators). */ diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index 9667fc96fd0f..13c3093e77b0 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -24,6 +24,8 @@ private: typedef list<DrvName> DrvNames; +string nextComponent(string::const_iterator & p, + const string::const_iterator end); int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 975f0e8309e6..a800d24290ae 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -49,24 +49,38 @@ InvalidPathError::InvalidPathError(const Path & path) : void EvalState::realiseContext(const PathSet & context) { PathSet drvs; + for (auto & i : context) { std::pair<string, string> decoded = decodeContext(i); Path ctx = decoded.first; assert(store->isStorePath(ctx)); if (!store->isValidPath(ctx)) throw InvalidPathError(ctx); - if (!decoded.second.empty() && nix::isDerivation(ctx)) + if (!decoded.second.empty() && nix::isDerivation(ctx)) { drvs.insert(decoded.first + "!" + decoded.second); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(decoded.first); + DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + allowedPaths->insert(i->second.path); + } + } } - if (!drvs.empty()) { - if (!settings.enableImportFromDerivation) - throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(drvs); - } + + if (drvs.empty()) return; + + if (!settings.enableImportFromDerivation) + throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); + + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); } @@ -539,7 +553,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : args[0]->attrs->lexicographicOrder()) { if (i->name == state.sIgnoreNulls) continue; - string key = i->name; + const string & key = i->name; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { @@ -575,7 +589,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (key == "args") { + if (i->name == state.sArgs) { state.forceList(*i->value, pos); for (unsigned int n = 0; n < i->value->listSize(); ++n) { string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); @@ -598,15 +612,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.builder = state.forceString(*i->value, context, posDrvName); else if (i->name == state.sSystem) drv.platform = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sName) - drvName = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHash") + else if (i->name == state.sOutputHash) outputHash = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHashAlgo") + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHashMode") + else if (i->name == state.sOutputHashMode) handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); - else if (key == "outputs") { + else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; @@ -620,14 +632,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = s; else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is '%1%'") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") handleHashMode(s); - else if (key == "outputs") + else if (i->name == state.sOutputHash) outputHash = s; + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; + else if (i->name == state.sOutputHashMode) handleHashMode(s); + else if (i->name == state.sOutputs) handleOutputs(tokenizeString<Strings>(s)); } @@ -1009,20 +1017,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu } -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - - path = state.checkSourcePath(path); - - PathFilter filter = [&](const Path & path) { + const auto path = settings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); /* Call the filter function. The first argument is the path, @@ -1031,7 +1032,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args mkString(arg1, path); Value fun2; - state.callFunction(*args[0], arg1, fun2, noPos); + state.callFunction(*filterFun, arg1, fun2, noPos); Value arg2; mkString(arg2, @@ -1044,16 +1045,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args state.callFunction(fun2, arg2, res, noPos); return state.forceBool(res, pos); - }; + }) : defaultPathFilter; - Path dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first - : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); + } + } else + dstPath = expectedStorePath; mkString(v, dstPath, {dstPath}); } +static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + } + if (path.empty()) + throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) + name = baseNameOf(path); + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + + /************************************************************* * Sets *************************************************************/ @@ -1068,8 +1132,11 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V state.mkList(v, args[0]->attrs->size()); size_t n = 0; - for (auto & i : args[0]->attrs->lexicographicOrder()) - mkString(*(v.listElems()[n++] = state.allocValue()), i->name); + for (auto & i : *args[0]->attrs) + mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + + std::sort(v.listElems(), v.listElems() + n, + [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } @@ -1891,6 +1958,26 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a } +static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) + break; + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto & component : components) { + auto listElem = v.listElems()[n++] = state.allocValue(); + mkString(*listElem, std::move(component)); + } +} + + /************************************************************* * Networking *************************************************************/ @@ -2071,6 +2158,7 @@ void EvalState::createBaseEnv() addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); // Sets addPrimOp("__attrNames", 1, prim_attrNames); @@ -2125,6 +2213,7 @@ void EvalState::createBaseEnv() // Versions addPrimOp("__parseDrvName", 1, prim_parseDrvName); addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); // Derivations addPrimOp("derivationStrict", 1, prim_derivationStrict); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 39d23b04a5ce..31bf3f84f6c7 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -9,6 +9,9 @@ struct RegisterPrimOp { typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; static PrimOps * primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); }; diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index d3aac6aba1ff..bcc05c2cdad6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -37,6 +37,10 @@ MixCommonArgs::MixCommonArgs(const string & programName) std::string cat = "config"; settings.convertToArgs(*this, cat); + + // Backward compatibility hack: nix-env already had a --system flag. + if (programName == "nix-env") longFlags.erase("system"); + hiddenCategories.insert(cat); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 90a4867163df..7d888202bbf1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -262,6 +262,7 @@ void printVersion(const string & programName) void showManPage(const string & name) { restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), NULL); throw SysError(format("command 'man %1%' failed") % name.c_str()); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 1dcc4f0ac942..8e4861232db5 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -22,6 +22,7 @@ public: int handleExceptions(const string & programName, std::function<void()> fun); +/* Don't forget to call initPlugins() after settings are initialized! */ void initNix(); void parseCmdLine(int argc, char * * argv, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ab971dd8b6d9..d1b278b8efbe 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, *nar); + auto narCompressed = compress(compression, *nar, parallelCompression); auto now2 = std::chrono::steady_clock::now(); narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 8492ff600eba..e20b968442b7 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -19,6 +19,8 @@ public: const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; private: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d4b93b5104c1..1d611ffbaba5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -49,7 +49,9 @@ #include <sys/param.h> #include <sys/mount.h> #include <sys/syscall.h> +#if HAVE_SECCOMP #include <seccomp.h> +#endif #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif @@ -1335,19 +1337,6 @@ void DerivationGoal::tryToBuild() { trace("trying to build"); - /* Check for the possibility that some other goal in this process - has locked the output since we checked in haveDerivation(). - (It can't happen between here and the lockPaths() call below - because we're not allowing multi-threading.) If so, put this - goal to sleep until another goal finishes, then try again. */ - for (auto & i : drv->outputs) - if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) { - debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal") - % drvPath % i.second.path); - worker.waitForAnyGoal(shared_from_this()); - return; - } - /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other @@ -2484,7 +2473,7 @@ void setupSeccomp() { #if __linux__ if (!settings.filterSyscalls) return; - +#if HAVE_SECCOMP scmp_filter_ctx ctx; if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) @@ -2530,6 +2519,11 @@ void setupSeccomp() if (seccomp_load(ctx) != 0) throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif #endif } @@ -3428,7 +3422,7 @@ void DerivationGoal::flushLine() else { if (settings.verboseBuild && (settings.printRepeatedBuilds || curRound == 1)) - printError(filterANSIEscapes(currentLogLine, true)); + printError(currentLogLine); else { logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); @@ -3670,7 +3664,7 @@ void SubstitutionGoal::tryNext() /* Update the total expected download size. */ auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); - maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize); + maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize); maintainExpectedDownload = narInfo && narInfo->fileSize @@ -3684,9 +3678,12 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) { - printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'") - % sub->getUri() % storePath); + if (worker.store.requireSigs + && !sub->isTrusted + && !info->checkSignatures(worker.store, worker.store.publicKeys)) + { + printError("warning: substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), storePath); tryNext(); return; } @@ -3736,6 +3733,17 @@ void SubstitutionGoal::tryToRun() return; } + /* If the store path is already locked (probably by a + DerivationGoal), then put this goal to sleep. Note: we don't + acquire a lock here since that breaks addToStore(), so below we + handle an AlreadyLocked exception from addToStore(). The check + here is just an optimisation to prevent having to redo a + download due to a locked path. */ + if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { + worker.waitForAWhile(shared_from_this()); + return; + } + maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3752,7 +3760,7 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), - storePath, repair); + storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -3775,8 +3783,14 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); + } catch (AlreadyLocked & e) { + /* Probably a DerivationGoal is already building this store + path. Sleep for a while and try again. */ + state = &SubstitutionGoal::init; + worker.waitForAWhile(shared_from_this()); + return; } catch (Error & e) { - printInfo(e.msg()); + printError(e.msg()); /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d3c96ddd6e61..f46e8326235f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -6,6 +6,7 @@ #include <algorithm> #include <map> #include <thread> +#include <dlfcn.h> namespace nix { @@ -37,6 +38,7 @@ Settings::Settings() , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; @@ -137,4 +139,46 @@ void MaxBuildJobsSetting::set(const std::string & str) throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } + +void initPlugins() +{ + for (const auto & pluginFile : settings.pluginFiles.get()) { + Paths pluginFiles; + try { + auto ents = readDirectory(pluginFile); + for (const auto & ent : ents) + pluginFiles.emplace_back(pluginFile + "/" + ent.name); + } catch (SysError & e) { + if (e.errNo != ENOTDIR) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ + void *handle = + dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror()); + } + } + /* We handle settings registrations here, since plugins can add settings */ + if (RegisterSetting::settingRegistrations) { + for (auto & registration : *RegisterSetting::settingRegistrations) + settings.addSetting(registration); + delete RegisterSetting::settingRegistrations; + } + settings.handleUnknownSettings(); +} + +RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations; + +RegisterSetting::RegisterSetting(AbstractSetting * s) +{ + if (!settingRegistrations) + settingRegistrations = new SettingRegistrations; + settingRegistrations->emplace_back(s); +} + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 20ac8fe4e9ae..dd01f832df0c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -82,6 +82,9 @@ public: /* The directory where the main programs are stored. */ Path nixBinDir; + /* The directory where the man pages are stored. */ + Path nixManDir; + /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; @@ -367,14 +370,28 @@ public: Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<Paths> pluginFiles{this, {}, "plugin-files", + "Plugins to dynamically load at nix initialization time."}; }; // FIXME: don't use a global variable. extern Settings settings; +/* This should be called after settings are initialized, but before + anything else */ +void initPlugins(); + extern const string nixVersion; +struct RegisterSetting +{ + typedef std::vector<AbstractSetting *> SettingRegistrations; + static SettingRegistrations * settingRegistrations; + RegisterSetting(AbstractSetting * s); +}; + } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7afecc1cfc62..4afe51ea91ec 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -992,8 +992,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a lock on this path). */ - Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS")); - if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end()) + if (!locksHeld.count(info.path)) outputLock.lockPaths({realPath}); if (repair || !isValidPath(info.path)) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 30bef3a799d4..bbd50e1c1451 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -104,6 +104,9 @@ private: public: + // Hack for build-remote.cc. + PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS")); + /* Initialise the local store, upgrading the schema if necessary. */ LocalStore(const Params & params); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 50c46ce6fe99..e11efa5c2b54 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +ifneq ($(OS), FreeBSD) + libstore_LDFLAGS += -ldl +endif libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb @@ -22,7 +25,7 @@ ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket endif -ifeq ($(OS), Linux) +ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif @@ -35,6 +38,7 @@ libstore_CXXFLAGS = \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ + -DNIX_MAN_DIR=\"$(mandir)\" \ -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ -DLSOF=\"$(lsof)\" diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 587f29598851..08d1efdbeb01 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths, { auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) - throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath); + if (lockedPaths->count(lockPath)) { + if (!wait) return false; + throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); + } lockedPaths->insert(lockPath); } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 2a7de611446e..db51f950a320 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -2,10 +2,8 @@ #include "util.hh" - namespace nix { - /* Open (possibly create) a lock file and return the file descriptor. -1 is returned if create is false and the lock could not be opened because it doesn't exist. Any other error throws an exception. */ @@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); +MakeError(AlreadyLocked, Error); class PathLocks { @@ -38,9 +37,6 @@ public: void setDeletion(bool deletePaths); }; - -// FIXME: not thread-safe! bool pathIsLockedByMe(const Path & path); - } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77ab87ef728e..8830edcc3449 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash, } -std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) const +std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, h, name); return std::pair<Path, Hash>(dstPath, h); } @@ -840,7 +839,7 @@ ref<Store> openStore(const std::string & uri_, for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { - store->warnUnknownSettings(); + store->handleUnknownSettings(); return ref<Store>(store); } } @@ -897,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters() auto addStore = [&](const std::string & uri) { if (done.count(uri)) return; done.insert(uri); - stores.push_back(openStore(uri)); + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + printError("warning: %s", e.what()); + } }; for (auto uri : settings.substituters.get()) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 70f23e1fcaf4..563aa566bd37 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -248,6 +248,8 @@ public: const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + protected: struct State @@ -305,9 +307,9 @@ public: /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter) const; + std::pair<Path, Hash> computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 5e2631ba3408..470c925ed7a6 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,7 @@ #include "compression.hh" #include "util.hh" #include "finally.hh" +#include "logging.hh" #include <lzma.h> #include <bzlib.h> @@ -151,10 +152,10 @@ static ref<std::string> decompressBrotli(const std::string & in) #endif // HAVE_BROTLI } -ref<std::string> compress(const std::string & method, const std::string & in) +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) { StringSink ssink; - auto sink = makeCompressionSink(method, ssink); + auto sink = makeCompressionSink(method, ssink, parallel); (*sink)(in); sink->finish(); return ssink.s; @@ -189,10 +190,9 @@ struct XzSink : CompressionSink lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - XzSink(Sink & nextSink) : nextSink(nextSink) - { - lzma_ret ret = lzma_easy_encoder( - &strm, 6, LZMA_CHECK_CRC64); + template <typename F> + XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { + lzma_ret ret = initEncoder(); if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); // FIXME: apply the x86 BCJ filter? @@ -200,6 +200,9 @@ struct XzSink : CompressionSink strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } + XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { + return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + }) {} ~XzSink() { @@ -253,6 +256,27 @@ struct XzSink : CompressionSink } }; +#ifdef HAVE_LZMA_MT +struct ParallelXzSink : public XzSink +{ + ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + return lzma_stream_encoder_mt(&strm, &mt_options); + }) {} +}; +#endif + struct BzipSink : CompressionSink { Sink & nextSink; @@ -449,8 +473,16 @@ struct BrotliSink : CompressionSink }; #endif // HAVE_BROTLI -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink) +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + if (parallel) { +#ifdef HAVE_LZMA_MT + if (method == "xz") + return make_ref<ParallelXzSink>(nextSink); +#endif + printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); + } + if (method == "none") return make_ref<NoneSink>(nextSink); else if (method == "xz") diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index e3e6f5a99303..a0d7530d74fc 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,7 +8,7 @@ namespace nix { -ref<std::string> compress(const std::string & method, const std::string & in); +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); ref<std::string> decompress(const std::string & method, const std::string & in); @@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink virtual void finish() = 0; }; -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink); +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index d46ca65a3863..ce6858f0d65a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -7,10 +7,12 @@ namespace nix { void Config::set(const std::string & name, const std::string & value) { auto i = _settings.find(name); - if (i == _settings.end()) - throw UsageError("unknown setting '%s'", name); - i->second.setting->set(value); - i->second.setting->overriden = true; + if (i == _settings.end()) { + extras.emplace(name, value); + } else { + i->second.setting->set(value); + i->second.setting->overriden = true; + } } void Config::addSetting(AbstractSetting * setting) @@ -21,34 +23,34 @@ void Config::addSetting(AbstractSetting * setting) bool set = false; - auto i = initials.find(setting->name); - if (i != initials.end()) { + auto i = extras.find(setting->name); + if (i != extras.end()) { setting->set(i->second); setting->overriden = true; - initials.erase(i); + extras.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = initials.find(alias); - if (i != initials.end()) { + auto i = extras.find(alias); + if (i != extras.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { setting->set(i->second); setting->overriden = true; - initials.erase(i); + extras.erase(i); set = true; } } } } -void Config::warnUnknownSettings() +void Config::handleUnknownSettings() { - for (auto & i : initials) - warn("unknown setting '%s'", i.first); + for (auto & s : extras) + warn("unknown setting '%s'", s.first); } StringMap Config::getSettings(bool overridenOnly) @@ -60,7 +62,7 @@ StringMap Config::getSettings(bool overridenOnly) return res; } -void Config::applyConfigFile(const Path & path, bool fatal) +void Config::applyConfigFile(const Path & path) { try { string contents = readFile(path); @@ -80,7 +82,31 @@ void Config::applyConfigFile(const Path & path, bool fatal) vector<string> tokens = tokenizeString<vector<string> >(line); if (tokens.empty()) continue; - if (tokens.size() < 2 || tokens[1] != "=") + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); string name = tokens[0]; @@ -88,12 +114,7 @@ void Config::applyConfigFile(const Path & path, bool fatal) vector<string>::iterator i = tokens.begin(); advance(i, 2); - try { - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - } catch (UsageError & e) { - if (fatal) throw; - warn("in configuration file '%s': %s", path, e.what()); - } + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow }; } catch (SysError &) { } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 9a32af528ec7..d2e7faf17434 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -48,25 +48,25 @@ private: Settings _settings; - StringMap initials; + StringMap extras; public: Config(const StringMap & initials) - : initials(initials) + : extras(initials) { } void set(const std::string & name, const std::string & value); void addSetting(AbstractSetting * setting); - void warnUnknownSettings(); + void handleUnknownSettings(); StringMap getSettings(bool overridenOnly = false); const Settings & _getSettings() { return _settings; } - void applyConfigFile(const Path & path, bool fatal = false); + void applyConfigFile(const Path & path); void resetOverriden(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 11e3c9dca58a..75e4767550f7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -189,7 +189,8 @@ Hash::Hash(const std::string & s, HashType type) else if (size == base64Len()) { auto d = base64Decode(std::string(s, pos)); - assert(d.size() == hashSize); + if (d.size() != hashSize) + throw BadHash("invalid base-64 hash '%s'", s); memcpy(hash, d.data(), hashSize); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6924e0080475..27a631a37d10 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -44,7 +44,7 @@ public: prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n"); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index e0ec66c01803..5ee0b88ef50f 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -21,13 +21,29 @@ public: MonitorFdHup(int fd) { thread = std::thread([fd]() { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - fds[0].events = 0; - if (poll(fds, 1, -1) == -1) abort(); // can't happen - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } }); }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 272997397794..341dedfdf038 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1178,36 +1178,51 @@ void ignoreException() } -string filterANSIEscapes(const string & s, bool nixOnly) -{ - string t, r; - enum { stTop, stEscape, stCSI } state = stTop; - for (auto c : s) { - if (state == stTop) { - if (c == '\e') { - state = stEscape; - r = c; - } else - t += c; - } else if (state == stEscape) { - r += c; - if (c == '[') - state = stCSI; - else { - t += r; - state = stTop; +std::string filterANSIEscapes(const std::string & s, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } - } else { - r += c; - if (c >= 0x40 && c <= 0x7e) { - if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b')) - t += r; - state = stTop; - r.clear(); + + if (last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; } } + + else if (*i == '\r') + // do nothing for now + i++; + + else { + t += *i++; w++; + } } - t += r; + return t; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 75eb9751524e..47e02bc898a6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -388,10 +388,12 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" -/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is - set, only filter escape codes generated by Nixpkgs' stdenv (used to - denote nesting etc.). */ -string filterANSIEscapes(const string & s, bool nixOnly = false); +/* Truncate a string to 'width' printable characters. Certain ANSI + escape sequences (such as colour setting) are copied but not + included in the character count. Other ANSI escape sequences are + filtered. Also, tabs are expanded to spaces. */ +std::string filterANSIEscapes(const std::string & s, + unsigned int width = std::numeric_limits<unsigned int>::max()); /* Base64 encoding/decoding. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 1581c282c75c..99f773451ffe 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -232,6 +232,8 @@ void mainWrapped(int argc, char * * argv) myArgs.parseCmdline(args); + initPlugins(); + if (packages && fromArgs) throw UsageError("'-p' and '-E' are mutually exclusive"); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 370f216abccd..ec9a7174ecb9 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -213,6 +213,9 @@ int main(int argc, char ** argv) } return true; }); + + initPlugins(); + switch (cmd) { case cNone: throw UsageError("no command specified"); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index cc663a96924d..37fe22f48134 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -77,6 +77,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + auto profilesDir = settings.nixStateDir + "/profiles"; if (removeOld) removeOldGenerations(profilesDir); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 861fc2e5cd64..dfb1b8fc5dc4 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -44,6 +44,8 @@ int main(int argc, char ** argv) return true; }); + initPlugins(); + if (sshHost.empty()) throw UsageError("no host name specified"); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index b5d49b6428ac..890bffa19aa5 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -994,7 +994,7 @@ static void daemonLoop(char * * argv) if (matchUser(user, group, trustedUsers)) trusted = true; - if (!trusted && !matchUser(user, group, allowedUsers)) + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) @@ -1060,6 +1060,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (stdio) { if (getStoreType() == tDaemon) { /* Forward on this connection to the real daemon */ diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 016caf6d2346..97e66cbd937e 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1393,6 +1393,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index e05040a42deb..dd262bea0918 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -151,6 +151,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index fef3eaa45538..fa7ee254500c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -89,6 +89,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (args.size() > 2) throw UsageError("too many arguments"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4fc3421c0dde..efef7f15c094 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1052,6 +1052,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/build.cc b/src/nix/build.cc index f7c99f12dbbf..b329ac38ac2b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -50,7 +50,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand void run(ref<Store> store) override { - auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables); + auto buildables = build(store, dryRun ? DryRun : Build, installables); + + if (dryRun) return; for (size_t i = 0; i < buildables.size(); ++i) { auto & b(buildables[i]); diff --git a/src/nix/command.hh b/src/nix/command.hh index a7863c49f37a..97a6fee7fd27 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -198,7 +198,7 @@ std::shared_ptr<Installable> parseInstallable( SourceExprCommand & cmd, ref<Store> store, const std::string & installable, bool useDefaultInstallables); -Buildables toBuildables(ref<Store> store, RealiseMode mode, +Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables); PathSet toStorePaths(ref<Store> store, RealiseMode mode, diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2ddea9e70a6a..f29429c1ac49 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -57,15 +57,15 @@ struct CmdCopy : StorePathsCommand return { Example{ "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache -r $(type -p firefox)" + "nix copy --to file:///tmp/cache $(type -p firefox)" }, Example{ "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server -r /run/current-system" + "nix copy --to ssh://server /run/current-system" }, Example{ "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server -r /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" + "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" }, }; } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index c3b06c22eba8..a3fdd8a2808d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -253,7 +253,7 @@ std::shared_ptr<Installable> parseInstallable( return installables.front(); } -Buildables toBuildables(ref<Store> store, RealiseMode mode, +Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables) { if (mode != Build) @@ -291,7 +291,7 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode, { PathSet outPaths; - for (auto & b : toBuildables(store, mode, installables)) + for (auto & b : build(store, mode, installables)) for (auto & output : b.outputs) outPaths.insert(output.second); diff --git a/src/nix/local.mk b/src/nix/local.mk index bddd53b168d3..f76da194467c 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -6,4 +6,6 @@ nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp) nix_LIBS = libexpr libmain libstore libutil libformat +nix_LDFLAGS = -pthread + $(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 69620595d8ca..e99622faf472 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -90,6 +90,16 @@ struct CmdLsStore : StoreCommand, MixLs expectArg("path", &path); } + Examples examples() override + { + return { + Example{ + "To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + }, + }; + } + std::string name() override { return "ls-store"; @@ -116,6 +126,16 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } + Examples examples() override + { + return { + Example{ + "To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello" + }, + }; + } + std::string name() override { return "ls-nar"; diff --git a/src/nix/main.cc b/src/nix/main.cc index 8f6bbe8f51ae..bb107ec7d3f6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -92,6 +92,8 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (!args.command) args.showHelpAndExit(); Finally f([]() { stopProgressBar(); }); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc new file mode 100644 index 000000000000..310942574a2a --- /dev/null +++ b/src/nix/ping-store.cc @@ -0,0 +1,35 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdPingStore : StoreCommand +{ + std::string name() override + { + return "ping-store"; + } + + std::string description() override + { + return "test whether a store can be opened"; + } + + Examples examples() override + { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1" + }, + }; + } + + void run(ref<Store> store) override + { + store->connect(); + } +}; + +static RegisterCommand r1(make_ref<CmdPingStore>()); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index fb9955190b40..e6553c06f4ae 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -3,8 +3,9 @@ #include "sync.hh" #include "store-api.hh" -#include <map> #include <atomic> +#include <map> +#include <thread> namespace nix { @@ -22,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n) return fields[n].i; } -/* Truncate a string to 'width' printable characters. ANSI escape - sequences are copied but not included in the character count. Also, - tabs are expanded to spaces. */ -static std::string ansiTruncate(const std::string & s, int width) -{ - if (width <= 0) return s; - - std::string t; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - if (*i == '\e') { - t += *i++; - if (i != s.end() && *i == '[') { - t += *i++; - while (i != s.end() && (*i < 0x40 || *i > 0x7e)) { - t += *i++; - } - if (i != s.end()) t += *i++; - } - } - - else if (*i == '\t') { - t += ' '; w++; - while (w < (size_t) width && w & 8) { - t += ' '; w++; - } - } - - else { - t += *i++; w++; - } - } - - return t; -} - class ProgressBar : public Logger { private: @@ -101,15 +64,28 @@ private: Sync<State> state_; + std::thread updateThread; + + std::condition_variable quitCV, updateCV; + public: ProgressBar() { + updateThread = std::thread([&]() { + auto state(state_.lock()); + while (state->active) { + state.wait(updateCV); + draw(*state); + state.wait_for(quitCV, std::chrono::milliseconds(50)); + } + }); } ~ProgressBar() { stop(); + updateThread.join(); } void stop() @@ -121,6 +97,8 @@ public: writeToStderr("\r\e[K"); if (status != "") writeToStderr("[" + status + "]\n"); + updateCV.notify_one(); + quitCV.notify_one(); } void log(Verbosity lvl, const FormatOrString & fs) override @@ -132,7 +110,7 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); - update(state); + draw(state); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -167,7 +145,12 @@ public: if (type == actSubstitute) { auto name = storePathToName(getS(fields, 0)); - i->s = fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", name, getS(fields, 1)); + auto sub = getS(fields, 1); + i->s = fmt( + hasPrefix(sub, "local") + ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s" + : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", + name, sub); } if (type == actQueryPathInfo) { @@ -180,7 +163,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(*state); + update(); } /* Check whether an activity has an ancestore with the specified @@ -215,7 +198,7 @@ public: state->its.erase(i); } - update(*state); + update(); } void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override @@ -225,7 +208,7 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(*state); + update(); } else if (type == resBuildLogLine) { @@ -238,25 +221,25 @@ public: info.lastLine = lastLine; state->activities.emplace_back(info); i->second = std::prev(state->activities.end()); - update(*state); + update(); } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(*state); + update(); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(*state); + update(); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(*state); + update(); } else if (type == resProgress) { @@ -267,7 +250,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(*state); + update(); } else if (type == resSetExpected) { @@ -279,17 +262,16 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(*state); + update(); } } void update() { - auto state(state_.lock()); - update(*state); + updateCV.notify_one(); } - void update(State & state) + void draw(State & state) { if (!state.active) return; @@ -323,7 +305,10 @@ public: } } - writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K"); + auto width = getWindowSize().second; + if (width <= 0) std::numeric_limits<decltype(width)>::max(); + + writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K"); } std::string getStatus(State & state) diff --git a/src/nix/run.cc b/src/nix/run.cc index ade87e63a49c..822654daf488 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -85,6 +85,10 @@ struct CmdRun : InstallablesCommand "To run GNU Hello:", "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" }, + Example{ + "To run GNU Hello in a chroot store:", + "nix run --store ~/my-nix nixpkgs.hello -c hello" + }, }; } diff --git a/tests/build-dry.sh b/tests/build-dry.sh new file mode 100644 index 000000000000..610e6070c5d7 --- /dev/null +++ b/tests/build-dry.sh @@ -0,0 +1,52 @@ +source common.sh + +################################################### +# Check that --dry-run isn't confused with read-only mode +# https://github.com/NixOS/nix/issues/1795 + +clearStore +clearCache + +# Ensure this builds successfully first +nix build -f dependencies.nix + +clearStore +clearCache + +# Try --dry-run using old command first +nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +# Now new command: +nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" + +# TODO: XXX: FIXME: #1793 +# Disable this part of the test until the problem is resolved: +if [ -n "$ISSUE_1795_IS_FIXED" ]; then +clearStore +clearCache + +# Try --dry-run using new command first +nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" +# Now old command: +nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +fi + +################################################### +# Check --dry-run doesn't create links with --dry-run +# https://github.com/NixOS/nix/issues/1849 +clearStore +clearCache + +RESULT=$TEST_ROOT/result-link +rm -f $RESULT + +nix-build dependencies.nix -o $RESULT --dry-run + +[[ ! -h $RESULT ]] || fail "nix-build --dry-run created output link" + +nix build -f dependencies.nix -o $RESULT --dry-run + +[[ ! -h $RESULT ]] || fail "nix build --dry-run created output link" + +nix build -f dependencies.nix -o $RESULT + +[[ -h $RESULT ]] diff --git a/tests/init.sh b/tests/init.sh index 41cca047d8fb..e5353598bcc4 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -16,7 +16,12 @@ mkdir "$NIX_CONF_DIR" cat > "$NIX_CONF_DIR"/nix.conf <<EOF build-users-group = keep-derivations = false +include nix.conf.extra +EOF + +cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF fsync-metadata = false +!include nix.conf.extra.not-there EOF # Initialise the database. diff --git a/tests/lang/data b/tests/lang/data new file mode 100644 index 000000000000..257cc5642cb1 --- /dev/null +++ b/tests/lang/data @@ -0,0 +1 @@ +foo diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix new file mode 100644 index 000000000000..e67168cf3edf --- /dev/null +++ b/tests/lang/eval-okay-path.nix @@ -0,0 +1,7 @@ +builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + } diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/lang/eval-okay-splitversion.exp new file mode 100644 index 000000000000..153ceb8186a0 --- /dev/null +++ b/tests/lang/eval-okay-splitversion.exp @@ -0,0 +1 @@ +[ "1" "2" "3" ] diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/lang/eval-okay-splitversion.nix new file mode 100644 index 000000000000..9e5c99d2e7f6 --- /dev/null +++ b/tests/lang/eval-okay-splitversion.nix @@ -0,0 +1 @@ +builtins.splitVersion "1.2.3" diff --git a/tests/local.mk b/tests/local.mk index e90b9f7da4ad..ec7ebfb0dedc 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -13,6 +13,7 @@ nix_tests = \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ + build-dry.sh \ build-remote.sh \ nar-access.sh \ structured-attrs.sh \ @@ -22,7 +23,8 @@ nix_tests = \ run.sh \ brotli.sh \ pure-eval.sh \ - check.sh + check.sh \ + plugins.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) @@ -31,4 +33,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh +installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) diff --git a/tests/misc.sh b/tests/misc.sh index 6d0ab3adcec8..eda0164167f2 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. -nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$" +nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$" diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 0bf5b42d84a8..0dc147fb34e9 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -2,7 +2,7 @@ { nixpkgs, system, nix }: -with import (nixpkgs + /nixos/lib/testing.nix) { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { @@ -29,10 +29,10 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { startAll; # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`; + my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; $client->succeed("mkdir -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_dsa"); - $client->succeed("chmod 600 /root/.ssh/id_dsa"); + $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); + $client->succeed("chmod 600 /root/.ssh/id_ed25519"); # Install the SSH key on the server. $server->succeed("mkdir -m 700 /root/.ssh"); diff --git a/tests/plugins.sh b/tests/plugins.sh new file mode 100644 index 000000000000..4b1baeddce32 --- /dev/null +++ b/tests/plugins.sh @@ -0,0 +1,7 @@ +source common.sh + +set -o pipefail + +res=$(nix eval '(builtins.anotherNull)' --option setting-set true --option plugin-files $PWD/plugins/libplugintest*) + +[ "$res"x = "nullx" ] diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk new file mode 100644 index 000000000000..1d2bac052fd2 --- /dev/null +++ b/tests/plugins/local.mk @@ -0,0 +1,9 @@ +libraries += libplugintest + +libplugintest_DIR := $(d) + +libplugintest_SOURCES := $(d)/plugintest.cc + +libplugintest_ALLOW_UNDEFINED := 1 + +libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc new file mode 100644 index 000000000000..8da15ebabd7d --- /dev/null +++ b/tests/plugins/plugintest.cc @@ -0,0 +1,19 @@ +#include "globals.hh" +#include "primops.hh" + +using namespace nix; + +static BaseSetting<bool> settingSet{false, "setting-set", + "Whether the plugin-defined setting was set"}; + +static RegisterSetting rs(&settingSet); + +static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v) +{ + if (settingSet) + mkNull(v); + else + mkBool(v, false); +} + +static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 75704ace2dba..d7a4b21989e5 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -46,13 +46,13 @@ in nix.buildMachines = [ { hostName = "slave1"; sshUser = "root"; - sshKey = "/root/.ssh/id_dsa"; + sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; maxJobs = 1; } { hostName = "slave2"; sshUser = "root"; - sshKey = "/root/.ssh/id_dsa"; + sshKey = "/root/.ssh/id_ed25519"; system = "i686-linux"; maxJobs = 1; } @@ -70,10 +70,10 @@ in startAll; # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`; + my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; $client->succeed("mkdir -p -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_dsa"); - $client->succeed("chmod 600 /root/.ssh/id_dsa"); + $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); + $client->succeed("chmod 600 /root/.ssh/id_ed25519"); # Install the SSH key on the slaves. $client->waitForUnit("network.target"); diff --git a/tests/restricted.sh b/tests/restricted.sh index 6c0392facf31..0605383cc86a 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -36,3 +36,5 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT) (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . + +[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] diff --git a/tests/user-envs.sh b/tests/user-envs.sh index c4192fdc59b2..ba63923113d8 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -24,6 +24,9 @@ rm -f $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr nix-env -qa '*' --description | grep -q silly +# Query the system. +nix-env -qa '*' --system | grep -q $system + # Install "foo-1.0". nix-env -i foo-1.0 |