diff options
124 files changed, 4815 insertions, 1527 deletions
diff --git a/Makefile.config.in b/Makefile.config.in index 53dca1fcf10a..45a70cd6dd1a 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -5,6 +5,7 @@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ +HAVE_READLINE = @HAVE_READLINE@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ @@ -15,6 +16,7 @@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ bro = @bro@ +lsof = @lsof@ datadir = @datadir@ datarootdir = @datarootdir@ docdir = @docdir@ @@ -26,6 +28,7 @@ localstatedir = @localstatedir@ mandir = @mandir@ pkglibdir = $(libdir)/$(PACKAGE_NAME) prefix = @prefix@ +sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ doc_generate = @doc_generate@ diff --git a/configure.ac b/configure.ac index 3e6a894e3b10..24a95ce56f3b 100644 --- a/configure.ac +++ b/configure.ac @@ -128,6 +128,7 @@ NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(pv, pv, pv) AC_PATH_PROG(bro, bro, bro) +AC_PATH_PROG(lsof, lsof, lsof) NEED_PROG(cat, cat) @@ -239,6 +240,12 @@ fi AC_SUBST(tarFlags) +AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], + [path of a statically-linked shell to use as /bin/sh in sandboxes]), + sandbox_shell=$withval) +AC_SUBST(sandbox_shell) + + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index d5bc1c592553..1957e1105e68 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -22,10 +22,7 @@ will call whenever it wants to build a derivation. The build hook will perform it in the usual way if possible, or it can accept it, in which case it is responsible for somehow getting the inputs of the build to another machine, doing the build there, and getting the -results back. The details of the build hook protocol are described in -the documentation of the <link -linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar> -variable</link>.</para> +results back.</para> <example xml:id='ex-remote-systems'><title>Remote machine configuration: <filename>remote-systems.conf</filename></title> @@ -103,14 +100,6 @@ requiredSystemFeatures = [ "kvm" ]; </orderedlist> -You should also set up the environment variable -<envar>NIX_CURRENT_LOAD</envar> to point at a directory (e.g., -<filename>/var/run/nix/current-load</filename>) that -<filename>build-remote</filename> uses to remember how many builds -it is currently executing remotely. It doesn't look at the actual -load on the remote machine, so if you have multiple instances of Nix -running, they should use the same <envar>NIX_CURRENT_LOAD</envar> -file. Maybe in the future <filename>build-remote</filename> will -look at the actual remote load.</para> +</para> </chapter> diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 6952829e8f71..616983bc7f0e 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -17,13 +17,32 @@ <refsection><title>Description</title> -<para>A number of persistent settings of Nix are stored in the file -<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> or -<filename>$NIX_CONF_DIR/nix.conf</filename> if <envar>NIX_CONF_DIR</envar> is set. -This file is a list of <literal><replaceable>name</replaceable> = +<para>Nix reads settings from two configuration files:</para> + +<itemizedlist> + + <listitem> + <para>The system-wide configuration file + <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> + (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or + <filename>$NIX_CONF_DIR/nix.conf</filename> if + <envar>NIX_CONF_DIR</envar> is set.</para> + </listitem> + + <listitem> + <para>The user configuration file + <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or + <filename>~/.config/nix/nix.conf</filename> if + <envar>XDG_CONFIG_HOME</envar> is not set.</para> + </listitem> + +</itemizedlist> + +<para>The configuration files consist of +<literal><replaceable>name</replaceable> = <replaceable>value</replaceable></literal> pairs, one per line. -Comments start with a <literal>#</literal> character. Here is an example -configuration file:</para> +Comments start with a <literal>#</literal> character. Here is an +example configuration file:</para> <programlisting> gc-keep-outputs = true # Nice for developers @@ -31,8 +50,9 @@ gc-keep-derivations = true # Idem env-keep-derivations = false </programlisting> -<para>You can override settings using the <option>--option</option> -flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> +<para>You can override settings on the command line using the +<option>--option</option> flag, e.g. <literal>--option gc-keep-outputs +false</literal>.</para> <para>The following settings are currently available: @@ -334,25 +354,16 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>use-binary-caches</literal></term> - - <listitem><para>If set to <literal>true</literal> (the default), - Nix will check the binary caches specified by - <option>binary-caches</option> and related options to obtain - binary substitutes.</para></listitem> - - </varlistentry> + <varlistentry><term><literal>substituters</literal></term> - - <varlistentry><term><literal>binary-caches</literal></term> - - <listitem><para>A list of URLs of binary caches, separated by + <listitem><para>A list of URLs of substituters, separated by whitespace. The default is <literal>https://cache.nixos.org</literal>.</para></listitem> </varlistentry> + <!-- <varlistentry><term><literal>binary-caches-files</literal></term> <listitem><para>A list of names of files that will be read to @@ -365,28 +376,28 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> option to read files created by untrusted users!</para></listitem> </varlistentry> + --> - <varlistentry><term><literal>trusted-binary-caches</literal></term> + <varlistentry><term><literal>trusted-substituters</literal></term> - <listitem><para>A list of URLs of binary caches, separated by + <listitem><para>A list of URLs of substituters, separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying <literal>--option - binary-caches <replaceable>urls</replaceable></literal> on the + substituters <replaceable>urls</replaceable></literal> on the command line. Unprivileged users are only allowed to pass a - subset of the URLs listed in <literal>binary-caches</literal> and - <literal>trusted-binary-caches</literal>.</para></listitem> + subset of the URLs listed in <literal>substituters</literal> and + <literal>trusted-substituters</literal>.</para></listitem> </varlistentry> - <varlistentry><term><literal>extra-binary-caches</literal></term> + <varlistentry><term><literal>extra-substituters</literal></term> <listitem><para>Additional binary caches appended to those - specified in <option>binary-caches</option> and - <option>binary-caches-files</option>. When used by unprivileged - users, untrusted binary caches (i.e. those not listed in - <option>trusted-binary-caches</option>) are silently + specified in <option>substituters</option>. When used by + unprivileged users, untrusted substituters (i.e. those not listed + in <option>trusted-substituters</option>) are silently ignored.</para></listitem> </varlistentry> @@ -413,7 +424,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>binary-caches-parallel-connections</literal></term> + <varlistentry><term><literal>http-connections</literal></term> <listitem><para>The maximum number of parallel TCP connections used to fetch files from binary caches and by other downloads. It @@ -422,15 +433,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>verify-https-binary-caches</literal></term> - - <listitem><para>Whether HTTPS binary caches are required to have a - certificate that can be verified. Defaults to - <literal>true</literal>.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>netrc-file</literal></term> <listitem><para>If set to an absolute path to a <filename>netrc</filename> diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index c757cb17bd10..a83aeaf2e575 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -148,139 +148,6 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen> </varlistentry> -<varlistentry xml:id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term> - - <listitem> - - <para>Specifies the location of the <emphasis>build hook</emphasis>, - which is a program (typically some script) that Nix will call - whenever it wants to build a derivation. This is used to implement - distributed builds<phrase condition="manual"> (see <xref - linkend="chap-distributed-builds" />)</phrase>.</para> - - <!-- - The protocol by - which the calling Nix process and the build hook communicate is as - follows. - - <para>The build hook is called with the following command-line - arguments: - - <orderedlist> - - <listitem><para>A boolean value <literal>0</literal> or - <literal>1</literal> specifying whether Nix can locally execute - more builds, as per the <link - linkend="opt-max-jobs"><option>- -max-jobs</option> option</link>. - The purpose of this argument is to allow the hook to not have to - maintain bookkeeping for the local machine.</para></listitem> - - <listitem><para>The Nix platform identifier for the local machine - (e.g., <literal>i686-linux</literal>).</para></listitem> - - <listitem><para>The Nix platform identifier for the derivation, - i.e., its <link linkend="attr-system"><varname>system</varname> - attribute</link>.</para></listitem> - - <listitem><para>The store path of the derivation.</para></listitem> - - </orderedlist> - - </para> - - <para>On the basis of this information, and whatever persistent - state the build hook keeps about other machines and their current - load, it has to decide what to do with the build. It should print - out on standard error one of the following responses (terminated by - a newline, <literal>"\n"</literal>): - - <variablelist> - - <varlistentry><term><literal># decline</literal></term> - - <listitem><para>The build hook is not willing or able to perform - the build; the calling Nix process should do the build itself, - if possible.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal># postpone</literal></term> - - <listitem><para>The build hook cannot perform the build now, but - can do so in the future (e.g., because all available build slots - on remote machines are in use). The calling Nix process should - postpone this build until at least one currently running build - has terminated.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal># accept</literal></term> - - <listitem><para>The build hook has accepted the - build.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - <para>After sending <literal># accept</literal>, the hook should - read one line from standard input, which will be the string - <literal>okay</literal>. It can then proceed with the build. - Before sending <literal>okay</literal>, Nix will store in the hook’s - current directory a number of text files that contain information - about the derivation: - - <variablelist> - - <varlistentry><term><filename>inputs</filename></term> - - <listitem><para>The set of store paths that are inputs to the - build process (one per line). These have to be copied - <emphasis>to</emphasis> the remote machine (in addition to the - store derivation itself).</para></listitem> - - </varlistentry> - - <varlistentry><term><filename>outputs</filename></term> - - <listitem><para>The set of store paths that are outputs of the - derivation (one per line). These have to be copied - <emphasis>from</emphasis> the remote machine if the build - succeeds.</para></listitem> - - </varlistentry> - - <varlistentry><term><filename>references</filename></term> - - <listitem><para>The reference graph of the inputs, in the format - accepted by the command <command>nix-store - - -register-validity</command>. It is necessary to run this - command on the remote machine after copying the inputs to inform - Nix on the remote machine that the inputs are valid - paths.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - <para>The hook should copy the inputs to the remote machine, - register the validity of the inputs, perform the remote build, and - copy the outputs back to the local machine. An exit code other than - <literal>0</literal> indicates that the hook has failed. An exit - code equal to 100 means that the remote build failed (as opposed to, - e.g., a network error).</para> - --> - - </listitem> - - -</varlistentry> - - <varlistentry xml:id="envar-remote"><term><envar>NIX_REMOTE</envar></term> <listitem><para>This variable should be set to diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 85f10e0760bc..8462cf8a0270 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -1136,7 +1136,7 @@ user environment elements, etc. --> <listitem><para>Print all of the meta-attributes of the derivation. This option is only available with - <option>--xml</option>.</para></listitem> + <option>--xml</option> or <option>--json</option>.</para></listitem> </varlistentry> diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml index 1e556c7ed7c4..3d03358bea3f 100644 --- a/doc/manual/command-ref/nix-instantiate.xml +++ b/doc/manual/command-ref/nix-instantiate.xml @@ -117,21 +117,19 @@ input.</para> <varlistentry><term><option>--xml</option></term> - <listitem><para>When used with <option>--parse</option> and - <option>--eval</option>, print the resulting expression as an - XML representation of the abstract syntax tree rather than as an - ATerm. The schema is the same as that used by the <link - linkend="builtin-toXML"><function>toXML</function> - built-in</link>.</para></listitem> + <listitem><para>When used with <option>--eval</option>, print the resulting + value as an XML representation of the abstract syntax tree rather than as + an ATerm. The schema is the same as that used by the <link + linkend="builtin-toXML"><function>toXML</function> built-in</link>. + </para></listitem> </varlistentry> <varlistentry><term><option>--json</option></term> - <listitem><para>When used with <option>--parse</option> and - <option>--eval</option>, print the resulting expression as an - JSON representation of the abstract syntax tree rather than as an - ATerm.</para></listitem> + <listitem><para>When used with <option>--eval</option>, print the resulting + value as an JSON representation of the abstract syntax tree rather + than as an ATerm.</para></listitem> </varlistentry> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index e9baff65961a..f46a93ae0d5d 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -65,7 +65,7 @@ available as <function>builtins.derivation</function>.</para> <replaceable>set</replaceable></term> <listitem><para>Return the names of the attributes in the set - <replaceable>set</replaceable> in a sorted list. For instance, + <replaceable>set</replaceable> in an alphabetically sorted list. For instance, <literal>builtins.attrNames { y = 1; x = "foo"; }</literal> evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem> @@ -213,10 +213,11 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> <varlistentry><term><function>builtins.match</function> <replaceable>regex</replaceable> <replaceable>str</replaceable></term> - <listitem><para>Returns a list if - <replaceable>regex</replaceable> matches - <replaceable>str</replaceable> precisely, otherwise returns <literal>null</literal>. - Each item in the list is a regex group. + <listitem><para>Returns a list if the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> + matches <replaceable>str</replaceable> precisely, otherwise returns + <literal>null</literal>. Each item in the list is a regex group. <programlisting> builtins.match "ab" "abc" @@ -236,6 +237,11 @@ builtins.match "a(b)(c)" "abc" Evaluates to <literal>[ "b" "c" ]</literal>. +<programlisting> +builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " +</programlisting> + +Evaluates to <literal>[ "foo" ]</literal>. </para></listitem> </varlistentry> @@ -292,6 +298,24 @@ with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixo stdenv.mkDerivation { … } </programlisting> + Note that when obtaining the hash with <varname>nix-prefetch-url + </varname> the option <varname>--unpack</varname> is required. + </para> + + <para>This function can also verify the contents against a hash. + In that case, the function takes a set instead of a URL. The set + requires the attribute <varname>url</varname> and the attribute + <varname>sha256</varname>, e.g. + +<programlisting> +with import (fetchTarball { + url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz; + sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; +}) {}; + +stdenv.mkDerivation { … } +</programlisting> + </para> <para>This function is not available if <link @@ -1163,6 +1187,19 @@ stdenv.mkDerivation (rec { </varlistentry> + <varlistentry><term><function>builtins.tryEval</function> + <replaceable>e</replaceable></term> + + <listitem><para>Try to evaluate <replaceable>e</replaceable>. + Return a set containing the attributes <literal>success</literal> + (<literal>true</literal> if <replaceable>e</replaceable> evaluated + successfully, <literal>false</literal> if an error was thrown) and + <literal>value</literal>, equalling <replaceable>e</replaceable> + if successful and <literal>false</literal> otherwise. + </para></listitem> + + </varlistentry> + <varlistentry><term><function>builtins.typeOf</function> <replaceable>e</replaceable></term> diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml index 0c58984ac489..be065da3eb2d 100644 --- a/doc/manual/introduction/about-nix.xml +++ b/doc/manual/introduction/about-nix.xml @@ -93,7 +93,7 @@ time window in which the package has some files from the old version and some files from the new version — which would be bad because a program might well crash if it’s started during that period.</para> -<para>And since package aren’t overwritten, the old versions are still +<para>And since packages aren’t overwritten, the old versions are still there after an upgrade. This means that you can <emphasis>roll back</emphasis> to the old version:</para> @@ -261,6 +261,12 @@ xlink:href="http://nixos.org/">NixOS homepage</link>.</para> xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU LGPLv2.1 or (at your option) any later version</link>.</para> +<para>Nix uses the <link +xlink:href="https://github.com/antirez/linenoise">linenoise +library</link>, which has the following license:</para> + +<programlisting><xi:include href="../../../src/linenoise/LICENSE" parse="text" /></programlisting> + </simplesect> diff --git a/doc/manual/introduction/quick-start.xml b/doc/manual/introduction/quick-start.xml index 5ae9f6ad543b..aa239b7538b4 100644 --- a/doc/manual/introduction/quick-start.xml +++ b/doc/manual/introduction/quick-start.xml @@ -40,7 +40,7 @@ libxslt-1.1.28 <step><para>Install some packages from the channel: <screen> -$ nix-env -i hello <replaceable>...</replaceable> </screen> +$ nix-env -i hello</screen> This should download pre-built packages; it should not build them locally (if it does, something went wrong).</para></step> diff --git a/doc/manual/release-notes/rl-1.11.xml b/doc/manual/release-notes/rl-1.11.xml index efb03d61393f..fe422dd1f893 100644 --- a/doc/manual/release-notes/rl-1.11.xml +++ b/doc/manual/release-notes/rl-1.11.xml @@ -122,13 +122,6 @@ $ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zl </listitem> <listitem> - <para>The Nix language now supports floating point numbers. They are - based on regular C++ <literal>float</literal> and compatible with - existing integers and number-related operations. Export and import to and - from JSON and XML works, too. - </para> - </listitem> - <listitem> <para>All "chroot"-containing strings got renamed to "sandbox". In particular, some Nix options got renamed, but the old names are still accepted as lower-priority aliases. diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml index d6864b3f55d1..b7f45fc44a36 100644 --- a/doc/manual/release-notes/rl-1.12.xml +++ b/doc/manual/release-notes/rl-1.12.xml @@ -17,6 +17,13 @@ have write access to the Nix database.</para> </listitem> + <listitem> + <para>The Nix language now supports floating point numbers. They are + based on regular C++ <literal>float</literal> and compatible with + existing integers and number-related operations. Export and import to and + from JSON and XML works, too. + </para> + </listitem> </itemizedlist> <para>This release has contributions from TBD.</para> diff --git a/local.mk b/local.mk index dc10e6870a87..0a225423741d 100644 --- a/local.mk +++ b/local.mk @@ -7,8 +7,7 @@ dist-files += configure config.h.in nix.spec perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr \ - -Wno-unneeded-internal-declaration +GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 5d57a5ec8ff1..c5ef97ee9a3f 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -16,8 +16,6 @@ <dict> <key>NIX_SSL_CERT_FILE</key> <string>/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt</string> - <key>XDG_CACHE_HOME</key> - <string>/root/.cache</string> </dict> </dict> </plist> diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index fcd799e177d0..5fc04a3f5713 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -7,4 +7,3 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process -Environment=XDG_CACHE_HOME=/root/.cache diff --git a/mk/lib.mk b/mk/lib.mk index bb82801d3b4e..1da51d879734 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -53,8 +53,8 @@ BUILD_SHARED_LIBS ?= 1 ifeq ($(BUILD_SHARED_LIBS), 1) ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) - GLOBAL_CFLAGS += -U__STRICT_ANSI__ - GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ + GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE + GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE else GLOBAL_CFLAGS += -fPIC GLOBAL_CXXFLAGS += -fPIC diff --git a/perl/Makefile b/perl/Makefile index cf655ae3d656..684a37e8121f 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include nix/config.h +GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include Makefile.config diff --git a/perl/configure.ac b/perl/configure.ac index d617c78535f6..9f49db4d2816 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -40,11 +40,6 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};') AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) -AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], - [path of the Nix store (defaults to /nix/store)]), - storedir=$withval, storedir='/nix/store') -AC_SUBST(storedir) - # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) @@ -52,7 +47,7 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], have_sodium=1], [have_sodium=]) AC_SUBST(HAVE_SODIUM, [$have_sodium]) -# Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl). +# Check for the required Perl dependencies (DBI and DBD::SQLite). perlFlags="-I$perllibdir" AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], @@ -63,10 +58,6 @@ AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], [prefix of the Perl DBD::SQLite library]), perlFlags="$perlFlags -I$withval") -AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH], - [prefix of the Perl WWW::Curl library]), - perlFlags="$perlFlags -I$withval") - AC_MSG_CHECKING([whether DBD::SQLite works]) if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then AC_MSG_RESULT(no) @@ -74,13 +65,6 @@ if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then fi AC_MSG_RESULT(yes) -AC_MSG_CHECKING([whether WWW::Curl works]) -if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl module WWW::Curl is missing.]) -fi -AC_MSG_RESULT(yes) - AC_SUBST(perlFlags) PKG_CHECK_MODULES([NIX], [nix-store]) @@ -109,7 +93,7 @@ for name in $ac_subst_vars; do done rm -f Makefile.config -ln -s ../mk mk +ln -sfn ../mk mk AC_CONFIG_FILES([]) AC_OUTPUT diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index f494e34a5e7b..4bdee7fd89f9 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -20,22 +20,15 @@ $useBindings = 1; %config = (); sub readConfig { - if (defined $ENV{'_NIX_OPTIONS'}) { - foreach my $s (split '\n', $ENV{'_NIX_OPTIONS'}) { - my ($n, $v) = split '=', $s, 2; - $config{$n} = $v; - } - } else { - my $config = "$confDir/nix.conf"; - return unless -f $config; - - open CONFIG, "<$config" or die "cannot open ‘$config’"; - while (<CONFIG>) { - /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; - $config{$1} = $2; - } - close CONFIG; + my $config = "$confDir/nix.conf"; + return unless -f $config; + + open CONFIG, "<$config" or die "cannot open ‘$config’"; + while (<CONFIG>) { + /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; + $config{$1} = $2; } + close CONFIG; } return 1; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f613e3df329a..aa14bfa6270a 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -1,3 +1,5 @@ +#include "config.h" + #include "EXTERN.h" #include "perl.h" #include "XSUB.h" @@ -25,9 +27,7 @@ static ref<Store> store() static std::shared_ptr<Store> _store; if (!_store) { try { - logger = makeDefaultLogger(); settings.loadConfFile(); - settings.update(); settings.lockCPU = false; _store = openStore(); } catch (Error & e) { diff --git a/perl/local.mk b/perl/local.mk index 35113bd960d2..b13d4c0d6396 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -20,11 +20,11 @@ Store_DIR := lib/Nix Store_SOURCES := $(Store_DIR)/Store.cc Store_CXXFLAGS = \ - -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \ + $(NIX_CFLAGS) \ + -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ -D_FILE_OFFSET_BITS=64 \ -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion \ - $(NIX_CFLAGS) + -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) diff --git a/release-common.nix b/release-common.nix new file mode 100644 index 000000000000..8047c75bdb74 --- /dev/null +++ b/release-common.nix @@ -0,0 +1,21 @@ +{ pkgs }: + +rec { + sh = pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_ASH y + CONFIG_ASH_BUILTIN_ECHO y + CONFIG_ASH_BUILTIN_TEST y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + ''; + }; + + configureFlags = + [ "--disable-init-state" + "--enable-gc" + "--with-sandbox-shell=${sh}/bin/busybox" + ]; +} diff --git a/release.nix b/release.nix index 8727c2520b1e..f1a553d01cc6 100644 --- a/release.nix +++ b/release.nix @@ -7,7 +7,7 @@ let pkgs = import <nixpkgs> {}; - systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" ]; + systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; jobs = rec { @@ -66,6 +66,8 @@ let with import <nixpkgs> { inherit system; }; + with import ./release-common.nix { inherit pkgs; }; + releaseTools.nixBuild { name = "nix"; src = tarball; @@ -74,6 +76,7 @@ let [ curl bzip2 xz brotli openssl pkgconfig sqlite boehmgc + ] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) @@ -82,11 +85,8 @@ let customMemoryManagement = false; }); - configureFlags = '' - --disable-init-state - --enable-gc - --sysconfdir=/etc - ''; + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; enableParallelBuilding = true; @@ -111,12 +111,11 @@ let buildInputs = [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ] - ++ lib.optional stdenv.isLinux libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} - --with-www-curl=${perlPackages.WWWCurl}/${pkgs.perl.libPrefix} ''; enableParallelBuilding = true; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 3cdf431041cb..ab95c09c8305 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -85,6 +85,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi - export PATH="$NIX_LINK/bin:$NIX_LINK/sbin:$__savedpath" + export PATH="$NIX_LINK/bin:$__savedpath" unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES fi diff --git a/shell.nix b/shell.nix index df0ad01df583..c4e2a20f8fa7 100644 --- a/shell.nix +++ b/shell.nix @@ -2,11 +2,13 @@ with import <nixpkgs> {}; +with import ./release-common.nix { inherit pkgs; }; + (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; buildInputs = - [ curl bison flex perl libxml2 libxslt + [ curl bison flex libxml2 libxslt bzip2 xz brotli pkgconfig sqlite libsodium boehmgc docbook5 docbook5_xsl @@ -16,15 +18,13 @@ with import <nixpkgs> {}; customMemoryManagement = false; }) autoreconfHook + + # For nix-perl + perl perlPackages.DBDSQLite ]; - configureFlags = - [ "--disable-init-state" - "--enable-gc" - "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; + inherit configureFlags; enableParallelBuilding = true; @@ -32,6 +32,9 @@ with import <nixpkgs> {}; shellHook = '' - configureFlags+=" --prefix=$(pwd)/inst" + export prefix=$(pwd)/inst + configureFlags+=" --prefix=$prefix" + PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH + PATH=$prefix/bin:$PATH ''; } diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp index 3d0b47b4a12e..cdd57fdf2bf1 100644 --- a/src/boost/format/feed_args.hpp +++ b/src/boost/format/feed_args.hpp @@ -41,6 +41,13 @@ namespace { std::streamsize w, const char c, std::ios::fmtflags f, + bool center) + __attribute__ ((unused)); + + void do_pad( std::string & s, + std::streamsize w, + const char c, + std::ios::fmtflags f, bool center) // applies centered / left / right padding to the string s. // Effects : string s is padded. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index d7aee288670a..7ffbdca7c0f4 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,6 +9,7 @@ #include <sys/time.h> #endif +#include "machines.hh" #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" @@ -22,131 +23,56 @@ using std::cin; static void handleAlarm(int sig) { } -class Machine { - const std::set<string> supportedFeatures; - const std::set<string> mandatoryFeatures; - -public: - const string hostName; - const std::vector<string> systemTypes; - const string sshKey; - const unsigned int maxJobs; - const unsigned int speedFactor; - bool enabled; - - bool allSupported(const std::set<string> & features) const { - return std::all_of(features.begin(), features.end(), - [&](const string & feature) { - return supportedFeatures.count(feature) || - mandatoryFeatures.count(feature); - }); - } - - bool mandatoryMet(const std::set<string> & features) const { - return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { - return features.count(feature); - }); - } - - Machine(decltype(hostName) hostName, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures) : - supportedFeatures(supportedFeatures), - mandatoryFeatures(mandatoryFeatures), - hostName(hostName), - systemTypes(systemTypes), - sshKey(sshKey), - maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), - enabled(true) - {}; -};; - -static std::vector<Machine> readConf() +std::string escapeUri(std::string uri) { - auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines"); - - auto machines = std::vector<Machine>{}; - auto lines = std::vector<string>{}; - try { - lines = tokenizeString<std::vector<string>>(readFile(conf), "\n"); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - for (auto line : lines) { - chomp(line); - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) { - continue; - } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 4) - throw FormatError("bad machines.conf file ‘%1%’", conf); - machines.emplace_back(tokens[0], - tokenizeString<std::vector<string>>(tokens[1], ","), - tokens[2], - stoull(tokens[3]), - sz >= 5 ? stoull(tokens[4]) : 1LL, - sz >= 6 ? - tokenizeString<std::set<string>>(tokens[5], ",") : - std::set<string>{}, - sz >= 7 ? - tokenizeString<std::set<string>>(tokens[6], ",") : - std::set<string>{}); - } - return machines; + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; } static string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) { - std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out); - fn_stream << "/"; - for (auto t : m.systemTypes) { - fn_stream << t << "-"; - } - fn_stream << m.hostName << "-" << slot; - return openLockFile(fn_stream.str(), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static char display_env[] = "DISPLAY="; -static char ssh_env[] = "SSH_ASKPASS="; - int main (int argc, char * * argv) { return handleExceptions(argv[0], [&]() { initNix(); /* Ensure we don't get any SSH passphrase or host key popups. */ - if (putenv(display_env) == -1 || - putenv(ssh_env) == -1) - throw SysError("setting SSH env vars"); + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); - if (argc != 4) + if (argc != 6) throw UsageError("called without required arguments"); auto store = openStore(); auto localSystem = argv[1]; - settings.maxSilentTime = stoull(string(argv[2])); - settings.buildTimeout = stoull(string(argv[3])); + settings.maxSilentTime = std::stoll(argv[2]); + settings.buildTimeout = std::stoll(argv[3]); + verbosity = (Verbosity) std::stoll(argv[4]); + settings.builders = argv[5]; - currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load"); + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on OS X. */ + currentLoad = settings.nixStateDir + "/current-load"; std::shared_ptr<Store> sshStore; AutoCloseFD bestSlotLock; - auto machines = readConf(); + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return; + } + string drvPath; - string hostName; + string storeUri; for (string line; getline(cin, line);) { auto tokens = tokenizeString<std::vector<string>>(line); auto sz = tokens.size(); @@ -173,6 +99,8 @@ int main (int argc, char * * argv) Machine * bestMachine = nullptr; unsigned long long bestLoad = 0; for (auto & m : machines) { + debug("considering building on ‘%s’", m.storeUri); + if (m.enabled && std::find(m.systemTypes.begin(), m.systemTypes.end(), neededSystem) != m.systemTypes.end() && @@ -233,16 +161,22 @@ int main (int argc, char * * argv) lock = -1; try { - sshStore = openStore("ssh-ng://" + bestMachine->hostName, - { {"ssh-key", bestMachine->sshKey }, - {"max-connections", "1" } }); - hostName = bestMachine->hostName; + + Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}}; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + } catch (std::exception & e) { printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...", - bestMachine->hostName, e.what()); + bestMachine->storeUri, e.what()); bestMachine->enabled = false; continue; } + goto connected; } } @@ -252,22 +186,32 @@ connected: string line; if (!getline(cin, line)) throw Error("hook caller didn't send inputs"); + auto inputs = tokenizeString<PathSet>(line); if (!getline(cin, line)) throw Error("hook caller didn't send outputs"); + auto outputs = tokenizeString<PathSet>(line); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true); + + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); if (!lockFile(uploadLock.get(), ltWrite, true)) printError("somebody is hogging the upload lock for ‘%s’, continuing..."); alarm(0); signal(SIGALRM, old); - copyPaths(store, ref<Store>(sshStore), inputs); + copyPaths(store, ref<Store>(sshStore), inputs, false, true); uploadLock = -1; - printError("building ‘%s’ on ‘%s’", drvPath, hostName); - sshStore->buildDerivation(drvPath, readDerivation(drvPath)); + BasicDerivation drv(readDerivation(drvPath)); + drv.inputSrcs = inputs; + + printError("building ‘%s’ on ‘%s’", drvPath, storeUri); + auto result = sshStore->buildDerivation(drvPath, drv); + + if (!result.success()) + throw Error("build of ‘%s’ on ‘%s’ failed: %s", drvPath, storeUri, result.errorMsg); PathSet missing; for (auto & path : outputs) @@ -275,7 +219,7 @@ connected: if (!missing.empty()) { setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing); + copyPaths(ref<Store>(sshStore), store, missing, false, true); } return; diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 62d5a010c247..64368a43ff73 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -7,5 +7,3 @@ build-remote_INSTALL_DIR := $(libexecdir)/nix build-remote_LIBS = libmain libutil libformat libstore build-remote_SOURCES := $(d)/build-remote.cc - -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d418ab4e43aa..0cdce602d7b2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -202,7 +202,7 @@ void initGC() GC_INIT(); - GC_oom_fn = oomHandler; + GC_set_oom_fn(oomHandler); /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases @@ -299,7 +299,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) { countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; - restricted = settings.get("restrict-eval", false); + restricted = settings.restrictEval; assert(gcInitialised); @@ -642,7 +642,7 @@ void EvalState::evalFile(const Path & path, Value & v) return; } - Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2); + printTalkative("evaluating file ‘%1%’", path2); Expr * e = parseExprFromFile(checkSourcePath(path2)); try { eval(e, v); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 5342739c53c4..4200e8fd6757 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -3,6 +3,7 @@ #include "eval-inline.hh" #include <cstring> +#include <regex> namespace nix { @@ -262,6 +263,9 @@ static string addToPath(const string & s1, const string & s2) } +static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); + + static void getDerivations(EvalState & state, Value & vIn, const string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, Done & done, @@ -285,7 +289,9 @@ static void getDerivations(EvalState & state, Value & vIn, bound to the attribute with the "lower" name should take precedence). */ for (auto & i : v.attrs->lexicographicOrder()) { - Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i->name); + debug("evaluating attribute ‘%1%’", i->name); + if (!std::regex_match(std::string(i->name), attrRegex)) + continue; string pathPrefix2 = addToPath(pathPrefix, i->name); if (combineChannels) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -304,7 +310,6 @@ static void getDerivations(EvalState & state, Value & vIn, else if (v.isList()) { for (unsigned int n = 0; n < v.listSize(); ++n) { - Activity act(*logger, lvlDebug, "evaluating list element"); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5b1ff0350cd1..40ca77258037 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -142,25 +142,34 @@ or { return OR_KW; } \{ { return '{'; } <INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { PUSH_STATE(STRING); return '"'; } +<INITIAL,INSIDE_DOLLAR_CURLY>\" { + PUSH_STATE(STRING); return '"'; + } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { - /* It is impossible to match strings ending with '$' with one - regex because trailing contexts are only valid at the end - of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); - return STR; - } + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext); + return STR; + } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -<STRING>\" { POP_STATE(); return '"'; } -<STRING>. return yytext[0]; /* just in case: shouldn't be reached */ +<STRING>\" { POP_STATE(); return '"'; } +<STRING>\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } <INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; } -<IND_STRING>\'\'\$ { +<IND_STRING>\'\'\$ | +<IND_STRING>\$ { yylval->e = new ExprIndStr("$"); return IND_STR; } @@ -178,7 +187,6 @@ or { return OR_KW; } yylval->e = new ExprIndStr("'"); return IND_STR; } -<IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ <INITIAL,INSIDE_DOLLAR_CURLY>{ diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 620050a13b05..daa3258f0d3c 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,8 +6,6 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_CXXFLAGS := -Wno-deprecated-register - libexpr_LIBS = libutil libstore libformat libexpr_LDFLAGS = diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index d07eedddaf6b..62982650a22e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -376,7 +376,7 @@ expr_simple $$ = stripIndentation(CUR_POS, data->symbols, *$2); } | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } - | HPATH { $$ = new ExprPath(getEnv("HOME", "") + string{$1 + 1}); } + | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); } | SPATH { string path($1 + 1, strlen($1) - 2); $$ = new ExprApp(CUR_POS, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 615cc8138433..99ffddaeb80c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -127,7 +127,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path); + printTalkative("evaluating file ‘%1%’", path); Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); @@ -326,8 +326,6 @@ typedef list<Value *> ValueList; static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { - Activity act(*logger, lvlDebug, "finding dependencies"); - state.forceAttrs(*args[0], pos); /* Get the start set. */ @@ -499,8 +497,6 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation. */ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) { - Activity act(*logger, lvlVomit, "evaluating derivation"); - state.forceAttrs(*args[0], pos); /* Figure out the name first (for stack backtraces). */ @@ -534,7 +530,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - string outputHash, outputHashAlgo; + std::experimental::optional<std::string> outputHash; + std::string outputHashAlgo; bool outputHashRecursive = false; StringSet outputs; @@ -543,7 +540,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : args[0]->attrs->lexicographicOrder()) { if (i->name == state.sIgnoreNulls) continue; string key = i->name; - Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); + vomit("processing attribute ‘%1%’", key); auto handleHashMode = [&](const std::string & s) { if (s == "recursive") outputHashRecursive = true; @@ -703,7 +700,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * throw EvalError(format("derivation names are not allowed to end in ‘%1%’, at %2%") % drvExtension % posDrvName); - if (outputHash != "") { + if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); @@ -711,13 +708,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = parseHashType(outputHashAlgo); if (ht == htUnknown) throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName); - Hash h = parseHash16or32(ht, outputHash); + Hash h = parseHash16or32(ht, *outputHash); outputHash = printHash(h); if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash); + drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); } else { @@ -1712,26 +1709,33 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, ‘null’ or a list containing substring matches. */ static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::regex regex(state.forceStringNoCtx(*args[0], pos), std::regex::extended); + auto re = state.forceStringNoCtx(*args[0], pos); - PathSet context; - const std::string str = state.forceString(*args[1], context, pos); + try { + std::regex regex(re, std::regex::extended); - std::smatch match; - if (!std::regex_match(str, match, regex)) { - mkNull(v); - return; - } + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); - // the first match is the whole string - const size_t len = match.size() - 1; - state.mkList(v, len); - for (size_t i = 0; i < len; ++i) { - if (!match[i+1].matched) - mkNull(*(v.listElems()[i] = state.allocValue())); - else - mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + std::smatch match; + if (!std::regex_match(str, match, regex)) { + mkNull(v); + return; + } + + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i+1].matched) + mkNull(*(v.listElems()[i] = state.allocValue())); + else + mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + } + + } catch (std::regex_error &) { + throw EvalError("invalid regular expression ‘%s’, at %s", re, pos); } } diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc index 09e2c077baba..3e4ece2cffdc 100644 --- a/src/libexpr/primops/fetchgit.cc +++ b/src/libexpr/primops/fetchgit.cc @@ -17,7 +17,7 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re runProgram("git", true, { "init", "--bare", cacheDir }); } - Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); + //Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); std::string localRef = "pid-" + std::to_string(getpid()); Path localRefFile = cacheDir + "/refs/heads/" + localRef; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 802e8ed2ee75..9df516f062ef 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -220,6 +220,14 @@ static inline void mkApp(Value & v, Value & left, Value & right) } +static inline void mkPrimOpApp(Value & v, Value & left, Value & right) +{ + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} + + static inline void mkStringNoCopy(Value & v, const char * s) { v.type = tString; diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 98693d78a7f4..9a7a893138de 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -22,7 +22,11 @@ MixCommonArgs::MixCommonArgs(const string & programName) [](Strings ss) { auto name = ss.front(); ss.pop_front(); auto value = ss.front(); - settings.set(name, value); + try { + settings.set(name, value); + } catch (UsageError & e) { + warn(e.what()); + } }); } diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 2c0d71edd815..a4de3dccf0a5 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -12,7 +12,7 @@ struct MixCommonArgs : virtual Args struct MixDryRun : virtual Args { - bool dryRun; + bool dryRun = false; MixDryRun() { @@ -20,4 +20,14 @@ struct MixDryRun : virtual Args } }; +struct MixJSON : virtual Args +{ + bool json = false; + + MixJSON() + { + mkFlag(0, "json", "produce JSON output", &json); + } +}; + } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a720afd6cdd4..d6c1c0c9cb49 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -106,8 +106,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - logger = makeDefaultLogger(); - /* Initialise OpenSSL locking. */ opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); @@ -140,9 +138,6 @@ void initNix() struct timeval tv; gettimeofday(&tv, 0); srandom(tv.tv_usec); - - if (char *pack = getenv("_NIX_OPTIONS")) - settings.unpack(pack); } @@ -158,13 +153,13 @@ struct LegacyArgs : public MixCommonArgs &settings.verboseBuild, false); mkFlag('K', "keep-failed", "keep temporary directories of failed builds", - &settings.keepFailed); + &(bool&) settings.keepFailed); mkFlag('k', "keep-going", "keep going after a build fails", - &settings.keepGoing); + &(bool&) settings.keepGoing); mkFlag(0, "fallback", "build from source if substitution fails", []() { - settings.set("build-fallback", "true"); + settings.tryFallback = true; }); mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { @@ -186,7 +181,7 @@ struct LegacyArgs : public MixCommonArgs &settings.readOnlyMode); mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", - &settings.useBuildHook, false); + &(bool&) settings.useBuildHook, false); mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", &settings.showTrace); @@ -220,7 +215,6 @@ void parseCmdLine(int argc, char * * argv, std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) { LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv)); - settings.update(); } @@ -265,7 +259,7 @@ int handleExceptions(const string & programName, std::function<void()> fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - interruptThrown = true; + setInterruptThrown(); throw; } } catch (Exit & e) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 25ad0d75b70a..46c5aa21b2eb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -79,10 +79,7 @@ struct BinaryCacheStoreAccessor : public FSAccessor BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) - , compression(get(params, "compression", "xz")) - , writeNARListing(get(params, "write-nar-listing", "0") == "1") { - auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); @@ -117,11 +114,6 @@ void BinaryCacheStore::init() } } -void BinaryCacheStore::notImpl() -{ - throw Error("operation not implemented for binary cache stores"); -} - std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) { std::promise<std::shared_ptr<std::string>> promise; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index d42b1abd2455..87d4aa43838e 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,20 +13,20 @@ struct NarInfo; class BinaryCacheStore : public Store { -private: +public: - std::unique_ptr<SecretKey> secretKey; + const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - std::string compression; +private: - bool writeNARListing; + std::unique_ptr<SecretKey> secretKey; protected: BinaryCacheStore(const Params & params); - [[noreturn]] void notImpl(); - public: virtual bool fileExists(const std::string & path) = 0; @@ -63,7 +63,7 @@ public: bool isValidPathUncached(const Path & path) override; PathSet queryAllValidPaths() override - { notImpl(); } + { unsupported(); } void queryPathInfoUncached(const Path & path, std::function<void(std::shared_ptr<ValidPathInfo>)> success, @@ -71,16 +71,16 @@ public: void queryReferrers(const Path & path, PathSet & referrers) override - { notImpl(); } + { unsupported(); } PathSet queryDerivationOutputs(const Path & path) override - { notImpl(); } + { unsupported(); } StringSet queryDerivationOutputNames(const Path & path) override - { notImpl(); } + { unsupported(); } Path queryPathFromHashPart(const string & hashPart) override - { notImpl(); } + { unsupported(); } bool wantMassQuery() override { return wantMassQuery_; } @@ -97,32 +97,29 @@ public: void narFromPath(const Path & path, Sink & sink) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { notImpl(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { notImpl(); } + { unsupported(); } void ensurePath(const Path & path) override - { notImpl(); } + { unsupported(); } void addTempRoot(const Path & path) override - { notImpl(); } + { unsupported(); } void addIndirectRoot(const Path & path) override - { notImpl(); } + { unsupported(); } Roots findRoots() override - { notImpl(); } + { unsupported(); } void collectGarbage(const GCOptions & options, GCResults & results) override - { notImpl(); } + { unsupported(); } ref<FSAccessor> getFSAccessor() override; void addSignatures(const Path & storePath, const StringSet & sigs) override - { notImpl(); } + { unsupported(); } std::shared_ptr<std::string> getBuildLog(const Path & path) override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 968e291129f3..44cae3431fc2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -34,13 +34,6 @@ #include <pwd.h> #include <grp.h> -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - /* Includes required for chroot support. */ #if __linux__ #include <sys/socket.h> @@ -127,6 +120,8 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + Activity act; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -175,7 +170,8 @@ public: virtual string key() = 0; protected: - void amDone(ExitCode result); + + virtual void amDone(ExitCode result); }; @@ -469,7 +465,7 @@ UserLock::UserLock() assert(settings.buildUsersGroup != ""); /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist") % settings.buildUsersGroup); @@ -590,11 +586,7 @@ struct HookInstance HookInstance::HookInstance() { - debug("starting build hook"); - - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; - buildHook = canonPath(buildHook); + debug("starting build hook ‘%s’", settings.buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -621,15 +613,17 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(buildHook), + baseNameOf(settings.buildHook), settings.thisSystem, - (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.buildTimeout).str() + std::to_string(settings.maxSilentTime), + std::to_string(settings.buildTimeout), + std::to_string(verbosity), + settings.builders }; - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError(format("executing ‘%1%’") % buildHook); + throw SysError("executing ‘%s’", settings.buildHook); }); pid.setSeparatePG(true); @@ -911,6 +905,12 @@ private: void repairClosure(); + void amDone(ExitCode result) + { + logger->event(evBuildFinished, act, result == ecSuccess); + Goal::amDone(result); + } + void done(BuildResult::Status status, const string & msg = ""); }; @@ -929,6 +929,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut state = &DerivationGoal::getDerivation; name = (format("building of ‘%1%’") % drvPath).str(); trace("created"); + + logger->event(evBuildCreated, act, drvPath); } @@ -944,6 +946,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); trace("created"); + logger->event(evBuildCreated, act, drvPath); + /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ worker.store.addTempRoot(drvPath); @@ -1075,12 +1079,8 @@ void DerivationGoal::haveDerivation() /* Reject doing a hash build of anything other than a fixed-output derivation. */ - if (buildMode == bmHash) { - if (drv->outputs.size() != 1 || - drv->outputs.find("out") == drv->outputs.end() || - drv->outputs["out"].hashAlgo == "") - throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); - } + if (buildMode == bmHash && !drv->isFixedOutput()) + throw Error("cannot do a hash build of non-fixed-output derivation ‘%1%’", drvPath); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1279,7 +1279,7 @@ void DerivationGoal::inputsRealised() /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1575,7 +1575,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; + if (!settings.useBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1608,8 +1608,15 @@ HookReply DerivationGoal::tryBuildHook() debug(format("hook reply is ‘%1%’") % reply); - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + settings.useBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; else if (reply != "accept") throw Error(format("bad hook reply ‘%1%’") % reply); @@ -1628,23 +1635,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - worker.store.computeFSClosure(drvPath, allInputs); - - string s; - for (auto & i : allInputs) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + remote system. */ + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - s = ""; - for (auto & i : missingPaths) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); hook->toHook.writeSide = -1; @@ -1697,12 +1693,7 @@ void DerivationGoal::startBuilder() /* Are we doing a chroot build? */ { - string x = settings.get("build-use-sandbox", - /* deprecated alias */ - settings.get("build-use-chroot", string("false"))); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { + if (settings.sandboxMode == smEnabled) { if (get(drv->env, "__noChroot") == "1") throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); @@ -1713,9 +1704,9 @@ void DerivationGoal::startBuilder() #endif useChroot = true; } - else if (x == "false") + else if (settings.sandboxMode == smDisabled) useChroot = false; - else if (x == "relaxed") + else if (settings.sandboxMode == smRelaxed) useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; } @@ -1739,7 +1730,14 @@ void DerivationGoal::startBuilder() /* In a sandbox, for determinism, always use the same temporary directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#elif __APPLE__ + // On Darwin, we canonize /tmp because its probably a symlink to /private/tmp. tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ @@ -1756,21 +1754,10 @@ void DerivationGoal::startBuilder() if (useChroot) { - string defaultChrootDirs; -#if __linux__ - if (worker.store.isInStore(BASH_PATH)) - defaultChrootDirs = "/bin/sh=" BASH_PATH; -#endif - /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = tokenizeString<StringSet>( - settings.get("build-sandbox-paths", - /* deprecated alias with lower priority */ - settings.get("build-chroot-dirs", defaultChrootDirs))); - PathSet dirs2 = tokenizeString<StringSet>( - settings.get("build-extra-chroot-dirs", - settings.get("build-extra-sandbox-paths", string("")))); + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; dirs.insert(dirs2.begin(), dirs2.end()); dirsInChroot.clear(); @@ -1796,14 +1783,14 @@ void DerivationGoal::startBuilder() try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + } catch (InvalidPath & e) { } catch (Error & e) { throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what()); } for (auto & i : closure) dirsInChroot[i] = i; - string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); - PathSet allowedPaths = tokenizeString<StringSet>(allowed); + PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); @@ -1823,7 +1810,7 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps") % drvPath % i); dirsInChroot[i] = i; } @@ -1859,11 +1846,11 @@ void DerivationGoal::startBuilder() Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - writeFile(chrootRootDir + "/etc/passwd", - (format( - "root:x:0:0:Nix build user:/:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n") % sandboxUid % sandboxGid).str()); + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -1900,6 +1887,7 @@ void DerivationGoal::startBuilder() dirsInChroot[i] = r; else { Path p = chrootRootDir + i; + debug("linking ‘%1%’ to ‘%2%’", p, r); if (link(r.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), @@ -2137,6 +2125,8 @@ void DerivationGoal::startBuilder() } debug(msg); } + + logger->event(evBuildStarted, act); } @@ -2382,10 +2372,8 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); -#ifdef __linux__ if (pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); -#endif ss.push_back("/dev/null"); ss.push_back("/dev/random"); ss.push_back("/dev/tty"); @@ -2414,17 +2402,14 @@ void DerivationGoal::runChild() /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - for (auto & i : dirsInChroot) { - struct stat st; - Path source = i.second.source; - Path target = chrootRootDir + i.first; - if (source == "/proc") continue; // backwards compatibility + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target); + struct stat st; if (stat(source.c_str(), &st) == -1) { - if (i.second.optional && errno == ENOENT) - continue; + if (optional && errno == ENOENT) + return; else - throw SysError(format("getting attributes of path ‘%1%’") % source); + throw SysError("getting attributes of path ‘%1%’", source); } if (S_ISDIR(st.st_mode)) createDirs(target); @@ -2433,7 +2418,12 @@ void DerivationGoal::runChild() writeFile(target, ""); } if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError(format("bind mount from ‘%1%’ to ‘%2%’ failed") % source % target); + throw SysError("bind mount from ‘%1%’ to ‘%2%’ failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ @@ -2444,7 +2434,7 @@ void DerivationGoal::runChild() /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1) + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) throw SysError("mounting /dev/shm"); /* Mount a new devpts on /dev/pts. Note that this @@ -2455,13 +2445,19 @@ void DerivationGoal::runChild() !pathExists(chrootRootDir + "/dev/ptmx") && !dirsInChroot.count("/dev/pts")) { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1) - throw SysError("mounting /dev/pts"); - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", "/dev/pts"); + doBind("/dev/ptmx", "/dev/ptmx"); + } } /* Do the chroot(). */ @@ -2602,7 +2598,7 @@ void DerivationGoal::runChild() sandboxProfile += "(version 1)\n"; /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.get("darwin-log-sandbox-violations", false)) { + if (settings.darwinLogSandboxViolations) { sandboxProfile += "(deny default)\n"; } else { sandboxProfile += "(deny default (with no-log))\n"; @@ -2749,7 +2745,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.get("run-diff-hook", false); + bool runDiffHook = settings.runDiffHook; bool keepPreviousRound = settings.keepFailed || runDiffHook; /* Check whether the output paths were created, and grep each @@ -2876,7 +2872,7 @@ void DerivationGoal::registerOutputs() contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); + debug("scanning for references inside ‘%1%’", path); HashResult hash; PathSet references = scanForReferences(actualPath, allPaths, hash); @@ -2990,7 +2986,7 @@ void DerivationGoal::registerOutputs() ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); - auto diffHook = settings.get("diff-hook", std::string("")); + auto diffHook = settings.diffHook; if (prevExists && diffHook != "" && runDiffHook) { try { auto diff = runProgram(diffHook, true, {prev, i->path}); @@ -3001,7 +2997,7 @@ void DerivationGoal::registerOutputs() } } - if (settings.get("enforce-determinism", true)) + if (settings.enforceDeterminism) throw NotDeterministic(msg); printError(msg); @@ -3051,13 +3047,11 @@ Path DerivationGoal::openLogFile() string baseName = baseNameOf(drvPath); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); createDirs(dir); - Path logFileName = (format("%1%/%2%%3%") - % dir - % string(baseName, 2) - % (settings.compressLog ? ".bz2" : "")).str(); + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName); @@ -3131,7 +3125,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide.get()) - printError(data); // FIXME? + printError(chomp(data)); } @@ -3151,6 +3145,7 @@ void DerivationGoal::flushLine() logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); } + logger->event(evBuildOutput, act, currentLogLine); currentLogLine = ""; currentLogLinePos = 0; } @@ -3265,6 +3260,12 @@ public: void handleEOF(int fd); Path getStorePath() { return storePath; } + + void amDone(ExitCode result) + { + logger->event(evSubstitutionFinished, act, result == ecSuccess); + Goal::amDone(result); + } }; @@ -3277,6 +3278,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool state = &SubstitutionGoal::init; name = (format("substitution of ‘%1%’") % storePath).str(); trace("created"); + logger->event(evSubstitutionCreated, act, storePath); } @@ -3402,16 +3404,18 @@ void SubstitutionGoal::tryToRun() trace("trying to run"); /* Make sure that we are allowed to start a build. Note that even - is maxBuildJobs == 0 (no local builds allowed), we still allow + if maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } printInfo(format("fetching path ‘%1%’...") % storePath); + logger->event(evSubstitutionStarted, act); + outPipe.create(); promise = std::promise<void>(); @@ -3658,7 +3662,7 @@ void Worker::run(const Goals & _topGoals) { for (auto & i : _topGoals) topGoals.insert(i); - Activity act(*logger, lvlDebug, "entered goal loop"); + debug("entered goal loop"); while (1) { @@ -3686,7 +3690,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && settings.maxBuildJobs == 0) throw Error( + if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( "unable to start any build; either increase ‘--max-jobs’ " "or enable distributed builds"); assert(!awake.empty()); @@ -3723,9 +3727,9 @@ void Worker::waitForInput() auto nearest = steady_time_point::max(); // nearest deadline for (auto & i : children) { if (!i.respectTimeouts) continue; - if (settings.maxSilentTime != 0) + if (0 != settings.maxSilentTime) nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (settings.buildTimeout != 0) + if (0 != settings.buildTimeout) nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { @@ -3803,7 +3807,7 @@ void Worker::waitForInput() } if (goal->getExitCode() == Goal::ecBusy && - settings.maxSilentTime != 0 && + 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { @@ -3814,7 +3818,7 @@ void Worker::waitForInput() } else if (goal->getExitCode() == Goal::ecBusy && - settings.buildTimeout != 0 && + 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index c5dbd57f8bc6..8a5cf3327d44 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -28,9 +28,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) DownloadRequest request(url); request.verifyTLS = false; - /* Show a progress indicator, even though stderr is not a tty. */ - request.showProgress = DownloadRequest::yes; - /* Note: have to use a fresh downloader here because we're in a forked process. */ auto data = makeDownloader()->download(request); diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 0fc86a1fe921..f56a6adab9c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -105,14 +105,12 @@ PublicKeys getDefaultPublicKeys() // FIXME: filter duplicates - for (auto s : settings.get("binary-cache-public-keys", - Strings{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="})) - { + for (auto s : settings.binaryCachePublicKeys.get()) { PublicKey key(s); publicKeys.emplace(key.name, key); } - for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + for (auto secretKeyFile : settings.secretKeyFiles.get()) { try { SecretKey secretKey(readFile(secretKeyFile)); publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index d1f760fdc301..63e498f0603a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -63,6 +63,7 @@ struct CurlDownloader : public Downloader CurlDownloader & downloader; DownloadRequest request; DownloadResult result; + Activity act; bool done = false; // whether either the success or failure function has been called std::function<void(const DownloadResult &)> success; std::function<void(std::exception_ptr exc)> failure; @@ -70,10 +71,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - bool showProgress = false; - double prevProgressTime{0}, startTime{0}; - unsigned int moveBack{1}; - unsigned int attempt = 0; /* Don't start this download until the specified time point @@ -87,12 +84,10 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) : downloader(downloader), request(request) { - showProgress = - request.showProgress == DownloadRequest::yes || - (request.showProgress == DownloadRequest::automatic && isatty(STDERR_FILENO)); - if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + + logger->event(evDownloadCreated, act, request.uri); } ~DownloadItem() @@ -109,6 +104,7 @@ struct CurlDownloader : public Downloader } catch (...) { ignoreException(); } + logger->event(evDownloadDestroyed, act); } template<class T> @@ -171,19 +167,7 @@ struct CurlDownloader : public Downloader int progressCallback(double dltotal, double dlnow) { - if (showProgress) { - double now = getTime(); - if (prevProgressTime <= now - 1) { - string s = (format(" [%1$.0f/%2$.0f KiB, %3$.1f KiB/s]") - % (dlnow / 1024.0) - % (dltotal / 1024.0) - % (now == startTime ? 0 : dlnow / 1024.0 / (now - startTime))).str(); - std::cerr << "\e[" << moveBack << "D" << s; - moveBack = s.size(); - std::cerr.flush(); - prevProgressTime = now; - } - } + logger->event(evDownloadProgress, act, dltotal, dlnow); return _isInterrupted; } @@ -201,13 +185,6 @@ struct CurlDownloader : public Downloader void init() { - // FIXME: handle parallel downloads. - if (showProgress) { - std::cerr << (format("downloading ‘%1%’... ") % request.uri); - std::cerr.flush(); - startTime = getTime(); - } - if (!req) req = curl_easy_init(); curl_easy_reset(req); @@ -220,7 +197,9 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion).c_str()); + curl_easy_setopt(req, CURLOPT_USERAGENT, + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif @@ -249,9 +228,11 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get()); + /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); result.data = std::make_shared<std::string>(); @@ -259,10 +240,6 @@ struct CurlDownloader : public Downloader void finish(CURLcode code) { - if (showProgress) - //std::cerr << "\e[" << moveBack << "D\e[K\n"; - std::cerr << "\n"; - long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); @@ -288,6 +265,7 @@ struct CurlDownloader : public Downloader try { result.data = decodeContent(encoding, ref<std::string>(result.data)); callSuccess(success, failure, const_cast<const DownloadResult &>(result)); + logger->event(evDownloadSucceeded, act, result.data->size()); } catch (...) { done = true; callFailure(failure, std::current_exception()); @@ -300,6 +278,11 @@ struct CurlDownloader : public Downloader || httpStatus == 504 || httpStatus == 522 || httpStatus == 524 || code == CURLE_COULDNT_RESOLVE_HOST || code == CURLE_RECV_ERROR + + // this seems to occur occasionally for retriable reasons, and shows up in an error like this: + // curl: (23) Failed writing body (315 != 16366) + || code == CURLE_WRITE_ERROR + // this is a generic SSL failure that in some cases (e.g., certificate error) is permanent but also appears in transient cases, so we consider it retryable || code == CURLE_SSL_CONNECT_ERROR #if LIBCURL_VERSION_NUM >= 0x073200 @@ -364,9 +347,9 @@ struct CurlDownloader : public Downloader curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); #endif curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - settings.get("binary-caches-parallel-connections", 25)); + settings.binaryCachesParallelConnections.get()); - enableHttp2 = settings.get("enable-http2", true); + enableHttp2 = settings.enableHttp2; wakeupPipe.create(); fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); @@ -606,7 +589,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa string expectedETag; - int ttl = settings.get("tarball-ttl", 60 * 60); + int ttl = settings.tarballTtl; bool skip = false; if (pathExists(fileLink) && pathExists(dataFile)) { @@ -645,6 +628,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(false, hash); store->addToStore(info, sink.s, false, true); storePath = info.path; @@ -682,7 +666,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa } if (expectedStorePath != "" && storePath != expectedStorePath) - throw nix::Error(format("hash mismatch in file downloaded from ‘%s’") % url); + throw nix::Error("store path mismatch in file downloaded from ‘%s’", url); return storePath; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index e2e16b361036..7d8982d64c4c 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -13,9 +13,8 @@ struct DownloadRequest std::string uri; std::string expectedETag; bool verifyTLS = true; - enum { yes, no, automatic } showProgress = yes; bool head = false; - size_t tries = 1; + size_t tries = 5; unsigned int baseRetryTimeMs = 250; DownloadRequest(const std::string & uri) : uri(uri) { } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 2b8ab063e18e..6e8bc692cdff 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -30,13 +30,13 @@ void Store::exportPaths(const Paths & paths, Sink & sink) std::reverse(sorted.begin(), sorted.end()); std::string doneLabel("paths exported"); - logger->incExpected(doneLabel, sorted.size()); + //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path); + //Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path); sink << 1; exportPath(path, sink); - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); } sink << 0; @@ -81,7 +81,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, info.path = readStorePath(*this, source); - Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path); + //Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path); info.references = readStorePaths<PathSet>(*this, source); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8e90913cc3f1..3cdbb114a79d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -426,22 +426,26 @@ void LocalStore::findRuntimeRoots(PathSet & roots) throw SysError("iterating /proc"); } +#if !defined(__linux__) try { - auto lsofRegex = std::regex(R"(^n(/.*)$)"); + std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = - tokenizeString<std::vector<string>>(runProgram("lsof", true, { "-n", "-w", "-F", "n" }), "\n"); + tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); for (const auto & line : lsofLines) { - auto match = std::smatch{}; + std::smatch match; if (std::regex_match(line, match, lsofRegex)) paths.emplace(match[1]); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ } +#endif +#if defined(__linux__) readFileRoots("/proc/sys/kernel/modprobe", paths); readFileRoots("/proc/sys/kernel/fbsplash", paths); readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); +#endif for (auto & i : paths) if (isInStore(i)) { @@ -611,7 +615,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) auto realPath = realStoreDir + "/" + baseNameOf(path); if (realPath == linksDir || realPath == trashDir) return; - Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); + //Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); if (!isStorePath(path) || !isValidPath(path)) { /* A lock file belonging to a path that we're building right @@ -679,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting ‘%1%’") % path); - state.results.bytesFreed += st.st_blocks * 512; + state.results.bytesFreed += st.st_blocks * 512ULL; } struct stat st; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8c900be77b8f..3dd2508a26d3 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,6 +1,7 @@ #include "globals.hh" #include "util.hh" #include "archive.hh" +#include "args.hh" #include <algorithm> #include <map> @@ -17,255 +18,91 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" +#else + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" +#endif Settings settings; - Settings::Settings() + : Config({}) + , nixPrefix(NIX_PREFIX) + , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) + , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) + , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { - nixPrefix = NIX_PREFIX; - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); - nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); - nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); - nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); - nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); - nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); - nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); - - // should be set with the other config options, but depends on nixLibexecDir -#ifdef __APPLE__ - preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies"; -#endif - - keepFailed = false; - keepGoing = false; - tryFallback = false; - maxBuildJobs = 1; - buildCores = std::max(1U, std::thread::hardware_concurrency()); - readOnlyMode = false; - thisSystem = SYSTEM; - maxSilentTime = 0; - buildTimeout = 0; - useBuildHook = true; - reservedSize = 8 * 1024 * 1024; - fsyncMetadata = true; - useSQLiteWAL = true; - syncBeforeRegistering = false; - useSubstitutes = true; buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - useSshSubstituter = true; - impersonateLinux26 = false; - keepLog = true; - compressLog = true; - maxLogSize = 0; - pollInterval = 5; - checkRootReachability = false; - gcKeepOutputs = false; - gcKeepDerivations = true; - autoOptimiseStore = false; - envKeepDerivations = false; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; - showTrace = false; - enableNativeCode = false; - netrcFile = fmt("%s/%s", nixConfDir, "netrc"); caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt")); - enableImportFromDerivation = true; -} - - -void Settings::loadConfFile() -{ - Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); - if (!pathExists(settingsFile)) return; - string contents = readFile(settingsFile); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector<string> tokens = tokenizeString<vector<string> >(line); - if (tokens.empty()) continue; - if (tokens.size() < 2 || tokens[1] != "=") - throw Error(format("illegal configuration line ‘%1%’ in ‘%2%’") % line % settingsFile); - - string name = tokens[0]; - - vector<string>::iterator i = tokens.begin(); - advance(i, 2); - settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow - }; -} - - -void Settings::set(const string & name, const string & value) -{ - settings[name] = value; - overrides[name] = value; -} - - -string Settings::get(const string & name, const string & def) -{ - auto i = settings.find(name); - if (i == settings.end()) return def; - return i->second; -} - - -Strings Settings::get(const string & name, const Strings & def) -{ - auto i = settings.find(name); - if (i == settings.end()) return def; - return tokenizeString<Strings>(i->second); -} - - -bool Settings::get(const string & name, bool def) -{ - bool res = def; - _get(res, name); - return res; -} + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s != "") builderFiles = tokenizeString<Strings>(s, ":"); +#if defined(__linux__) && defined(SANDBOX_SHELL) + sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); +#endif -int Settings::get(const string & name, int def) -{ - int res = def; - _get(res, name); - return res; + allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); } - -void Settings::update() +void Settings::loadConfFile() { - _get(tryFallback, "build-fallback"); + applyConfigFile(nixConfDir + "/nix.conf"); - auto s = get("build-max-jobs", std::string("1")); - if (s == "auto") - maxBuildJobs = std::max(1U, std::thread::hardware_concurrency()); - else - if (!string2Int(s, maxBuildJobs)) - throw Error("configuration setting ‘build-max-jobs’ should be ‘auto’ or an integer"); + /* We only want to send overrides to the daemon, i.e. stuff from + ~/.nix/nix.conf or the command line. */ + resetOverriden(); - _get(buildCores, "build-cores"); - _get(thisSystem, "system"); - _get(maxSilentTime, "build-max-silent-time"); - _get(buildTimeout, "build-timeout"); - _get(reservedSize, "gc-reserved-space"); - _get(fsyncMetadata, "fsync-metadata"); - _get(useSQLiteWAL, "use-sqlite-wal"); - _get(syncBeforeRegistering, "sync-before-registering"); - _get(useSubstitutes, "build-use-substitutes"); - _get(buildUsersGroup, "build-users-group"); - _get(impersonateLinux26, "build-impersonate-linux-26"); - _get(keepLog, "build-keep-log"); - _get(compressLog, "build-compress-log"); - _get(maxLogSize, "build-max-log-size"); - _get(pollInterval, "build-poll-interval"); - _get(checkRootReachability, "gc-check-reachability"); - _get(gcKeepOutputs, "gc-keep-outputs"); - _get(gcKeepDerivations, "gc-keep-derivations"); - _get(autoOptimiseStore, "auto-optimise-store"); - _get(envKeepDerivations, "env-keep-derivations"); - _get(sshSubstituterHosts, "ssh-substituter-hosts"); - _get(useSshSubstituter, "use-ssh-substituter"); - _get(enableNativeCode, "allow-unsafe-native-code-during-evaluation"); - _get(useCaseHack, "use-case-hack"); - _get(preBuildHook, "pre-build-hook"); - _get(keepGoing, "keep-going"); - _get(keepFailed, "keep-failed"); - _get(netrcFile, "netrc-file"); - _get(enableImportFromDerivation, "allow-import-from-derivation"); + applyConfigFile(getConfigDir() + "/nix/nix.conf"); } - -void Settings::_get(string & res, const string & name) +void Settings::set(const string & name, const string & value) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = i->second; + Config::set(name, value); } - -void Settings::_get(bool & res, const string & name) +unsigned int Settings::getDefaultCores() { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (i->second == "true") res = true; - else if (i->second == "false") res = false; - else throw Error(format("configuration option ‘%1%’ should be either ‘true’ or ‘false’, not ‘%2%’") - % name % i->second); + return std::max(1U, std::thread::hardware_concurrency()); } +const string nixVersion = PACKAGE_VERSION; -void Settings::_get(StringSet & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res.clear(); - Strings ss = tokenizeString<Strings>(i->second); - res.insert(ss.begin(), ss.end()); -} - -void Settings::_get(Strings & res, const string & name) +template<> void BaseSetting<SandboxMode>::set(const std::string & str) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = tokenizeString<Strings>(i->second); + if (str == "true") value = smEnabled; + else if (str == "relaxed") value = smRelaxed; + else if (str == "false") value = smDisabled; + else throw UsageError("option '%s' has invalid value '%s'", name, str); } - -template<class N> void Settings::_get(N & res, const string & name) +template<> std::string BaseSetting<SandboxMode>::to_string() { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (!string2Int(i->second, res)) - throw Error(format("configuration setting ‘%1%’ should have an integer value") % name); + if (value == smEnabled) return "true"; + else if (value == smRelaxed) return "relaxed"; + else if (value == smDisabled) return "false"; + else abort(); } - -string Settings::pack() +template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) { - string s; - for (auto & i : settings) { - if (i.first.find('\n') != string::npos || - i.first.find('=') != string::npos || - i.second.find('\n') != string::npos) - throw Error("illegal option name/value"); - s += i.first; s += '='; s += i.second; s += '\n'; - } - return s; + AbstractSetting::toJSON(out); } - -void Settings::unpack(const string & pack) { - Strings lines = tokenizeString<Strings>(pack, "\n"); - for (auto & i : lines) { - string::size_type eq = i.find('='); - if (eq == string::npos) - throw Error("illegal option name/value"); - set(i.substr(0, eq), i.substr(eq + 1)); - } -} - - -Settings::SettingsMap Settings::getOverrides() +void MaxBuildJobsSetting::set(const std::string & str) { - return overrides; + if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + else if (!string2Int(str, value)) + throw UsageError("configuration setting ‘%s’ should be ‘auto’ or an integer", name); } - -const string nixVersion = PACKAGE_VERSION; - - } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ccec300f776e..af37ec61d7a1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,7 +1,7 @@ #pragma once #include "types.hh" -#include "logging.hh" +#include "config.hh" #include <map> #include <sys/types.h> @@ -9,10 +9,48 @@ namespace nix { +typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; + +extern bool useCaseHack; // FIXME + +struct CaseHackSetting : public BaseSetting<bool> +{ + CaseHackSetting(Config * options, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<bool>(useCaseHack, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str) override + { + BaseSetting<bool>::set(str); + nix::useCaseHack = true; + } +}; + +struct MaxBuildJobsSetting : public BaseSetting<unsigned int> +{ + MaxBuildJobsSetting(Config * options, + unsigned int def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<unsigned int>(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str) override; +}; -struct Settings { +class Settings : public Config { - typedef std::map<string, string> SettingsMap; + unsigned int getDefaultCores(); + +public: Settings(); @@ -20,29 +58,13 @@ struct Settings { void set(const string & name, const string & value); - string get(const string & name, const string & def); - - Strings get(const string & name, const Strings & def); - - bool get(const string & name, bool def); - - int get(const string & name, int def); - - void update(); - - string pack(); - - void unpack(const string & pack); - - SettingsMap getOverrides(); + Path nixPrefix; /* The directory where we store sources and derived files. */ Path nixStore; Path nixDataDir; /* !!! fix */ - Path nixPrefix; - /* The directory where we log various operations. */ Path nixLogDir; @@ -61,17 +83,14 @@ struct Settings { /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; - /* Whether to keep temporary directories of failed builds. */ - bool keepFailed; + Setting<bool> keepFailed{this, false, "keep-failed", + "Whether to keep temporary directories of failed builds."}; - /* Whether to keep building subgoals when a sibling (another - subgoal of the same goal) fails. */ - bool keepGoing; + Setting<bool> keepGoing{this, false, "keep-going", + "Whether to keep building derivations when another build fails."}; - /* Whether, if we cannot realise the known closure corresponding - to a derivation, we should try to normalise the derivation - instead. */ - bool tryFallback; + Setting<bool> tryFallback{this, false, "build-fallback", + "Whether to fall back to building when substitution fails."}; /* Whether to show build log output in real time. */ bool verboseBuild = true; @@ -80,132 +99,228 @@ struct Settings { the log to show if a build fails. */ size_t logLines = 10; - /* Maximum number of parallel build jobs. 0 means unlimited. */ - unsigned int maxBuildJobs; + MaxBuildJobsSetting maxBuildJobs{this, 1, "build-max-jobs", + "Maximum number of parallel build jobs. \"auto\" means use number of cores."}; - /* Number of CPU cores to utilize in parallel within a build, - i.e. by passing this number to Make via '-j'. 0 means that the - number of actual CPU cores on the local host ought to be - auto-detected. */ - unsigned int buildCores; + Setting<unsigned int> buildCores{this, getDefaultCores(), "build-cores", + "Number of CPU cores to utilize in parallel within a build, " + "i.e. by passing this number to Make via '-j'. 0 means that the " + "number of actual CPU cores on the local host ought to be " + "auto-detected."}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ - bool readOnlyMode; + bool readOnlyMode = false; + + Setting<std::string> thisSystem{this, SYSTEM, "system", + "The canonical Nix system name."}; - /* The canonical system name, as returned by config.guess. */ - string thisSystem; + Setting<time_t> maxSilentTime{this, 0, "build-max-silent-time", + "The maximum time in seconds that a builer can go without " + "producing any output on stdout/stderr before it is killed. " + "0 means infinity."}; - /* The maximum time in seconds that a builer can go without - producing any output on stdout/stderr before it is killed. 0 - means infinity. */ - time_t maxSilentTime; + Setting<time_t> buildTimeout{this, 0, "build-timeout", + "The maximum duration in seconds that a builder can run. " + "0 means infinity."}; - /* The maximum duration in seconds that a builder can run. 0 - means infinity. */ - time_t buildTimeout; + Setting<bool> useBuildHook{this, true, "remote-builds", + "Whether to use build hooks (for distributed builds)."}; - /* Whether to use build hooks (for distributed builds). Sometimes - users want to disable this from the command-line. */ - bool useBuildHook; + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + "The path of the helper program that executes builds to remote machines."}; - /* Amount of reserved space for the garbage collector - (/nix/var/nix/db/reserved). */ - off_t reservedSize; + Setting<std::string> builders{this, "", "builders", + "A semicolon-separated list of build machines, in the format of nix.machines."}; - /* Whether SQLite should use fsync. */ - bool fsyncMetadata; + Setting<Strings> builderFiles{this, + {nixConfDir + "/machines"}, "builder-files", + "A list of files specifying build machines."}; - /* Whether SQLite should use WAL mode. */ - bool useSQLiteWAL; + Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", + "Amount of reserved disk space for the garbage collector."}; - /* Whether to call sync() before registering a path as valid. */ - bool syncBeforeRegistering; + Setting<bool> fsyncMetadata{this, true, "fsync-metadata", + "Whether SQLite should use fsync()."}; - /* Whether to use substitutes. */ - bool useSubstitutes; + Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", + "Whether SQLite should use WAL mode."}; - /* The Unix group that contains the build users. */ - string buildUsersGroup; + Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", + "Whether to call sync() before registering a path as valid."}; - /* Set of ssh connection strings for the ssh substituter */ - Strings sshSubstituterHosts; + Setting<bool> useSubstitutes{this, true, "build-use-substitutes", + "Whether to use substitutes."}; - /* Whether to use the ssh substituter at all */ - bool useSshSubstituter; + Setting<std::string> buildUsersGroup{this, "", "build-users-group", + "The Unix group that contains the build users."}; - /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ - bool impersonateLinux26; + Setting<bool> impersonateLinux26{this, false, "build-impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels."}; - /* Whether to store build logs. */ - bool keepLog; + Setting<bool> keepLog{this, true, "build-keep-log", + "Whether to store build logs."}; - /* Whether to compress logs. */ - bool compressLog; + Setting<bool> compressLog{this, true, "build-compress-log", + "Whether to compress logs."}; - /* Maximum number of bytes a builder can write to stdout/stderr - before being killed (0 means no limit). */ - unsigned long maxLogSize; + Setting<unsigned long> maxLogSize{this, 0, "build-max-log-size", + "Maximum number of bytes a builder can write to stdout/stderr " + "before being killed (0 means no limit)."}; /* When build-repeat > 0 and verboseBuild == true, whether to print repeated builds (i.e. builds other than the first one) to stderr. Hack to prevent Hydra logs from being polluted. */ bool printRepeatedBuilds = true; - /* How often (in seconds) to poll for locks. */ - unsigned int pollInterval; + Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", + "How often (in seconds) to poll for locks."}; - /* Whether to check if new GC roots can in fact be found by the - garbage collector. */ - bool checkRootReachability; + Setting<bool> checkRootReachability{this, false, "gc-check-reachability", + "Whether to check if new GC roots can in fact be found by the " + "garbage collector."}; - /* Whether the garbage collector should keep outputs of live - derivations. */ - bool gcKeepOutputs; + Setting<bool> gcKeepOutputs{this, false, "gc-keep-outputs", + "Whether the garbage collector should keep outputs of live derivations."}; - /* Whether the garbage collector should keep derivers of live - paths. */ - bool gcKeepDerivations; + Setting<bool> gcKeepDerivations{this, true, "gc-keep-derivations", + "Whether the garbage collector should keep derivers of live paths."}; - /* Whether to automatically replace files with identical contents - with hard links. */ - bool autoOptimiseStore; + Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", + "Whether to automatically replace files with identical contents with hard links."}; - /* Whether to add derivations as a dependency of user environments - (to prevent them from being GCed). */ - bool envKeepDerivations; + Setting<bool> envKeepDerivations{this, false, "env-keep-derivations", + "Whether to add derivations as a dependency of user environments " + "(to prevent them from being GCed)."}; /* Whether to lock the Nix client and worker to the same CPU. */ bool lockCPU; /* Whether to show a stack trace if Nix evaluation fails. */ - bool showTrace; + bool showTrace = false; + + Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting<SandboxMode> sandboxMode{this, smDisabled, "build-use-sandbox", + "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", + {"build-use-chroot"}}; + + Setting<PathSet> sandboxPaths{this, {}, "build-sandbox-paths", + "The paths to make available inside the build sandbox.", + {"build-chroot-dirs"}}; + + Setting<PathSet> extraSandboxPaths{this, {}, "build-extra-sandbox-paths", + "Additional paths to make available inside the build sandbox.", + {"build-extra-chroot-dirs"}}; + + Setting<bool> restrictEval{this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and to disallow fetching files from the network."}; - /* Whether native-code enabling primops should be enabled */ - bool enableNativeCode; + Setting<size_t> buildRepeat{this, 0, "build-repeat", + "The number of times to repeat a build in order to verify determinism."}; - /* The hook to run just before a build to set derivation-specific - build settings */ - Path preBuildHook; +#if __linux__ + Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", + "The size of /dev/shm in the build sandbox."}; - /* Path to the netrc file used to obtain usernames/passwords for - downloads. */ - Path netrcFile; + Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", + "The build directory inside the sandbox."}; +#endif + + Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; + +#if __APPLE__ + Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", + "Whether to log Darwin sandbox access violations to the system log."}; +#endif + + Setting<bool> runDiffHook{this, false, "run-diff-hook", + "Whether to run the program specified by the diff-hook setting " + "repeated builds produce a different result. Typically used to " + "plug in diffoscope."}; + + PathSetting diffHook{this, true, "", "diff-hook", + "A program that prints out the differences between the two paths " + "specified on its command line."}; + + Setting<bool> enforceDeterminism{this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output."}; + + Setting<Strings> binaryCachePublicKeys{this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "binary-cache-public-keys", + "Trusted public keys for secure substitution."}; + + Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", + "Secret keys with which to sign local builds."}; + + Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<bool> enableHttp2{this, true, "enable-http2", + "Whether to enable HTTP/2 support."}; + + Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", + "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + + Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches", + "Obsolete."}; + + Setting<Strings> substituters{this, + nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + "substituters", + "The URIs of substituters (such as https://cache.nixos.org/).", + {"binary-caches"}}; + + // FIXME: provide a way to add to option values. + Setting<Strings> extraSubstituters{this, {}, "extra-substituters", + "Additional URIs of substituters.", + {"extra-binary-caches"}}; + + Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", + "Disabled substituters that may be enabled via the substituters option by untrusted users.", + {"trusted-binary-caches"}}; + + Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", + "Which users or groups are trusted to ask the daemon to do unsafe things."}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", + "Which users or groups are allowed to connect to the daemon."}; + + Setting<bool> printMissing{this, true, "print-missing", + "Whether to print what paths need to be built or downloaded."}; + + Setting<std::string> preBuildHook{this, +#if __APPLE__ + nixLibexecDir + "/nix/resolve-system-dependencies", +#else + "", +#endif + "pre-build-hook", + "A program to run just before a build to set derivation-specific build settings."}; + + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", + "Path to the netrc file used to obtain usernames/passwords for downloads."}; /* Path to the SSL CA file used */ Path caFile; - /* Whether we allow import-from-derivation */ - bool enableImportFromDerivation; + Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + CaseHackSetting useCaseHack{this, "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; -private: - SettingsMap settings, overrides; + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - void _get(string & res, const string & name); - void _get(bool & res, const string & name); - void _get(StringSet & res, const string & name); - void _get(Strings & res, const string & name); - template<class N> void _get(N & res, const string & name); + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 37a7d6ace142..cead81514ab4 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -50,7 +50,6 @@ protected: { try { DownloadRequest request(cacheUri + "/" + path); - request.showProgress = DownloadRequest::no; request.head = true; request.tries = 5; getDownloader()->download(request); @@ -76,7 +75,6 @@ protected: std::function<void(std::exception_ptr exc)> failure) override { DownloadRequest request(cacheUri + "/" + path); - request.showProgress = DownloadRequest::no; request.tries = 8; getDownloader()->enqueueDownload(request, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0e838846c794..e09932e3d182 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "worker-protocol.hh" #include "ssh.hh" +#include "derivations.hh" namespace nix { @@ -12,11 +13,19 @@ static std::string uriScheme = "ssh://"; struct LegacySSHStore : public Store { + const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; + + // Hack for getting remote build log output. + const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + struct Connection { std::unique_ptr<SSHMaster::Connection> sshConn; FdSink to; FdSource from; + int remoteVersion; }; std::string host; @@ -29,16 +38,17 @@ struct LegacySSHStore : public Store : Store(params) , host(host) , connections(make_ref<Pool<Connection>>( - std::max(1, std::stoi(get(params, "max-connections", "1"))), + std::max(1, (int) maxConnections), [this]() { return openConnection(); }, [](const ref<Connection> & r) { return true; } )) , master( host, - get(params, "ssh-key", ""), + sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - get(params, "compress", "") == "true") + compress, + logFD) { } @@ -49,8 +59,6 @@ struct LegacySSHStore : public Store conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - int remoteVersion; - try { conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; conn->to.flush(); @@ -58,8 +66,8 @@ struct LegacySSHStore : public Store unsigned int magic = readInt(conn->from); if (magic != SERVE_MAGIC_2) throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host); - remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host); } catch (EndOfFile & e) { @@ -144,12 +152,6 @@ struct LegacySSHStore : public Store sink(*savedNAR.data); } - /* Unsupported methods. */ - [[noreturn]] void unsupported() - { - throw Error("operation not supported on SSH stores"); - } - PathSet queryAllValidPaths() override { unsupported(); } void queryReferrers(const Path & path, PathSet & referrers) override @@ -173,12 +175,36 @@ struct LegacySSHStore : public Store const PathSet & references, bool repair) override { unsupported(); } - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { unsupported(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { unsupported(); } + { + auto conn(connections->get()); + + conn->to + << cmdBuildDerivation + << drvPath + << drv + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to + << settings.buildRepeat + << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status) readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + + return status; + } void ensurePath(const Path & path) override { unsupported(); } @@ -201,9 +227,6 @@ struct LegacySSHStore : public Store void addSignatures(const Path & storePath, const StringSet & sigs) override { unsupported(); } - bool isTrusted() override - { return true; } - void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -239,6 +262,11 @@ struct LegacySSHStore : public Store return readStorePaths<PathSet>(*this, conn->from); } + + void connect() override + { + auto conn(connections->get()); + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 57e1b8a09fe6..bf28a1c70c62 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -9,9 +9,6 @@ namespace nix { LocalFSStore::LocalFSStore(const Params & params) : Store(params) - , rootDir(get(params, "root")) - , stateDir(canonPath(get(params, "state", rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir))) - , logDir(canonPath(get(params, "log", rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir))) { } @@ -34,7 +31,7 @@ struct LocalStoreAccessor : public FSAccessor auto realPath = toRealPath(path); struct stat st; - if (lstat(path.c_str(), &st)) { + if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; throw SysError(format("getting status of ‘%1%’") % path); } @@ -54,7 +51,7 @@ struct LocalStoreAccessor : public FSAccessor { auto realPath = toRealPath(path); - auto entries = nix::readDirectory(path); + auto entries = nix::readDirectory(realPath); StringSet res; for (auto & entry : entries) @@ -76,7 +73,8 @@ struct LocalStoreAccessor : public FSAccessor ref<FSAccessor> LocalFSStore::getFSAccessor() { - return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); + return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( + std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } void LocalFSStore::narFromPath(const Path & path, Sink & sink) @@ -88,6 +86,8 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink) const string LocalFSStore::drvsLogDir = "drvs"; + + std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) { auto path(path_); @@ -110,8 +110,8 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) Path logPath = j == 0 - ? (format("%1%/%2%/%3%/%4%") % logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() - : (format("%1%/%2%/%3%") % logDir % drvsLogDir % baseName).str(); + ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8610841d7229..207e8a40b6d3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -38,13 +38,14 @@ namespace nix { LocalStore::LocalStore(const Params & params) : Store(params) , LocalFSStore(params) - , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir)) + , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"} + , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") , trashDir(realStoreDir + "/trash") - , requireSigs(trim(settings.get("signed-binary-caches", std::string("*"))) != "") // FIXME: rename option , publicKeys(getDefaultPublicKeys()) { auto state(_state.lock()); @@ -74,7 +75,7 @@ LocalStore::LocalStore(const Params & params) mode_t perm = 01775; - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError(format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist") % settings.buildUsersGroup); @@ -914,10 +915,16 @@ void LocalStore::invalidatePath(State & state, const Path & path) void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { + assert(info.narHash); + Hash h = hashString(htSHA256, *nar); if (h != info.narHash) - throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % - info.path % info.narHash.to_string() % h.to_string()); + throw Error("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’", + info.path, info.narHash.to_string(), h.to_string()); + + if (nar->size() != info.narSize) + throw Error("szie mismatch importing path ‘%s’; expected %s, got %s", + info.path, info.narSize, nar->size()); if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys)) throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path); @@ -1003,7 +1010,6 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, info.path = dstPath; info.narHash = hash.first; info.narSize = hash.second; - info.ultimate = true; info.ca = makeFixedOutputCA(recursive, h); registerValidPath(info); } @@ -1066,7 +1072,6 @@ Path LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = references; - info.ultimate = true; info.ca = "text:" + hash.to_string(); registerValidPath(info); } @@ -1332,9 +1337,9 @@ void LocalStore::signPathInfo(ValidPathInfo & info) { // FIXME: keep secret keys in memory. - auto secretKeyFiles = settings.get("secret-key-files", Strings()); + auto secretKeyFiles = settings.secretKeyFiles; - for (auto & secretKeyFile : secretKeyFiles) { + for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); info.sign(secretKey); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 28e9a31c9feb..f2c40e96464b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -26,14 +26,9 @@ struct Derivation; struct OptimiseStats { - unsigned long filesLinked; - unsigned long long bytesFreed; - unsigned long long blocksFreed; - OptimiseStats() - { - filesLinked = 0; - bytesFreed = blocksFreed = 0; - } + unsigned long filesLinked = 0; + unsigned long long bytesFreed = 0; + unsigned long long blocksFreed = 0; }; @@ -72,6 +67,8 @@ private: public: + PathSetting realStoreDir_; + const Path realStoreDir; const Path dbDir; const Path linksDir; @@ -81,7 +78,9 @@ public: private: - bool requireSigs; + Setting<bool> requireSigs{(Store*) this, + settings.signedBinaryCaches != "", // FIXME + "require-sigs", "whether store paths should have a trusted signature on import"}; PublicKeys publicKeys; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 9d5c04dca0c5..e06002587f94 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -27,7 +27,8 @@ libstore_CXXFLAGS = \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ - -DBASH_PATH="\"$(bash)\"" + -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ + -DLSOF=\"$(lsof)\" $(d)/local-store.cc: $(d)/schema.sql.hh diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc new file mode 100644 index 000000000000..7491037b2d79 --- /dev/null +++ b/src/libstore/machines.cc @@ -0,0 +1,91 @@ +#include "machines.hh" +#include "util.hh" +#include "globals.hh" + +#include <algorithm> + +namespace nix { + +Machine::Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) : + storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) +{} + +bool Machine::allSupported(const std::set<string> & features) const { + return std::all_of(features.begin(), features.end(), + [&](const string & feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); +} + +bool Machine::mandatoryMet(const std::set<string> & features) const { + return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string & feature) { + return features.count(feature); + }); +} + +void parseMachines(const std::string & s, Machines & machines) +{ + for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + chomp(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; + auto tokens = tokenizeString<std::vector<string>>(line); + auto sz = tokens.size(); + if (sz < 1) + throw FormatError("bad machine specification ‘%s’", line); + + auto isSet = [&](size_t n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; + + machines.emplace_back(tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() +{ + Machines machines; + + for (auto & file : settings.builderFiles.get()) { + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + } + } + + parseMachines(settings.builders, machines); + + return machines; +} + +} diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh new file mode 100644 index 000000000000..de92eb924e4a --- /dev/null +++ b/src/libstore/machines.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct Machine { + + const string storeUri; + const std::vector<string> systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set<string> supportedFeatures; + const std::set<string> mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set<string> & features) const; + + bool mandatoryMet(const std::set<string> & features) const; + + Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); +}; + +typedef std::vector<Machine> Machines; + +void parseMachines(const std::string & s, Machines & machines); + +Machines getMachines(); + +} diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 4cb5de7449ea..82595e76a9b5 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -2,6 +2,8 @@ #include "archive.hh" #include <map> +#include <stack> +#include <algorithm> namespace nix { @@ -16,16 +18,16 @@ struct NarMember size_t start, size; std::string target; + + /* If this is a directory, all the children of the directory. */ + std::map<std::string, NarMember> children; }; struct NarIndexer : ParseSink, StringSource { - // FIXME: should store this as a tree. Now we're vulnerable to - // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}). - typedef std::map<Path, NarMember> Members; - Members members; + NarMember root; + std::stack<NarMember*> parents; - Path currentPath; std::string currentStart; bool isExec = false; @@ -33,28 +35,45 @@ struct NarIndexer : ParseSink, StringSource { } + void createMember(const Path & path, NarMember member) { + size_t level = std::count(path.begin(), path.end(), '/'); + while(parents.size() > level) { + parents.pop(); + } + + if(parents.empty()) { + root = std::move(member); + parents.push(&root); + } else { + if(parents.top()->type != FSAccessor::Type::tDirectory) { + throw Error(format("NAR file missing parent directory of path ‘%1%’") % path); + } + auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); + parents.push(&result.first->second); + } + } + void createDirectory(const Path & path) override { - members.emplace(path, - NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 }); } void createRegularFile(const Path & path) override { - currentPath = path; + createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 }); } void isExecutable() override { - isExec = true; + parents.top()->isExecutable = true; } void preallocateContents(unsigned long long size) override { currentStart = string(s, pos, 16); assert(size <= std::numeric_limits<size_t>::max()); - members.emplace(currentPath, - NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size}); + parents.top()->size = (size_t)size; + parents.top()->start = pos; } void receiveContents(unsigned char * data, unsigned int len) override @@ -68,16 +87,42 @@ struct NarIndexer : ParseSink, StringSource void createSymlink(const Path & path, const string & target) override { - members.emplace(path, + createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } - Members::iterator find(const Path & path) + NarMember* find(const Path & path) { - auto i = members.find(path); - if (i == members.end()) + Path canon = path == "" ? "" : canonPath(path); + NarMember* current = &root; + auto end = path.end(); + for(auto it = path.begin(); it != end; ) { + // because it != end, the remaining component is non-empty so we need + // a directory + if(current->type != FSAccessor::Type::tDirectory) return nullptr; + + // skip slash (canonPath above ensures that this is always a slash) + assert(*it == '/'); + it += 1; + + // lookup current component + auto next = std::find(it, end, '/'); + auto child = current->children.find(std::string(it, next)); + if(child == current->children.end()) return nullptr; + current = &child->second; + + it = next; + } + + return current; + } + + NarMember& at(const Path & path) { + auto result = find(path); + if(result == nullptr) { throw Error(format("NAR file does not contain path ‘%1%’") % path); - return i; + } + return *result; } }; @@ -93,44 +138,41 @@ struct NarAccessor : public FSAccessor Stat stat(const Path & path) override { - auto i = indexer.members.find(path); - if (i == indexer.members.end()) + auto i = indexer.find(path); + if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false}; - return {i->second.type, i->second.size, i->second.isExecutable}; + return {i->type, i->size, i->isExecutable}; } StringSet readDirectory(const Path & path) override { - auto i = indexer.find(path); + auto i = indexer.at(path); - if (i->second.type != FSAccessor::Type::tDirectory) + if (i.type != FSAccessor::Type::tDirectory) throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); - ++i; StringSet res; - while (i != indexer.members.end() && isInDir(i->first, path)) { - // FIXME: really bad performance. - if (i->first.find('/', path.size() + 1) == std::string::npos) - res.insert(std::string(i->first, path.size() + 1)); - ++i; + for(auto&& child : i.children) { + res.insert(child.first); + } return res; } std::string readFile(const Path & path) override { - auto i = indexer.find(path); - if (i->second.type != FSAccessor::Type::tRegular) + auto i = indexer.at(path); + if (i.type != FSAccessor::Type::tRegular) throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); - return std::string(*nar, i->second.start, i->second.size); + return std::string(*nar, i.start, i.size); } std::string readLink(const Path & path) override { - auto i = indexer.find(path); - if (i->second.type != FSAccessor::Type::tSymlink) + auto i = indexer.at(path); + if (i.type != FSAccessor::Type::tSymlink) throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); - return i->second.target; + return i.target; } }; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cf234e35d373..56167c4dfae8 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -220,8 +220,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - if (st.st_size) - printInfo(format("‘%1%’ has maximum number of links") % linkPath); + debug("‘%s’ has reached maximum number of links", linkPath); return; } throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path); @@ -241,7 +240,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) for (auto & i : paths) { addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); + //Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c9c590787450..be8819bbc004 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -43,7 +43,7 @@ template Paths readStorePaths(Store & store, Source & from); RemoteStore::RemoteStore(const Params & params) : Store(params) , connections(make_ref<Pool<Connection>>( - std::max(1, std::stoi(get(params, "max-connections", "1"))), + std::max(1, (int) maxConnections), [this]() { return openConnectionWrapper(); }, [](const ref<Connection> & r) { return r->to.good() && r->from.good(); } )) @@ -100,7 +100,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() throw Error(format("socket path ‘%1%’ is too long") % socketPath); strcpy(addr.sun_path, socketPath.c_str()); - if (connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); conn->from.fd = conn->fd.get(); @@ -166,9 +166,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - Settings::SettingsMap overrides = settings.getOverrides(); - if (overrides["ssh-auth-sock"] == "") - overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK"); + auto overrides = settings.getSettings(true); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second; @@ -416,7 +414,9 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, try { conn->to.written = 0; conn->to.warn = true; + connections->incCapacity(); dumpPath(srcPath, conn->to, filter); + connections->decCapacity(); conn->to.warn = false; conn->processStderr(); } catch (SysError & e) { @@ -613,6 +613,12 @@ void RemoteStore::queryMissing(const PathSet & targets, } +void RemoteStore::connect() +{ + auto conn(connections->get()); +} + + RemoteStore::Connection::~Connection() { try { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index db8da7eaa8c5..ed430e4cabb6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,6 +22,9 @@ class RemoteStore : public virtual Store { public: + const Setting<int> maxConnections{(Store*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -89,6 +92,8 @@ public: PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) override; + void connect() override; + protected: struct Connection diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 3053f908c4e2..245455296013 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -125,22 +125,22 @@ S3Helper::DownloadResult S3Helper::getObject( struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { + const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; + const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + std::string bucketName; Stats stats; S3Helper s3Helper; - std::string narinfoCompression, lsCompression, logCompression; - S3BinaryCacheStoreImpl( const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1)) - , narinfoCompression(get(params, "narinfo-compression", "")) - , lsCompression(get(params, "ls-compression", "")) - , logCompression(get(params, "log-compression", "")) + , s3Helper(region) { diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 2a81a8b1ebe5..bb536fadfd51 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -14,16 +14,19 @@ class SSHStore : public RemoteStore { public: + const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; + SSHStore(const std::string & host, const Params & params) : Store(params) , RemoteStore(params) , host(host) , master( host, - get(params, "ssh-key", ""), + sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - get(params, "compress", "") == "true") + compress) { } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index e54f3f4ba284..6edabaa3a1d9 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -31,6 +31,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string throw SysError("duping over stdin"); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) + throw SysError("duping over stderr"); Strings args = { "ssh", host.c_str(), "-x", "-a" }; addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index b4396467e54e..18dea227ad1f 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -13,6 +13,7 @@ private: const std::string keyFile; const bool useMaster; const bool compress; + const int logFD; struct State { @@ -27,11 +28,12 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress) + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1) : host(host) , keyFile(keyFile) , useMaster(useMaster) , compress(compress) + , logFD(logFD) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 59348c5d0b5f..e6cbd53dc80a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -167,7 +167,7 @@ void checkStoreName(const string & name) collisions (for security). For instance, it shouldn't be feasible to come up with a derivation whose output path collides with the path for a copied source. The former would have a <s> starting with - "output:out:", while the latter would have a <2> starting with + "output:out:", while the latter would have a <s> starting with "source:". */ @@ -241,8 +241,8 @@ Path Store::computeStorePathForText(const string & name, const string & s, Store::Store(const Params & params) - : storeDir(get(params, "store", settings.nixStore)) - , state({std::stoi(get(params, "path-info-cache-size", "65536"))}) + : Config(params) + , state({(size_t) pathInfoCacheSize}) { } @@ -482,21 +482,23 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths if (showClosureSize) jsonPath.attr("closureSize", getClosureSize(storePath)); - if (!includeImpureInfo) continue; + if (includeImpureInfo) { - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); + if (info->deriver != "") + jsonPath.attr("deriver", info->deriver); - if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); - if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + if (info->ultimate) + jsonPath.attr("ultimate", info->ultimate); + + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto & sig : info->sigs) + jsonSigs.elem(sig); + } - if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); - for (auto & sig : info->sigs) - jsonSigs.elem(sig); } } } @@ -523,6 +525,17 @@ const Store::Stats & Store::getStats() } +void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & path : paths) + if (isDerivation(path)) + unsupported(); + + if (queryValidPaths(paths).size() != paths.size()) + unsupported(); +} + + void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, bool repair, bool dontCheckSigs) { @@ -531,15 +544,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); - if (srcStore->isTrusted()) - dontCheckSigs = true; - if (!info->narHash && dontCheckSigs) { auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); info = info2; } + assert(info->narHash); + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + assert(info->narHash); + dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); } @@ -698,10 +718,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStore(const std::string & uri_) +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) { auto uri(uri_); - Store::Params params; + Store::Params params(extraParams); auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { @@ -711,17 +732,16 @@ ref<Store> openStore(const std::string & uri_) } uri = uri_.substr(0, q); } - return openStore(uri, params); -} -ref<Store> openStore(const std::string & uri, const Store::Params & params) -{ for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); - if (store) return ref<Store>(store); + if (store) { + store->warnUnknownSettings(); + return ref<Store>(store); + } } - throw Error(format("don't know how to open Nix store ‘%s’") % uri); + throw Error("don't know how to open Nix store ‘%s’", uri); } @@ -731,7 +751,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir) return tDaemon; } else if (uri == "local") { return tLocal; - } else if (uri == "") { + } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) return tLocal; else if (pathExists(settings.nixDaemonSocketFile)) @@ -779,14 +799,10 @@ std::list<ref<Store>> getDefaultSubstituters() state->stores.push_back(openStore(uri)); }; - Strings defaultSubstituters; - if (settings.nixStore == "/nix/store") - defaultSubstituters.push_back("https://cache.nixos.org/"); - - for (auto uri : settings.get("substituters", settings.get("binary-caches", defaultSubstituters))) + for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.get("extra-binary-caches", Strings())) + for (auto uri : settings.extraSubstituters.get()) addStore(uri); state->done = true; @@ -795,7 +811,8 @@ std::list<ref<Store>> getDefaultSubstituters() } -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute) +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute, bool dontCheckSigs) { PathSet valid = to->queryValidPaths(storePaths, substitute); @@ -805,7 +822,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool std::string copiedLabel = "copied"; - logger->setExpected(copiedLabel, missing.size()); + //logger->setExpected(copiedLabel, missing.size()); ThreadPool pool; @@ -821,13 +838,14 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool checkInterrupt(); if (!to->isValidPath(storePath)) { - Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); + //Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); - copyStorePath(from, to, storePath); + copyStorePath(from, to, storePath, false, dontCheckSigs); - logger->incProgress(copiedLabel); + //logger->incProgress(copiedLabel); } else - logger->incExpected(copiedLabel, -1); + ; + //logger->incExpected(copiedLabel, -1); }); pool.process(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 68c59a9f2922..929c95a0f2f8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -6,6 +6,7 @@ #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" +#include "config.hh" #include <atomic> #include <limits> @@ -17,6 +18,12 @@ namespace nix { +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) +MakeError(Unsupported, Error) + + struct BasicDerivation; struct Derivation; class FSAccessor; @@ -81,12 +88,7 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed; - - GCResults() - { - bytesFreed = 0; - } + unsigned long long bytesFreed = 0; }; @@ -111,9 +113,8 @@ struct ValidPathInfo uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it was built - locally or is content-addressable (e.g. added via addToStore() - or the result of a fixed-output derivation). */ + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ bool ultimate = false; StringSet sigs; // note: not necessarily verified @@ -229,13 +230,17 @@ struct BuildResult }; -class Store : public std::enable_shared_from_this<Store> +class Store : public std::enable_shared_from_this<Store>, public Config { public: typedef std::map<std::string, std::string> Params; - const Path storeDir; + const PathSetting storeDir_{this, false, settings.nixStore, + "store", "path to the Nix store"}; + const Path storeDir = storeDir_; + + const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; protected: @@ -414,7 +419,7 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for @@ -564,10 +569,6 @@ public: const Stats & getStats(); - /* Whether this store paths from this store can be imported even - if they lack a signature. */ - virtual bool isTrusted() { return false; } - /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr<std::string> getBuildLog(const Path & path) @@ -580,19 +581,39 @@ public: state.lock()->pathInfoCache.clear(); } + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect() { }; + protected: Stats stats; + /* Unsupported methods. */ + [[noreturn]] void unsupported() + { + throw Unsupported("requested operation is not supported by store ‘%s’", getUri()); + } + }; class LocalFSStore : public virtual Store { public: - const Path rootDir; - const Path stateDir; - const Path logDir; + + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to emit the call to the Option constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; + const static string drvsLogDir; LocalFSStore(const Params & params); @@ -646,23 +667,35 @@ void removeTempRoots(); /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: - * ‘direct’: The Nix store in /nix/store and database in + * ‘local’: The Nix store in /nix/store and database in /nix/var/nix/db, accessed directly. * ‘daemon’: The Nix store accessed via a Unix domain socket connection to nix-daemon. + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + * ‘file://<path>’: A binary cache stored in <path>. - If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on - whether the user has write access to the local Nix store/database. - set to true *unless* you're going to collect garbage. */ -ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); + * ‘https://<path>’: A binary cache accessed via HTTP. + + * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]<host>’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. -ref<Store> openStore(const std::string & uri, const Store::Params & params); + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), + const Store::Params & extraParams = Store::Params()); -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false); +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute = false, bool dontCheckSigs = false); enum StoreType { tDaemon, @@ -710,10 +743,4 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); - -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) - - } diff --git a/src/libutil/config.cc b/src/libutil/config.cc new file mode 100644 index 000000000000..f7a46bfee63f --- /dev/null +++ b/src/libutil/config.cc @@ -0,0 +1,231 @@ +#include "config.hh" +#include "args.hh" +#include "json.hh" + +namespace nix { + +void Config::set(const std::string & name, const std::string & value) +{ + auto i = _settings.find(name); + if (i == _settings.end()) + throw UsageError("unknown setting '%s'", name); + i->second.setting->set(value); + i->second.setting->overriden = true; +} + +void Config::addSetting(AbstractSetting * setting) +{ + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData(true, setting)); + + bool set = false; + + auto i = initials.find(setting->name); + if (i != initials.end()) { + setting->set(i->second); + setting->overriden = true; + initials.erase(i); + set = true; + } + + for (auto & alias : setting->aliases) { + auto i = initials.find(alias); + if (i != initials.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { + setting->set(i->second); + setting->overriden = true; + initials.erase(i); + set = true; + } + } + } +} + +void Config::warnUnknownSettings() +{ + for (auto & i : initials) + warn("unknown setting '%s'", i.first); +} + +StringMap Config::getSettings(bool overridenOnly) +{ + StringMap res; + for (auto & opt : _settings) + if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + res.emplace(opt.first, opt.second.setting->to_string()); + return res; +} + +void Config::applyConfigFile(const Path & path, bool fatal) +{ + try { + string contents = readFile(path); + + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2 || tokens[1] != "=") + throw UsageError("illegal configuration line ‘%1%’ in ‘%2%’", line, path); + + string name = tokens[0]; + + vector<string>::iterator i = tokens.begin(); + advance(i, 2); + + try { + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + } catch (UsageError & e) { + if (fatal) throw; + warn("in configuration file '%s': %s", path, e.what()); + } + }; + } catch (SysError &) { } +} + +void Config::resetOverriden() +{ + for (auto & s : _settings) + s.second.setting->overriden = false; +} + +void Config::toJSON(JSONObject & out) +{ + for (auto & s : _settings) + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } +} + +AbstractSetting::AbstractSetting( + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases) + : name(name), description(description), aliases(aliases) +{ +} + +void AbstractSetting::toJSON(JSONPlaceholder & out) +{ + out.write(to_string()); +} + +template<typename T> +void BaseSetting<T>::toJSON(JSONPlaceholder & out) +{ + out.write(value); +} + +template<> void BaseSetting<std::string>::set(const std::string & str) +{ + value = str; +} + +template<> std::string BaseSetting<std::string>::to_string() +{ + return value; +} + +template<typename T> +void BaseSetting<T>::set(const std::string & str) +{ + static_assert(std::is_integral<T>::value, "Integer required."); + if (!string2Int(str, value)) + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template<typename T> +std::string BaseSetting<T>::to_string() +{ + static_assert(std::is_integral<T>::value, "Integer required."); + return std::to_string(value); +} + +template<> void BaseSetting<bool>::set(const std::string & str) +{ + if (str == "true" || str == "yes" || str == "1") + value = true; + else if (str == "false" || str == "no" || str == "0") + value = false; + else + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); +} + +template<> std::string BaseSetting<bool>::to_string() +{ + return value ? "true" : "false"; +} + +template<> void BaseSetting<Strings>::set(const std::string & str) +{ + value = tokenizeString<Strings>(str); +} + +template<> std::string BaseSetting<Strings>::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template<> void BaseSetting<StringSet>::set(const std::string & str) +{ + value = tokenizeString<StringSet>(str); +} + +template<> std::string BaseSetting<StringSet>::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template class BaseSetting<int>; +template class BaseSetting<unsigned int>; +template class BaseSetting<long>; +template class BaseSetting<unsigned long>; +template class BaseSetting<long long>; +template class BaseSetting<unsigned long long>; +template class BaseSetting<bool>; +template class BaseSetting<std::string>; + +void PathSetting::set(const std::string & str) +{ + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); +} + +} diff --git a/src/libutil/config.hh b/src/libutil/config.hh new file mode 100644 index 000000000000..77620d47d371 --- /dev/null +++ b/src/libutil/config.hh @@ -0,0 +1,192 @@ +#include <map> +#include <set> + +#include "types.hh" + +#pragma once + +namespace nix { + +class Args; +class AbstractSetting; +class JSONPlaceholder; +class JSONObject; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting<T> members: + + class MyClass : private Config + { + Setting<int> foo{this, 123, "foo", "the number of foos to use"}; + Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config +{ + friend class AbstractSetting; + + struct SettingData + { + bool isAlias; + AbstractSetting * setting; + SettingData(bool isAlias, AbstractSetting * setting) + : isAlias(isAlias), setting(setting) + { } + }; + + std::map<std::string, SettingData> _settings; + + StringMap initials; + +public: + + Config(const StringMap & initials) + : initials(initials) + { } + + void set(const std::string & name, const std::string & value); + + void addSetting(AbstractSetting * setting); + + void warnUnknownSettings(); + + StringMap getSettings(bool overridenOnly = false); + + void applyConfigFile(const Path & path, bool fatal = false); + + void resetOverriden(); + + void toJSON(JSONObject & out); +}; + +class AbstractSetting +{ + friend class Config; + +public: + + const std::string name; + const std::string description; + const std::set<std::string> aliases; + + int created = 123; + + bool overriden = false; + +protected: + + AbstractSetting( + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases); + + virtual ~AbstractSetting() + { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } + + virtual void set(const std::string & value) = 0; + + virtual std::string to_string() = 0; + + virtual void toJSON(JSONPlaceholder & out); + + bool isOverriden() { return overriden; } +}; + +/* A setting of type T. */ +template<typename T> +class BaseSetting : public AbstractSetting +{ +protected: + + T value; + +public: + + BaseSetting(const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : AbstractSetting(name, description, aliases) + , value(def) + { } + + operator const T &() const { return value; } + operator T &() { return value; } + const T & get() const { return value; } + bool operator ==(const T & v2) const { return value == v2; } + bool operator !=(const T & v2) const { return value != v2; } + void operator =(const T & v) { assign(v); } + virtual void assign(const T & v) { value = v; } + + void set(const std::string & str) override; + + std::string to_string() override; + + void toJSON(JSONPlaceholder & out) override; +}; + +template<typename T> +std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt) +{ + str << (const T &) opt; + return str; +} + +template<typename T> +bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; } + +template<typename T> +class Setting : public BaseSetting<T> +{ +public: + Setting(Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<T>(def, name, description, aliases) + { + options->addSetting(this); + } + + void operator =(const T & v) { this->assign(v); } +}; + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public BaseSetting<Path> +{ + bool allowEmpty; + +public: + + PathSetting(Config * options, + bool allowEmpty, + const Path & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<Path>(def, name, description, aliases) + , allowEmpty(allowEmpty) + { + options->addSetting(this); + } + + void set(const std::string & str) override; + + Path operator +(const char * p) const { return value + p; } + + void operator =(const Path & v) { this->assign(v); } +}; + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9f4afd93c2fc..fa1bb5d97183 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -224,7 +224,7 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, unsigned int len) + const unsigned char * bytes, size_t len) { if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 6023d1d4fb84..b8b8ef9c8cca 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -19,49 +19,32 @@ void toJSON(std::ostream & str, const char * start, const char * end) str << '"'; } -void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} - void toJSON(std::ostream & str, const char * s) { if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); } -void toJSON(std::ostream & str, unsigned long long n) -{ - str << n; -} - -void toJSON(std::ostream & str, unsigned long n) -{ - str << n; -} - -void toJSON(std::ostream & str, long n) -{ - str << n; -} +template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; } +template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; } +template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; } +template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; } +template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; } +template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; } -void toJSON(std::ostream & str, unsigned int n) +template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { - str << n; -} - -void toJSON(std::ostream & str, int n) -{ - str << n; + toJSON(str, s.c_str(), s.c_str() + s.size()); } -void toJSON(std::ostream & str, double f) +template<> void toJSON<bool>(std::ostream & str, const bool & b) { - str << f; + str << (b ? "true" : "false"); } -void toJSON(std::ostream & str, bool b) +template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b) { - str << (b ? "true" : "false"); + str << "null"; } JSONWriter::JSONWriter(std::ostream & str, bool indent) diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 03eecb732586..595e9bbe3491 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -7,15 +7,10 @@ namespace nix { void toJSON(std::ostream & str, const char * start, const char * end); -void toJSON(std::ostream & str, const std::string & s); void toJSON(std::ostream & str, const char * s); -void toJSON(std::ostream & str, unsigned long long n); -void toJSON(std::ostream & str, unsigned long n); -void toJSON(std::ostream & str, long n); -void toJSON(std::ostream & str, unsigned int n); -void toJSON(std::ostream & str, int n); -void toJSON(std::ostream & str, double f); -void toJSON(std::ostream & str, bool b); + +template<typename T> +void toJSON(std::ostream & str, const T & n); class JSONWriter { diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh new file mode 100644 index 000000000000..d073e486c2eb --- /dev/null +++ b/src/libutil/lazy.hh @@ -0,0 +1,48 @@ +#include <exception> +#include <functional> +#include <mutex> + +namespace nix { + +/* A helper class for lazily-initialized variables. + + Lazy<T> var([]() { return value; }); + + declares a variable of type T that is initialized to 'value' (in a + thread-safe way) on first use, that is, when var() is first + called. If the initialiser code throws an exception, then all + subsequent calls to var() will rethrow that exception. */ +template<typename T> +class Lazy +{ + + typedef std::function<T()> Init; + + Init init; + + std::once_flag done; + + T value; + + std::exception_ptr ex; + +public: + + Lazy(Init init) : init(init) + { } + + const T & operator () () + { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) std::rethrow_exception(ex); + return value; + } +}; + +} diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d9e8d22d7685..2d0acca24216 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,9 +1,16 @@ #include "logging.hh" #include "util.hh" +#include <atomic> + namespace nix { -Logger * logger = 0; +Logger * logger = makeDefaultLogger(); + +void Logger::warn(const std::string & msg) +{ + log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); +} class SimpleLogger : public Logger { @@ -37,12 +44,7 @@ public: writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - log(lvl, fs); - } - - void stopActivity(Activity & activity) override + void event(const Event & ev) override { } }; @@ -52,7 +54,7 @@ Verbosity verbosity = lvlInfo; void warnOnce(bool & haveWarned, const FormatOrString & fs) { if (!haveWarned) { - printError(format("warning: %1%") % fs.s); + warn(fs.s); haveWarned = true; } } @@ -74,4 +76,8 @@ Logger * makeDefaultLogger() return new SimpleLogger(); } +std::atomic<uint64_t> Activity::nextId{(uint64_t) getpid() << 32}; + +Activity::Activity() : id(nextId++) { }; + } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 3f83664794f7..ddfc336fee07 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -13,7 +13,64 @@ typedef enum { lvlVomit } Verbosity; -class Activity; +class Activity +{ + static std::atomic<uint64_t> nextId; +public: + typedef uint64_t t; + const t id; + Activity(); + Activity(const Activity & act) : id(act.id) { }; + Activity(uint64_t id) : id(id) { }; +}; + +typedef enum { + evBuildCreated = 0, + evBuildStarted = 1, + evBuildOutput = 2, + evBuildFinished = 3, + evDownloadCreated = 4, + evDownloadDestroyed = 5, + evDownloadProgress = 6, + evDownloadSucceeded = 7, + evSubstitutionCreated = 8, + evSubstitutionStarted = 9, + evSubstitutionFinished = 10, +} EventType; + +struct Event +{ + struct Field + { + // FIXME: use std::variant. + enum { tInt, tString } type; + uint64_t i = 0; + std::string s; + Field(const std::string & s) : type(tString), s(s) { } + Field(const char * s) : type(tString), s(s) { } + Field(const uint64_t & i) : type(tInt), i(i) { } + Field(const Activity & act) : type(tInt), i(act.id) { } + }; + + typedef std::vector<Field> Fields; + + EventType type; + Fields fields; + + std::string getS(size_t n) const + { + assert(n < fields.size()); + assert(fields[n].type == Field::tString); + return fields[n].s; + } + + uint64_t getI(size_t n) const + { + assert(n < fields.size()); + assert(fields[n].type == Field::tInt); + return fields[n].i; + } +}; class Logger { @@ -30,34 +87,18 @@ public: log(lvlInfo, fs); } - virtual void setExpected(const std::string & label, uint64_t value = 1) { } - virtual void setProgress(const std::string & label, uint64_t value = 1) { } - virtual void incExpected(const std::string & label, uint64_t value = 1) { } - virtual void incProgress(const std::string & label, uint64_t value = 1) { } + virtual void warn(const std::string & msg); -private: - - virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; - - virtual void stopActivity(Activity & activity) = 0; - -}; - -class Activity -{ -public: - Logger & logger; - - Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) - : logger(logger) + template<typename... Args> + void event(EventType type, const Args & ... args) { - logger.startActivity(*this, lvl, fs); + Event ev; + ev.type = type; + nop{(ev.fields.emplace_back(Event::Field(args)), 1)...}; + event(ev); } - ~Activity() - { - logger.stopActivity(*this); - } + virtual void event(const Event & ev) = 0; }; extern Logger * logger; @@ -82,6 +123,14 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) +template<typename... Args> +inline void warn(const std::string & fs, Args... args) +{ + boost::format f(fs); + nop{boost::io::detail::feed(f, args)...}; + logger->warn(f.str()); +} + void warnOnce(bool & haveWarned, const FormatOrString & fs); void writeToStderr(const string & s); diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index 20df21948849..7033090020ef 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -68,6 +68,22 @@ public: state_->max = max; } + void incCapacity() + { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() + { + auto state_(state.lock()); + state_->max--; + } + ~Pool() { auto state_(state.lock()); diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 97d79af9b5d6..9f32d31addbf 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -7,6 +7,7 @@ #include <list> #include <set> #include <memory> +#include <map> #include <boost/format.hpp> @@ -31,6 +32,11 @@ using std::vector; using boost::format; +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template<typename... T> nop(T...) {} }; + + struct FormatOrString { string s; @@ -45,16 +51,6 @@ struct FormatOrString ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion takes place). */ -inline void formatHelper(boost::format & f) -{ -} - -template<typename T, typename... Args> -inline void formatHelper(boost::format & f, T x, Args... args) -{ - formatHelper(f % x, args...); -} - inline std::string fmt(const std::string & s) { return s; @@ -74,7 +70,7 @@ template<typename... Args> inline std::string fmt(const std::string & fs, Args... args) { boost::format f(fs); - formatHelper(f, args...); + nop{boost::io::detail::feed(f, args)...}; return f.str(); } @@ -141,6 +137,7 @@ private: typedef list<string> Strings; typedef set<string> StringSet; +typedef std::map<std::string, std::string> StringMap; /* Paths are just strings. */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a640a64c724e..16f4b232e6c5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,3 +1,4 @@ +#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" @@ -13,10 +14,12 @@ #include <thread> #include <future> -#include <sys/wait.h> -#include <unistd.h> #include <fcntl.h> #include <limits.h> +#include <pwd.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> #ifdef __APPLE__ #include <sys/syscall.h> @@ -96,6 +99,8 @@ Path absPath(Path path, Path dir) Path canonPath(const Path & path, bool resolveSymlinks) { + assert(path != ""); + string s; if (path[0] != '/') @@ -367,7 +372,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); + //Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -415,18 +420,50 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +static Lazy<Path> getHome2([]() { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) { + char buf[16384]; + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(getuid(), &pwbuf, buf, sizeof(buf), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return homeDir; +}); + +Path getHome() { return getHome2(); } + + Path getCacheDir() { Path cacheDir = getEnv("XDG_CACHE_HOME"); - if (cacheDir.empty()) { - Path homeDir = getEnv("HOME"); - if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set"); - cacheDir = homeDir + "/.cache"; - } + if (cacheDir.empty()) + cacheDir = getHome() + "/.cache"; return cacheDir; } +Path getConfigDir() +{ + Path configDir = getEnv("XDG_CONFIG_HOME"); + if (configDir.empty()) + configDir = getHome() + "/.config"; + return configDir; +} + + +Path getDataDir() +{ + Path dataDir = getEnv("XDG_DATA_HOME"); + if (dataDir.empty()) + dataDir = getHome() + "/.local/share"; + return dataDir; +} + + Paths createDirs(const Path & path) { Paths created; @@ -932,7 +969,12 @@ void closeOnExec(int fd) bool _isInterrupted = false; -thread_local bool interruptThrown = false; +static thread_local bool interruptThrown = false; + +void setInterruptThrown() +{ + interruptThrown = true; +} void _interrupted() { @@ -1047,9 +1089,9 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & suffix) +bool hasPrefix(const string & s, const string & prefix) { - return s.compare(0, suffix.size(), suffix) == 0; + return s.compare(0, prefix.size(), prefix) == 0; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0e6941e4a8db..7ea32e8d9f14 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -110,9 +110,18 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Return the path to $XDG_CACHE_HOME/.cache. */ +/* Return $HOME or the user's home directory from /etc/passwd. */ +Path getHome(); + +/* Return $XDG_CACHE_HOME or $HOME/.cache. */ Path getCacheDir(); +/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +Path getConfigDir(); + +/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +Path getDataDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -264,7 +273,7 @@ void closeOnExec(int fd); extern bool _isInterrupted; -extern thread_local bool interruptThrown; +void setInterruptThrown(); void _interrupted(); @@ -355,6 +364,8 @@ void ignoreException(); #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_BLUE "\e[34;1m" /* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE new file mode 100644 index 000000000000..18e814865a54 --- /dev/null +++ b/src/linenoise/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com> +Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/linenoise/linenoise.c b/src/linenoise/linenoise.c new file mode 100644 index 000000000000..fce14a7c53a3 --- /dev/null +++ b/src/linenoise/linenoise.c @@ -0,0 +1,1199 @@ +/* linenoise.c -- guerrilla line editing library against the idea that a + * line editing lib needs to be 20,000 lines of C code. + * + * You can find the latest source code at: + * + * http://github.com/antirez/linenoise + * + * Does a number of crazy assumptions that happen to be true in 99.9999% of + * the 2010 UNIX computers around. + * + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2016, Salvatore Sanfilippo <antirez at gmail dot com> + * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ------------------------------------------------------------------------ + * + * References: + * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html + * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html + * + * Todo list: + * - Filter bogus Ctrl+<char> combinations. + * - Win32 support + * + * Bloat: + * - History search like Ctrl+r in readline? + * + * List of escape sequences used by this program, we do everything just + * with three sequences. In order to be so cheap we may have some + * flickering effect with some slow terminal, but the lesser sequences + * the more compatible. + * + * EL (Erase Line) + * Sequence: ESC [ n K + * Effect: if n is 0 or missing, clear from cursor to end of line + * Effect: if n is 1, clear from beginning of line to cursor + * Effect: if n is 2, clear entire line + * + * CUF (CUrsor Forward) + * Sequence: ESC [ n C + * Effect: moves cursor forward n chars + * + * CUB (CUrsor Backward) + * Sequence: ESC [ n D + * Effect: moves cursor backward n chars + * + * The following is used to get the terminal width if getting + * the width with the TIOCGWINSZ ioctl fails + * + * DSR (Device Status Report) + * Sequence: ESC [ 6 n + * Effect: reports the current cusor position as ESC [ n ; m R + * where n is the row and m is the column + * + * When multi line mode is enabled, we also use an additional escape + * sequence. However multi line editing is disabled by default. + * + * CUU (Cursor Up) + * Sequence: ESC [ n A + * Effect: moves cursor up of n chars. + * + * CUD (Cursor Down) + * Sequence: ESC [ n B + * Effect: moves cursor down of n chars. + * + * When linenoiseClearScreen() is called, two additional escape sequences + * are used in order to clear the screen and position the cursor at home + * position. + * + * CUP (Cursor position) + * Sequence: ESC [ H + * Effect: moves the cursor to upper left corner + * + * ED (Erase display) + * Sequence: ESC [ 2 J + * Effect: clear the whole screen + * + */ + +#include <termios.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/ioctl.h> +#include <unistd.h> +#include "linenoise.h" + +#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 +#define LINENOISE_MAX_LINE 4096 +static char *unsupported_term[] = {"dumb","cons25","emacs",NULL}; +static linenoiseCompletionCallback *completionCallback = NULL; +static linenoiseHintsCallback *hintsCallback = NULL; +static linenoiseFreeHintsCallback *freeHintsCallback = NULL; + +static struct termios orig_termios; /* In order to restore at exit.*/ +static int rawmode = 0; /* For atexit() function to check if restore is needed*/ +static int mlmode = 0; /* Multi line mode. Default is single line. */ +static int atexit_registered = 0; /* Register atexit just 1 time. */ +static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN; +static int history_len = 0; +static char **history = NULL; + +/* The linenoiseState structure represents the state during line editing. + * We pass this state to functions implementing specific editing + * functionalities. */ +struct linenoiseState { + int ifd; /* Terminal stdin file descriptor. */ + int ofd; /* Terminal stdout file descriptor. */ + char *buf; /* Edited line buffer. */ + size_t buflen; /* Edited line buffer size. */ + const char *prompt; /* Prompt to display. */ + size_t plen; /* Prompt length. */ + size_t pos; /* Current cursor position. */ + size_t oldpos; /* Previous refresh cursor position. */ + size_t len; /* Current edited line length. */ + size_t cols; /* Number of columns in terminal. */ + size_t maxrows; /* Maximum num of rows used so far (multiline mode) */ + int history_index; /* The history index we are currently editing. */ +}; + +enum KEY_ACTION{ + KEY_NULL = 0, /* NULL */ + CTRL_A = 1, /* Ctrl+a */ + CTRL_B = 2, /* Ctrl-b */ + CTRL_C = 3, /* Ctrl-c */ + CTRL_D = 4, /* Ctrl-d */ + CTRL_E = 5, /* Ctrl-e */ + CTRL_F = 6, /* Ctrl-f */ + CTRL_H = 8, /* Ctrl-h */ + TAB = 9, /* Tab */ + CTRL_K = 11, /* Ctrl+k */ + CTRL_L = 12, /* Ctrl+l */ + ENTER = 13, /* Enter */ + CTRL_N = 14, /* Ctrl-n */ + CTRL_P = 16, /* Ctrl-p */ + CTRL_T = 20, /* Ctrl-t */ + CTRL_U = 21, /* Ctrl+u */ + CTRL_W = 23, /* Ctrl+w */ + ESC = 27, /* Escape */ + BACKSPACE = 127 /* Backspace */ +}; + +static void linenoiseAtExit(void); +int linenoiseHistoryAdd(const char *line); +static void refreshLine(struct linenoiseState *l); + +/* Debugging macro. */ +#if 0 +FILE *lndebug_fp = NULL; +#define lndebug(...) \ + do { \ + if (lndebug_fp == NULL) { \ + lndebug_fp = fopen("/tmp/lndebug.txt","a"); \ + fprintf(lndebug_fp, \ + "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \ + (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \ + (int)l->maxrows,old_rows); \ + } \ + fprintf(lndebug_fp, ", " __VA_ARGS__); \ + fflush(lndebug_fp); \ + } while (0) +#else +#define lndebug(fmt, ...) +#endif + +/* ======================= Low level terminal handling ====================== */ + +/* Set if to use or not the multi line mode. */ +void linenoiseSetMultiLine(int ml) { + mlmode = ml; +} + +/* Return true if the terminal name is in the list of terminals we know are + * not able to understand basic escape sequences. */ +static int isUnsupportedTerm(void) { + char *term = getenv("TERM"); + int j; + + if (term == NULL) return 0; + for (j = 0; unsupported_term[j]; j++) + if (!strcasecmp(term,unsupported_term[j])) return 1; + return 0; +} + +/* Raw mode: 1960 magic shit. */ +static int enableRawMode(int fd) { + struct termios raw; + + if (!isatty(STDIN_FILENO)) goto fatal; + if (!atexit_registered) { + atexit(linenoiseAtExit); + atexit_registered = 1; + } + if (tcgetattr(fd,&orig_termios) == -1) goto fatal; + + raw = orig_termios; /* modify the original mode */ + /* input modes: no break, no CR to NL, no parity check, no strip char, + * no start/stop output control. */ + raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON); + /* output modes - disable post processing */ + raw.c_oflag &= ~(OPOST); + /* control modes - set 8 bit chars */ + raw.c_cflag |= (CS8); + /* local modes - choing off, canonical off, no extended functions, + * no signal chars (^Z,^C) */ + raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); + /* control chars - set return condition: min number of bytes and timer. + * We want read to return every single byte, without timeout. */ + raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ + + /* put terminal in raw mode after flushing */ + if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal; + rawmode = 1; + return 0; + +fatal: + errno = ENOTTY; + return -1; +} + +static void disableRawMode(int fd) { + /* Don't even check the return value as it's too late. */ + if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1) + rawmode = 0; +} + +/* Use the ESC [6n escape sequence to query the horizontal cursor position + * and return it. On error -1 is returned, on success the position of the + * cursor. */ +static int getCursorPosition(int ifd, int ofd) { + char buf[32]; + int cols, rows; + unsigned int i = 0; + + /* Report cursor location */ + if (write(ofd, "\x1b[6n", 4) != 4) return -1; + + /* Read the response: ESC [ rows ; cols R */ + while (i < sizeof(buf)-1) { + if (read(ifd,buf+i,1) != 1) break; + if (buf[i] == 'R') break; + i++; + } + buf[i] = '\0'; + + /* Parse it. */ + if (buf[0] != ESC || buf[1] != '[') return -1; + if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1; + return cols; +} + +/* Try to get the number of columns in the current terminal, or assume 80 + * if it fails. */ +static int getColumns(int ifd, int ofd) { + struct winsize ws; + + if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) { + /* ioctl() failed. Try to query the terminal itself. */ + int start, cols; + + /* Get the initial position so we can restore it later. */ + start = getCursorPosition(ifd,ofd); + if (start == -1) goto failed; + + /* Go to right margin and get position. */ + if (write(ofd,"\x1b[999C",6) != 6) goto failed; + cols = getCursorPosition(ifd,ofd); + if (cols == -1) goto failed; + + /* Restore position. */ + if (cols > start) { + char seq[32]; + snprintf(seq,32,"\x1b[%dD",cols-start); + if (write(ofd,seq,strlen(seq)) == -1) { + /* Can't recover... */ + } + } + return cols; + } else { + return ws.ws_col; + } + +failed: + return 80; +} + +/* Clear the screen. Used to handle ctrl+l */ +void linenoiseClearScreen(void) { + if (write(STDOUT_FILENO,"\x1b[H\x1b[2J",7) <= 0) { + /* nothing to do, just to avoid warning. */ + } +} + +/* Beep, used for completion when there is nothing to complete or when all + * the choices were already shown. */ +static void linenoiseBeep(void) { + fprintf(stderr, "\x7"); + fflush(stderr); +} + +/* ============================== Completion ================================ */ + +/* Free a list of completion option populated by linenoiseAddCompletion(). */ +static void freeCompletions(linenoiseCompletions *lc) { + size_t i; + for (i = 0; i < lc->len; i++) + free(lc->cvec[i]); + if (lc->cvec != NULL) + free(lc->cvec); +} + +/* This is an helper function for linenoiseEdit() and is called when the + * user types the <tab> key in order to complete the string currently in the + * input. + * + * The state of the editing is encapsulated into the pointed linenoiseState + * structure as described in the structure definition. */ +static int completeLine(struct linenoiseState *ls) { + linenoiseCompletions lc = { 0, NULL }; + int nread, nwritten; + char c = 0; + + completionCallback(ls->buf,&lc); + if (lc.len == 0) { + linenoiseBeep(); + } else { + size_t stop = 0, i = 0; + + while(!stop) { + /* Show completion or original buffer */ + if (i < lc.len) { + struct linenoiseState saved = *ls; + + ls->len = ls->pos = strlen(lc.cvec[i]); + ls->buf = lc.cvec[i]; + refreshLine(ls); + ls->len = saved.len; + ls->pos = saved.pos; + ls->buf = saved.buf; + } else { + refreshLine(ls); + } + + nread = read(ls->ifd,&c,1); + if (nread <= 0) { + freeCompletions(&lc); + return -1; + } + + switch(c) { + case 9: /* tab */ + i = (i+1) % (lc.len+1); + if (i == lc.len) linenoiseBeep(); + break; + case 27: /* escape */ + /* Re-show original buffer */ + if (i < lc.len) refreshLine(ls); + stop = 1; + break; + default: + /* Update buffer and return */ + if (i < lc.len) { + nwritten = snprintf(ls->buf,ls->buflen,"%s",lc.cvec[i]); + ls->len = ls->pos = nwritten; + } + stop = 1; + break; + } + } + } + + freeCompletions(&lc); + return c; /* Return last read character */ +} + +/* Register a callback function to be called for tab-completion. */ +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { + completionCallback = fn; +} + +/* Register a hits function to be called to show hits to the user at the + * right of the prompt. */ +void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) { + hintsCallback = fn; +} + +/* Register a function to free the hints returned by the hints callback + * registered with linenoiseSetHintsCallback(). */ +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) { + freeHintsCallback = fn; +} + +/* This function is used by the callback function registered by the user + * in order to add completion options given the input string when the + * user typed <tab>. See the example.c source code for a very easy to + * understand example. */ +void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { + size_t len = strlen(str); + char *copy, **cvec; + + copy = malloc(len+1); + if (copy == NULL) return; + memcpy(copy,str,len+1); + cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); + if (cvec == NULL) { + free(copy); + return; + } + lc->cvec = cvec; + lc->cvec[lc->len++] = copy; +} + +/* =========================== Line editing ================================= */ + +/* We define a very simple "append buffer" structure, that is an heap + * allocated string where we can append to. This is useful in order to + * write all the escape sequences in a buffer and flush them to the standard + * output in a single call, to avoid flickering effects. */ +struct abuf { + char *b; + int len; +}; + +static void abInit(struct abuf *ab) { + ab->b = NULL; + ab->len = 0; +} + +static void abAppend(struct abuf *ab, const char *s, int len) { + char *new = realloc(ab->b,ab->len+len); + + if (new == NULL) return; + memcpy(new+ab->len,s,len); + ab->b = new; + ab->len += len; +} + +static void abFree(struct abuf *ab) { + free(ab->b); +} + +/* Helper of refreshSingleLine() and refreshMultiLine() to show hints + * to the right of the prompt. */ +void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { + char seq[64]; + if (hintsCallback && plen+l->len < l->cols) { + int color = -1, bold = 0; + char *hint = hintsCallback(l->buf,&color,&bold); + if (hint) { + int hintlen = strlen(hint); + int hintmaxlen = l->cols-(plen+l->len); + if (hintlen > hintmaxlen) hintlen = hintmaxlen; + if (bold == 1 && color == -1) color = 37; + if (color != -1 || bold != 0) + snprintf(seq,64,"\033[%d;%d;49m",bold,color); + abAppend(ab,seq,strlen(seq)); + abAppend(ab,hint,hintlen); + if (color != -1 || bold != 0) + abAppend(ab,"\033[0m",4); + /* Call the function to free the hint returned. */ + if (freeHintsCallback) freeHintsCallback(hint); + } + } +} + +/* Single line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshSingleLine(struct linenoiseState *l) { + char seq[64]; + size_t plen = strlen(l->prompt); + int fd = l->ofd; + char *buf = l->buf; + size_t len = l->len; + size_t pos = l->pos; + struct abuf ab; + + while((plen+pos) >= l->cols) { + buf++; + len--; + pos--; + } + while (plen+len > l->cols) { + len--; + } + + abInit(&ab); + /* Cursor to left edge */ + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + abAppend(&ab,buf,len); + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + /* Erase to right */ + snprintf(seq,64,"\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + /* Move cursor to original position. */ + snprintf(seq,64,"\r\x1b[%dC", (int)(pos+plen)); + abAppend(&ab,seq,strlen(seq)); + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Multi line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshMultiLine(struct linenoiseState *l) { + char seq[64]; + int plen = strlen(l->prompt); + int rows = (plen+l->len+l->cols-1)/l->cols; /* rows used by current buf. */ + int rpos = (plen+l->oldpos+l->cols)/l->cols; /* cursor relative row. */ + int rpos2; /* rpos after refresh. */ + int col; /* colum position, zero-based. */ + int old_rows = l->maxrows; + int fd = l->ofd, j; + struct abuf ab; + + /* Update maxrows if needed. */ + if (rows > (int)l->maxrows) l->maxrows = rows; + + /* First step: clear all the lines used before. To do so start by + * going to the last row. */ + abInit(&ab); + if (old_rows-rpos > 0) { + lndebug("go down %d", old_rows-rpos); + snprintf(seq,64,"\x1b[%dB", old_rows-rpos); + abAppend(&ab,seq,strlen(seq)); + } + + /* Now for every row clear it, go up. */ + for (j = 0; j < old_rows-1; j++) { + lndebug("clear+up"); + snprintf(seq,64,"\r\x1b[0K\x1b[1A"); + abAppend(&ab,seq,strlen(seq)); + } + + /* Clean the top line. */ + lndebug("clear"); + snprintf(seq,64,"\r\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + abAppend(&ab,l->buf,l->len); + + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + + /* If we are at the very end of the screen with our prompt, we need to + * emit a newline and move the prompt to the first column. */ + if (l->pos && + l->pos == l->len && + (l->pos+plen) % l->cols == 0) + { + lndebug("<newline>"); + abAppend(&ab,"\n",1); + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + rows++; + if (rows > (int)l->maxrows) l->maxrows = rows; + } + + /* Move cursor to right position. */ + rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */ + lndebug("rpos2 %d", rpos2); + + /* Go up till we reach the expected positon. */ + if (rows-rpos2 > 0) { + lndebug("go-up %d", rows-rpos2); + snprintf(seq,64,"\x1b[%dA", rows-rpos2); + abAppend(&ab,seq,strlen(seq)); + } + + /* Set column. */ + col = (plen+(int)l->pos) % (int)l->cols; + lndebug("set col %d", 1+col); + if (col) + snprintf(seq,64,"\r\x1b[%dC", col); + else + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + + lndebug("\n"); + l->oldpos = l->pos; + + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Calls the two low level functions refreshSingleLine() or + * refreshMultiLine() according to the selected mode. */ +static void refreshLine(struct linenoiseState *l) { + if (mlmode) + refreshMultiLine(l); + else + refreshSingleLine(l); +} + +/* Insert the character 'c' at cursor current position. + * + * On error writing to the terminal -1 is returned, otherwise 0. */ +int linenoiseEditInsert(struct linenoiseState *l, char c) { + if (l->len < l->buflen) { + if (l->len == l->pos) { + l->buf[l->pos] = c; + l->pos++; + l->len++; + l->buf[l->len] = '\0'; + if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { + /* Avoid a full update of the line in the + * trivial case. */ + if (write(l->ofd,&c,1) == -1) return -1; + } else { + refreshLine(l); + } + } else { + memmove(l->buf+l->pos+1,l->buf+l->pos,l->len-l->pos); + l->buf[l->pos] = c; + l->len++; + l->pos++; + l->buf[l->len] = '\0'; + refreshLine(l); + } + } + return 0; +} + +/* Move cursor on the left. */ +void linenoiseEditMoveLeft(struct linenoiseState *l) { + if (l->pos > 0) { + l->pos--; + refreshLine(l); + } +} + +/* Move cursor on the right. */ +void linenoiseEditMoveRight(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos++; + refreshLine(l); + } +} + +/* Move cursor to the start of the line. */ +void linenoiseEditMoveHome(struct linenoiseState *l) { + if (l->pos != 0) { + l->pos = 0; + refreshLine(l); + } +} + +/* Move cursor to the end of the line. */ +void linenoiseEditMoveEnd(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos = l->len; + refreshLine(l); + } +} + +/* Substitute the currently edited line with the next or previous history + * entry as specified by 'dir'. */ +#define LINENOISE_HISTORY_NEXT 0 +#define LINENOISE_HISTORY_PREV 1 +void linenoiseEditHistoryNext(struct linenoiseState *l, int dir) { + if (history_len > 1) { + /* Update the current history entry before to + * overwrite it with the next one. */ + free(history[history_len - 1 - l->history_index]); + history[history_len - 1 - l->history_index] = strdup(l->buf); + /* Show the new entry */ + l->history_index += (dir == LINENOISE_HISTORY_PREV) ? 1 : -1; + if (l->history_index < 0) { + l->history_index = 0; + return; + } else if (l->history_index >= history_len) { + l->history_index = history_len-1; + return; + } + strncpy(l->buf,history[history_len - 1 - l->history_index],l->buflen); + l->buf[l->buflen-1] = '\0'; + l->len = l->pos = strlen(l->buf); + refreshLine(l); + } +} + +/* Delete the character at the right of the cursor without altering the cursor + * position. Basically this is what happens with the "Delete" keyboard key. */ +void linenoiseEditDelete(struct linenoiseState *l) { + if (l->len > 0 && l->pos < l->len) { + memmove(l->buf+l->pos,l->buf+l->pos+1,l->len-l->pos-1); + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Backspace implementation. */ +void linenoiseEditBackspace(struct linenoiseState *l) { + if (l->pos > 0 && l->len > 0) { + memmove(l->buf+l->pos-1,l->buf+l->pos,l->len-l->pos); + l->pos--; + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Delete the previosu word, maintaining the cursor at the start of the + * current word. */ +void linenoiseEditDeletePrevWord(struct linenoiseState *l) { + size_t old_pos = l->pos; + size_t diff; + + while (l->pos > 0 && l->buf[l->pos-1] == ' ') + l->pos--; + while (l->pos > 0 && l->buf[l->pos-1] != ' ') + l->pos--; + diff = old_pos - l->pos; + memmove(l->buf+l->pos,l->buf+old_pos,l->len-old_pos+1); + l->len -= diff; + refreshLine(l); +} + +/* This function is the core of the line editing capability of linenoise. + * It expects 'fd' to be already in "raw mode" so that every key pressed + * will be returned ASAP to read(). + * + * The resulting string is put into 'buf' when the user type enter, or + * when ctrl+d is typed. + * + * The function returns the length of the current buffer. */ +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +{ + struct linenoiseState l; + + /* Populate the linenoise state that we pass to functions implementing + * specific editing functionalities. */ + l.ifd = stdin_fd; + l.ofd = stdout_fd; + l.buf = buf; + l.buflen = buflen; + l.prompt = prompt; + l.plen = strlen(prompt); + l.oldpos = l.pos = 0; + l.len = 0; + l.cols = getColumns(stdin_fd, stdout_fd); + l.maxrows = 0; + l.history_index = 0; + + /* Buffer starts empty. */ + l.buf[0] = '\0'; + l.buflen--; /* Make sure there is always space for the nulterm */ + + /* The latest history entry is always our current buffer, that + * initially is just an empty string. */ + linenoiseHistoryAdd(""); + + if (write(l.ofd,prompt,l.plen) == -1) return -1; + while(1) { + char c; + int nread; + char seq[3]; + + nread = read(l.ifd,&c,1); + if (nread <= 0) return l.len; + + /* Only autocomplete when the callback is set. It returns < 0 when + * there was an error reading from fd. Otherwise it will return the + * character that should be handled next. */ + if (c == 9 && completionCallback != NULL) { + c = completeLine(&l); + /* Return on errors */ + if (c < 0) return l.len; + /* Read next character when 0 */ + if (c == 0) continue; + } + + switch(c) { + case ENTER: /* enter */ + history_len--; + free(history[history_len]); + if (mlmode) linenoiseEditMoveEnd(&l); + if (hintsCallback) { + /* Force a refresh without hints to leave the previous + * line as the user typed it after a newline. */ + linenoiseHintsCallback *hc = hintsCallback; + hintsCallback = NULL; + refreshLine(&l); + hintsCallback = hc; + } + return (int)l.len; + case CTRL_C: /* ctrl-c */ + errno = EAGAIN; + return -1; + case BACKSPACE: /* backspace */ + case 8: /* ctrl-h */ + linenoiseEditBackspace(&l); + break; + case CTRL_D: /* ctrl-d, remove char at right of cursor, or if the + line is empty, act as end-of-file. */ + if (l.len > 0) { + linenoiseEditDelete(&l); + } else { + history_len--; + free(history[history_len]); + return -1; + } + break; + case CTRL_T: /* ctrl-t, swaps current character with previous. */ + if (l.pos > 0 && l.pos < l.len) { + int aux = buf[l.pos-1]; + buf[l.pos-1] = buf[l.pos]; + buf[l.pos] = aux; + if (l.pos != l.len-1) l.pos++; + refreshLine(&l); + } + break; + case CTRL_B: /* ctrl-b */ + linenoiseEditMoveLeft(&l); + break; + case CTRL_F: /* ctrl-f */ + linenoiseEditMoveRight(&l); + break; + case CTRL_P: /* ctrl-p */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case CTRL_N: /* ctrl-n */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case ESC: /* escape sequence */ + /* Read the next two bytes representing the escape sequence. + * Use two calls to handle slow terminals returning the two + * chars at different times. */ + if (read(l.ifd,seq,1) == -1) break; + if (read(l.ifd,seq+1,1) == -1) break; + + /* ESC [ sequences. */ + if (seq[0] == '[') { + if (seq[1] >= '0' && seq[1] <= '9') { + /* Extended escape, read additional byte. */ + if (read(l.ifd,seq+2,1) == -1) break; + if (seq[2] == '~') { + switch(seq[1]) { + case '3': /* Delete key. */ + linenoiseEditDelete(&l); + break; + } + } + } else { + switch(seq[1]) { + case 'A': /* Up */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case 'B': /* Down */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case 'C': /* Right */ + linenoiseEditMoveRight(&l); + break; + case 'D': /* Left */ + linenoiseEditMoveLeft(&l); + break; + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; + } + } + } + + /* ESC O sequences. */ + else if (seq[0] == 'O') { + switch(seq[1]) { + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; + } + } + break; + default: + if (linenoiseEditInsert(&l,c)) return -1; + break; + case CTRL_U: /* Ctrl+u, delete the whole line. */ + buf[0] = '\0'; + l.pos = l.len = 0; + refreshLine(&l); + break; + case CTRL_K: /* Ctrl+k, delete from current to end of line. */ + buf[l.pos] = '\0'; + l.len = l.pos; + refreshLine(&l); + break; + case CTRL_A: /* Ctrl+a, go to the start of the line */ + linenoiseEditMoveHome(&l); + break; + case CTRL_E: /* ctrl+e, go to the end of the line */ + linenoiseEditMoveEnd(&l); + break; + case CTRL_L: /* ctrl+l, clear screen */ + linenoiseClearScreen(); + refreshLine(&l); + break; + case CTRL_W: /* ctrl+w, delete previous word */ + linenoiseEditDeletePrevWord(&l); + break; + } + } + return l.len; +} + +/* This special mode is used by linenoise in order to print scan codes + * on screen for debugging / development purposes. It is implemented + * by the linenoise_example program using the --keycodes option. */ +void linenoisePrintKeyCodes(void) { + char quit[4]; + + printf("Linenoise key codes debugging mode.\n" + "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); + if (enableRawMode(STDIN_FILENO) == -1) return; + memset(quit,' ',4); + while(1) { + char c; + int nread; + + nread = read(STDIN_FILENO,&c,1); + if (nread <= 0) continue; + memmove(quit,quit+1,sizeof(quit)-1); /* shift string to left. */ + quit[sizeof(quit)-1] = c; /* Insert current char on the right. */ + if (memcmp(quit,"quit",sizeof(quit)) == 0) break; + + printf("'%c' %02x (%d) (type quit to exit)\n", + isprint(c) ? c : '?', (int)c, (int)c); + printf("\r"); /* Go left edge manually, we are in raw mode. */ + fflush(stdout); + } + disableRawMode(STDIN_FILENO); +} + +/* This function calls the line editing function linenoiseEdit() using + * the STDIN file descriptor set in raw mode. */ +static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { + int count; + + if (buflen == 0) { + errno = EINVAL; + return -1; + } + + if (enableRawMode(STDIN_FILENO) == -1) return -1; + count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); + disableRawMode(STDIN_FILENO); + printf("\n"); + return count; +} + +/* This function is called when linenoise() is called with the standard + * input file descriptor not attached to a TTY. So for example when the + * program using linenoise is called in pipe or with a file redirected + * to its standard input. In this case, we want to be able to return the + * line regardless of its length (by default we are limited to 4k). */ +static char *linenoiseNoTTY(void) { + char *line = NULL; + size_t len = 0, maxlen = 0; + + while(1) { + if (len == maxlen) { + if (maxlen == 0) maxlen = 16; + maxlen *= 2; + char *oldval = line; + line = realloc(line,maxlen); + if (line == NULL) { + if (oldval) free(oldval); + return NULL; + } + } + int c = fgetc(stdin); + if (c == EOF || c == '\n') { + if (c == EOF && len == 0) { + free(line); + return NULL; + } else { + line[len] = '\0'; + return line; + } + } else { + line[len] = c; + len++; + } + } +} + +/* The high level function that is the main API of the linenoise library. + * This function checks if the terminal has basic capabilities, just checking + * for a blacklist of stupid terminals, and later either calls the line + * editing function or uses dummy fgets() so that you will be able to type + * something even in the most desperate of the conditions. */ +char *linenoise(const char *prompt) { + char buf[LINENOISE_MAX_LINE]; + int count; + + if (!isatty(STDIN_FILENO)) { + /* Not a tty: read from file / pipe. In this mode we don't want any + * limit to the line size, so we call a function to handle that. */ + return linenoiseNoTTY(); + } else if (isUnsupportedTerm()) { + size_t len; + + printf("%s",prompt); + fflush(stdout); + if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL; + len = strlen(buf); + while(len && (buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + buf[len] = '\0'; + } + return strdup(buf); + } else { + count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt); + if (count == -1) return NULL; + return strdup(buf); + } +} + +/* This is just a wrapper the user may want to call in order to make sure + * the linenoise returned buffer is freed with the same allocator it was + * created with. Useful when the main program is using an alternative + * allocator. */ +void linenoiseFree(void *ptr) { + free(ptr); +} + +/* ================================ History ================================= */ + +/* Free the history, but does not reset it. Only used when we have to + * exit() to avoid memory leaks are reported by valgrind & co. */ +static void freeHistory(void) { + if (history) { + int j; + + for (j = 0; j < history_len; j++) + free(history[j]); + free(history); + } +} + +/* At exit we'll try to fix the terminal to the initial conditions. */ +static void linenoiseAtExit(void) { + disableRawMode(STDIN_FILENO); + freeHistory(); +} + +/* This is the API call to add a new entry in the linenoise history. + * It uses a fixed array of char pointers that are shifted (memmoved) + * when the history max length is reached in order to remove the older + * entry and make room for the new one, so it is not exactly suitable for huge + * histories, but will work well for a few hundred of entries. + * + * Using a circular buffer is smarter, but a bit more complex to handle. */ +int linenoiseHistoryAdd(const char *line) { + char *linecopy; + + if (history_max_len == 0) return 0; + + /* Initialization on first call. */ + if (history == NULL) { + history = malloc(sizeof(char*)*history_max_len); + if (history == NULL) return 0; + memset(history,0,(sizeof(char*)*history_max_len)); + } + + /* Don't add duplicated lines. */ + if (history_len && !strcmp(history[history_len-1], line)) return 0; + + /* Add an heap allocated copy of the line in the history. + * If we reached the max length, remove the older line. */ + linecopy = strdup(line); + if (!linecopy) return 0; + if (history_len == history_max_len) { + free(history[0]); + memmove(history,history+1,sizeof(char*)*(history_max_len-1)); + history_len--; + } + history[history_len] = linecopy; + history_len++; + return 1; +} + +/* Set the maximum length for the history. This function can be called even + * if there is already some history, the function will make sure to retain + * just the latest 'len' elements if the new history length value is smaller + * than the amount of items already inside the history. */ +int linenoiseHistorySetMaxLen(int len) { + char **new; + + if (len < 1) return 0; + if (history) { + int tocopy = history_len; + + new = malloc(sizeof(char*)*len); + if (new == NULL) return 0; + + /* If we can't copy everything, free the elements we'll not use. */ + if (len < tocopy) { + int j; + + for (j = 0; j < tocopy-len; j++) free(history[j]); + tocopy = len; + } + memset(new,0,sizeof(char*)*len); + memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy); + free(history); + history = new; + } + history_max_len = len; + if (history_len > history_max_len) + history_len = history_max_len; + return 1; +} + +/* Save the history in the specified file. On success 0 is returned + * otherwise -1 is returned. */ +int linenoiseHistorySave(const char *filename) { + mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO); + FILE *fp; + int j; + + fp = fopen(filename,"w"); + umask(old_umask); + if (fp == NULL) return -1; + chmod(filename,S_IRUSR|S_IWUSR); + for (j = 0; j < history_len; j++) + fprintf(fp,"%s\n",history[j]); + fclose(fp); + return 0; +} + +/* Load the history from the specified file. If the file does not exist + * zero is returned and no operation is performed. + * + * If the file exists and the operation succeeded 0 is returned, otherwise + * on error -1 is returned. */ +int linenoiseHistoryLoad(const char *filename) { + FILE *fp = fopen(filename,"r"); + char buf[LINENOISE_MAX_LINE]; + + if (fp == NULL) return -1; + + while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) { + char *p; + + p = strchr(buf,'\r'); + if (!p) p = strchr(buf,'\n'); + if (p) *p = '\0'; + linenoiseHistoryAdd(buf); + } + fclose(fp); + return 0; +} diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h new file mode 100644 index 000000000000..ed20232c576e --- /dev/null +++ b/src/linenoise/linenoise.h @@ -0,0 +1,73 @@ +/* linenoise.h -- VERSION 1.0 + * + * Guerrilla line editing library against the idea that a line editing lib + * needs to be 20,000 lines of C code. + * + * See linenoise.c for more information. + * + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com> + * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __LINENOISE_H +#define __LINENOISE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct linenoiseCompletions { + size_t len; + char **cvec; +} linenoiseCompletions; + +typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); +typedef void(linenoiseFreeHintsCallback)(void *); +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); +void linenoiseSetHintsCallback(linenoiseHintsCallback *); +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); +void linenoiseAddCompletion(linenoiseCompletions *, const char *); + +char *linenoise(const char *prompt); +void linenoiseFree(void *ptr); +int linenoiseHistoryAdd(const char *line); +int linenoiseHistorySetMaxLen(int len); +int linenoiseHistorySave(const char *filename); +int linenoiseHistoryLoad(const char *filename); +void linenoiseClearScreen(void); +void linenoiseSetMultiLine(int ml); +void linenoisePrintKeyCodes(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __LINENOISE_H */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 065447684aaf..7167e96f11d3 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -15,6 +15,7 @@ #include "shared.hh" using namespace nix; +using namespace std::string_literals; extern char * * environ; @@ -325,7 +326,7 @@ int main(int argc, char ** argv) if (packages) { instArgs.push_back("--expr"); std::ostringstream joined; - joined << "with import <nixpkgs> { }; runCommand \"shell\" { buildInputs = [ "; + joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; for (const auto & i : exprs) joined << '(' << i << ") "; joined << "]; } \"\""; @@ -408,8 +409,20 @@ int main(int argc, char ** argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = settings.buildCores; + auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", "")); + + bool keepTmp = false; + int fileNr = 0; + for (auto & var : drv.env) - env[var.first] = var.second; + if (passAsFile.count(var.first)) { + keepTmp = true; + string fn = ".attr-" + std::to_string(fileNr++); + Path p = (Path) tmpDir + "/" + fn; + writeFile(p, var.second); + env[var.first + "Path"] = p; + } else + env[var.first] = var.second; restoreAffinity(); @@ -419,14 +432,14 @@ int main(int argc, char ** argv) // the current $PATH directories. auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( - "rm -rf '%1%'; " + (keepTmp ? "" : "rm -rf '%1%'; "s) + "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" "set +e; " - "[ -n \"$PS1\" ] && PS1=\"\\n\\[\\033[1;32m\\][nix-shell:\\w]$\\[\\033[0m\\] \"; " + R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "unset NIX_INDENT_MAKE; " diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 361627823126..f2742bc3bbda 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -103,19 +103,15 @@ static void update(const StringSet & channelNames) auto unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - try { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " - "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); - unpacked = true; - } catch (ExecError & e) { - } + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); + unpacked = true; } if (!unpacked) { // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it // Check if the channel advertises a binary cache. DownloadRequest request(url + "/binary-cache-url"); - request.showProgress = DownloadRequest::no; try { auto dlRes = dl->download(request); extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; @@ -172,9 +168,7 @@ int main(int argc, char ** argv) setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1); // Figure out the name of the `.nix-channels' file to use - auto home = getEnv("HOME"); - if (home.empty()) - throw Error("$HOME not set"); + auto home = getHome(); channelsList = home + "/.nix-channels"; nixDefExpr = home + "/.nix-defexpr"; diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index ed43bffbc8c8..dc324abcb3ba 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -58,6 +58,6 @@ int main(int argc, char ** argv) PathSet closure; from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, closure, useSubstitutes); + copyPaths(from, to, closure, useSubstitutes, true); }); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8786e2561b9c..44127635ded8 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -82,12 +82,7 @@ class TunnelLogger : public Logger defaultLogger->log(lvl, fs); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - log(lvl, fs); - } - - void stopActivity(Activity & activity) override + void event(const Event & ev) override { } }; @@ -436,31 +431,70 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe } case wopSetOptions: { - from >> settings.keepFailed; - from >> settings.keepGoing; - settings.set("build-fallback", readInt(from) ? "true" : "false"); + settings.keepFailed = readInt(from); + settings.keepGoing = readInt(from); + settings.tryFallback = readInt(from); verbosity = (Verbosity) readInt(from); - settings.set("build-max-jobs", std::to_string(readInt(from))); - settings.set("build-max-silent-time", std::to_string(readInt(from))); + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); settings.useBuildHook = readInt(from) != 0; settings.verboseBuild = lvlError == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace - settings.set("build-cores", std::to_string(readInt(from))); - settings.set("build-use-substitutes", readInt(from) ? "true" : "false"); + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from); + + StringMap overrides; if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { unsigned int n = readInt(from); for (unsigned int i = 0; i < n; i++) { string name = readString(from); string value = readString(from); - if (name == "build-timeout" || name == "use-ssh-substituter") + overrides.emplace(name, value); + } + } + + startWork(); + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting<Strings> & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString<Strings>(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == settings.connectTimeout.name) settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; else - settings.set(trusted ? name : "untrusted-" + name, value); + debug("ignoring untrusted setting '%s'", name); + } catch (UsageError & e) { + warn(e.what()); } } - settings.update(); - startWork(); + stopWork(); break; } @@ -582,6 +616,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe from >> info.ca >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; + if (!trusted) + info.ultimate = false; TeeSink tee(from); parseDump(tee, tee.source); @@ -878,8 +914,8 @@ static void daemonLoop(char * * argv) struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.get("trusted-users", Strings({"root"})); - Strings allowedUsers = settings.get("allowed-users", Strings({"*"})); + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = true; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 908c09bc8c8a..464bcee4a848 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -192,17 +192,9 @@ static void loadDerivations(EvalState & state, Path nixExprPath, } -static Path getHomeDir() -{ - Path homeDir(getEnv("HOME", "")); - if (homeDir == "") throw Error("HOME environment variable not set"); - return homeDir; -} - - static Path getDefNixExprPath() { - return getHomeDir() + "/.nix-defexpr"; + return getHome() + "/.nix-defexpr"; } @@ -997,7 +989,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); + //Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); if (globals.prebuiltOnly && validPaths.find(i.queryOutPath()) == validPaths.end() && @@ -1188,7 +1180,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError(format("exactly one argument expected")); Path profile = absPath(opArgs.front()); - Path profileLink = getHomeDir() + "/.nix-profile"; + Path profileLink = getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1413,7 +1405,7 @@ int main(int argc, char * * argv) globals.profile = getEnv("NIX_PROFILE", ""); if (globals.profile == "") { - Path profileLink = getHomeDir() + "/.nix-profile"; + Path profileLink = getHome() + "/.nix-profile"; globals.profile = pathExists(profileLink) ? absPath(readLink(profileLink), dirOf(profileLink)) : canonPath(settings.nixStateDir + "/profiles/default"); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c1b0b0ea092d..25f0b1bd692e 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -19,7 +19,7 @@ using namespace nix; static Expr * parseStdin(EvalState & state) { - Activity act(*logger, lvlTalkative, format("parsing standard input")); + //Activity act(*logger, lvlTalkative, format("parsing standard input")); return state.parseExprFromString(drainFD(0), absPath(".")); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3dc167191c83..950222812e2b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -145,7 +145,7 @@ static void opRealise(Strings opFlags, Strings opArgs) unknown = PathSet(); } - if (settings.get("print-missing", true)) + if (settings.printMissing) printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (dryRun) return; @@ -795,11 +795,11 @@ static void opServe(Strings opFlags, Strings opArgs) settings.maxSilentTime = readInt(in); settings.buildTimeout = readInt(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - in >> settings.maxLogSize; + settings.maxLogSize = readNum<unsigned long>(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.set("build-repeat", std::to_string(readInt(in))); - settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); - settings.set("run-diff-hook", "true"); + settings.buildRepeat = readInt(in); + settings.enforceDeterminism = readInt(in); + settings.runDiffHook = true; } settings.printRepeatedBuilds = false; }; diff --git a/src/nix/build.cc b/src/nix/build.cc index 812464d7582b..00bda1fd1045 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,12 +1,11 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" using namespace nix; -struct CmdBuild : StoreCommand, MixDryRun, MixInstallables +struct CmdBuild : MixDryRun, InstallablesCommand { CmdBuild() { @@ -24,22 +23,9 @@ struct CmdBuild : StoreCommand, MixDryRun, MixInstallables void run(ref<Store> store) override { - auto elems = evalInstallables(store); + auto paths = buildInstallables(store, dryRun); - PathSet pathsToBuild; - - for (auto & elem : elems) { - if (elem.isDrv) - pathsToBuild.insert(elem.drvPath); - else - pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - - printMissing(store, pathsToBuild); - - if (dryRun) return; - - store->buildPaths(pathsToBuild); + printInfo("build result: %s", showPaths(paths)); } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index a1b2c120a5d9..3c82e0df57f7 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "derivations.hh" namespace nix { @@ -79,6 +80,13 @@ StoreCommand::StoreCommand() mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); } +ref<Store> StoreCommand::getStore() +{ + if (!_store) + _store = createStore(); + return ref<Store>(_store); +} + ref<Store> StoreCommand::createStore() { return openStore(storeUri); @@ -91,23 +99,24 @@ void StoreCommand::run() StorePathsCommand::StorePathsCommand() { - expectArgs("paths", &storePaths); mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); mkFlag(0, "all", "apply operation to the entire store", &all); } void StorePathsCommand::run(ref<Store> store) { + Paths storePaths; + if (all) { - if (storePaths.size()) + if (installables.size()) throw UsageError("‘--all’ does not expect arguments"); for (auto & p : store->queryAllValidPaths()) storePaths.push_back(p); } else { - for (auto & storePath : storePaths) - storePath = store->followLinksToStorePath(storePath); + for (auto & p : buildInstallables(store, false)) + storePaths.push_back(p); if (recursive) { PathSet closure; @@ -120,4 +129,14 @@ void StorePathsCommand::run(ref<Store> store) run(store, storePaths); } +void StorePathCommand::run(ref<Store> store) +{ + auto storePaths = buildInstallables(store, false); + + if (storePaths.size() != 1) + throw UsageError("this command requires exactly one store path"); + + run(store, *storePaths.begin()); +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index fa6c21abf8ad..4800b5c912e4 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -4,6 +4,9 @@ namespace nix { +struct Value; +class EvalState; + /* A command is an argument parser that can be executed by calling its run() method. */ struct Command : virtual Args @@ -33,16 +36,72 @@ struct StoreCommand : virtual Command std::string storeUri; StoreCommand(); void run() override; + ref<Store> getStore(); virtual ref<Store> createStore(); virtual void run(ref<Store>) = 0; + +private: + std::shared_ptr<Store> _store; +}; + +struct Installable +{ + virtual std::string what() = 0; + + virtual PathSet toBuildable() + { + throw Error("argument ‘%s’ cannot be built", what()); + } + + virtual Value * toValue(EvalState & state) + { + throw Error("argument ‘%s’ cannot be evaluated", what()); + } +}; + +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : virtual Args, StoreCommand +{ + std::vector<std::shared_ptr<Installable>> installables; + Path file; + + InstallablesCommand() + { + mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); + expectArgs("installables", &_installables); + } + + /* Return a value representing the Nix expression from which we + are installing. This is either the file specified by ‘--file’, + or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs + = import ...; bla = import ...; }’. */ + Value * getSourceExpr(EvalState & state); + + std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss); + + PathSet buildInstallables(ref<Store> store, bool dryRun); + + ref<EvalState> getEvalState(); + + void prepare() override; + + virtual bool useDefaultInstallables() { return true; } + +private: + + Strings _installables; + + std::shared_ptr<EvalState> evalState; + + Value * vSourceExpr = 0; }; /* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public StoreCommand +struct StorePathsCommand : public InstallablesCommand { private: - Paths storePaths; bool recursive = false; bool all = false; @@ -55,6 +114,18 @@ public: virtual void run(ref<Store> store, Paths storePaths) = 0; void run(ref<Store> store) override; + + bool useDefaultInstallables() override { return !all; } +}; + +/* A command that operates on exactly one store path. */ +struct StorePathCommand : public InstallablesCommand +{ + using StoreCommand::run; + + virtual void run(ref<Store> store, const Path & storePath) = 0; + + void run(ref<Store> store) override; }; typedef std::map<std::string, ref<Command>> Commands; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc new file mode 100644 index 000000000000..1a1866437b07 --- /dev/null +++ b/src/nix/dump-path.cc @@ -0,0 +1,35 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdDumpPath : StorePathCommand +{ + std::string name() override + { + return "dump-path"; + } + + std::string description() override + { + return "dump a store path to stdout (in NAR format)"; + } + + Examples examples() override + { + return { + Example{ + "To get a NAR from the binary cache https://cache.nixos.org/:", + "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" + }, + }; + } + + void run(ref<Store> store, const Path & storePath) override + { + FdSink sink(STDOUT_FILENO); + store->narFromPath(storePath, sink); + } +}; + +static RegisterCommand r1(make_ref<CmdDumpPath>()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc new file mode 100644 index 000000000000..632b555771c2 --- /dev/null +++ b/src/nix/edit.cc @@ -0,0 +1,75 @@ +#include "command.hh" +#include "shared.hh" +#include "eval.hh" +#include "attr-path.hh" + +#include <unistd.h> + +using namespace nix; + +struct CmdEdit : InstallablesCommand +{ + std::string name() override + { + return "edit"; + } + + std::string description() override + { + return "open the Nix expression of a Nix package in $EDITOR"; + } + + Examples examples() override + { + return { + Example{ + "To open the Nix expression of the GNU Hello package:", + "nix edit nixpkgs.hello" + }, + }; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + + for (auto & i : installables) { + auto v = i->toValue(*state); + + Value * v2; + try { + auto dummyArgs = state->allocBindings(0); + v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v); + } catch (Error &) { + throw Error("package ‘%s’ has no source location information", i->what()); + } + + auto pos = state->forceString(*v2); + debug("position is %s", pos); + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute ‘%s’", pos); + + std::string filename(pos, 0, colon); + int lineno = std::stoi(std::string(pos, colon + 1)); + + auto editor = getEnv("EDITOR", "cat"); + + Strings args{editor}; + + if (editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos) + args.push_back(fmt("+%d", lineno)); + + args.push_back(filename); + + execvp(editor.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("cannot run editor ‘%s’", editor); + } + } +}; + +static RegisterCommand r1(make_ref<CmdEdit>()); diff --git a/src/nix/eval.cc b/src/nix/eval.cc new file mode 100644 index 000000000000..981cefa800c4 --- /dev/null +++ b/src/nix/eval.cc @@ -0,0 +1,55 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "eval.hh" +#include "json.hh" +#include "value-to-json.hh" + +using namespace nix; + +struct CmdEval : MixJSON, InstallablesCommand +{ + bool raw = false; + + CmdEval() + { + mkFlag(0, "raw", "print strings unquoted", &raw); + } + + std::string name() override + { + return "eval"; + } + + std::string description() override + { + return "evaluate a Nix expression"; + } + + void run(ref<Store> store) override + { + if (raw && json) + throw UsageError("--raw and --json are mutually exclusive"); + + auto state = getEvalState(); + + auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr; + + for (auto & i : installables) { + auto v = i->toValue(*state); + if (raw) { + std::cout << state->forceString(*v); + } else if (json) { + PathSet context; + auto jsonElem = jsonOut->placeholder(); + printValueAsJSON(*state, true, *v, jsonElem, context); + } else { + state->forceValueDeep(*v); + std::cout << *v << "\n"; + } + } + } +}; + +static RegisterCommand r1(make_ref<CmdEval>()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 8341bbc5a3a4..f23308b9bc30 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,21 +1,26 @@ +#include "command.hh" #include "attr-path.hh" #include "common-opts.hh" #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" #include "get-drvs.hh" -#include "installables.hh" #include "store-api.hh" +#include "shared.hh" + +#include <regex> namespace nix { -Value * MixInstallables::buildSourceExpr(EvalState & state) +Value * InstallablesCommand::getSourceExpr(EvalState & state) { - Value * vRoot = state.allocValue(); + if (vSourceExpr) return vSourceExpr; + + vSourceExpr = state.allocValue(); if (file != "") { Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); - state.eval(e, *vRoot); + state.eval(e, *vSourceExpr); } else { @@ -24,7 +29,7 @@ Value * MixInstallables::buildSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vRoot, searchPath.size()); + state.mkAttrs(*vSourceExpr, searchPath.size()); std::unordered_set<std::string> seen; @@ -32,76 +37,219 @@ Value * MixInstallables::buildSourceExpr(EvalState & state) if (i.first == "") continue; if (seen.count(i.first)) continue; seen.insert(i.first); - if (!pathExists(i.second)) continue; - mkApp(*state.allocAttr(*vRoot, state.symbols.create(i.first)), +#if 0 + auto res = state.resolveSearchPathElem(i); + if (!res.first) continue; + if (!pathExists(res.second)) continue; + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), state.getBuiltin("import"), - mkString(*state.allocValue(), i.second)); + mkString(*state.allocValue(), res.second)); +#endif + Value * v1 = state.allocValue(); + mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); + Value * v2 = state.allocValue(); + mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), + state.getBuiltin("import"), *v2); } - vRoot->attrs->sort(); + vSourceExpr->attrs->sort(); } - return vRoot; + return vSourceExpr; } -UserEnvElems MixInstallables::evalInstallables(ref<Store> store) +struct InstallableStoreDrv : Installable { - UserEnvElems res; - - for (auto & installable : installables) { - - if (std::string(installable, 0, 1) == "/") { - - if (store->isStorePath(installable)) { - - if (isDerivation(installable)) { - UserEnvElem elem; - // FIXME: handle empty case, drop version - elem.attrPath = {storePathToName(installable)}; - elem.isDrv = true; - elem.drvPath = installable; - res.push_back(elem); - } - - else { - UserEnvElem elem; - // FIXME: handle empty case, drop version - elem.attrPath = {storePathToName(installable)}; - elem.isDrv = false; - elem.outPaths = {installable}; - res.push_back(elem); - } - } + Path storePath; - else - throw UsageError(format("don't know what to do with ‘%1%’") % installable); - } + InstallableStoreDrv(const Path & storePath) : storePath(storePath) { } + + std::string what() override { return storePath; } + + PathSet toBuildable() override + { + return {storePath}; + } +}; + +struct InstallableStorePath : Installable +{ + Path storePath; + + InstallableStorePath(const Path & storePath) : storePath(storePath) { } + + std::string what() override { return storePath; } + + PathSet toBuildable() override + { + return {storePath}; + } +}; + +struct InstallableExpr : Installable +{ + InstallablesCommand & installables; + std::string text; + + InstallableExpr(InstallablesCommand & installables, const std::string & text) + : installables(installables), text(text) { } + + std::string what() override { return text; } + + PathSet toBuildable() override + { + auto state = installables.getEvalState(); + + auto v = toValue(*state); + + // FIXME + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_)); + + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); + + PathSet res; + + for (auto & i : drvs) + res.insert(i.queryDrvPath()); + + return res; + } + + Value * toValue(EvalState & state) override + { + auto v = state.allocValue(); + state.eval(state.parseExprFromString(text, absPath(".")), *v); + return v; + } +}; + +struct InstallableAttrPath : Installable +{ + InstallablesCommand & installables; + std::string attrPath; + + InstallableAttrPath(InstallablesCommand & installables, const std::string & attrPath) + : installables(installables), attrPath(attrPath) + { } + + std::string what() override { return attrPath; } - else { + PathSet toBuildable() override + { + auto state = installables.getEvalState(); - EvalState state({}, store); + auto v = toValue(*state); - auto vRoot = buildSourceExpr(state); + // FIXME + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_)); - std::map<string, string> autoArgs_; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); - Value & v(*findAlongAttrPath(state, installable, autoArgs, *vRoot)); - state.forceValue(v); + PathSet res; - DrvInfos drvs; - getDerivations(state, v, "", autoArgs, drvs, false); + for (auto & i : drvs) + res.insert(i.queryDrvPath()); - for (auto & i : drvs) { - UserEnvElem elem; - elem.isDrv = true; - elem.drvPath = i.queryDrvPath(); - res.push_back(elem); + return res; + } + + Value * toValue(EvalState & state) override + { + auto source = installables.getSourceExpr(state); + + // FIXME + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + + Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); + state.forceValue(*v); + + return v; + } +}; + +// FIXME: extend +std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; +static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); + +std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings ss) +{ + std::vector<std::shared_ptr<Installable>> result; + + if (ss.empty() && useDefaultInstallables()) { + if (file == "") + file = "."; + ss = Strings{""}; + } + + for (auto & s : ss) { + + if (s.find("/") != std::string::npos) { + + auto path = store->toStorePath(store->followLinksToStore(s)); + + if (store->isStorePath(path)) { + if (isDerivation(path)) + result.push_back(std::make_shared<InstallableStoreDrv>(path)); + else + result.push_back(std::make_shared<InstallableStorePath>(path)); } } + + else if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(*this, s)); + + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared<InstallableAttrPath>(*this, s)); + + else + throw UsageError("don't know what to do with argument ‘%s’", s); } - return res; + return result; +} + +PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun) +{ + PathSet buildables; + + for (auto & i : installables) { + auto b = i->toBuildable(); + buildables.insert(b.begin(), b.end()); + } + + if (dryRun) + printMissing(store, buildables); + else + store->buildPaths(buildables); + + PathSet outPaths; + for (auto & path : buildables) + if (isDerivation(path)) { + Derivation drv = store->derivationFromPath(path); + for (auto & output : drv.outputs) + outPaths.insert(output.second.path); + } else + outPaths.insert(path); + + return outPaths; +} + +ref<EvalState> InstallablesCommand::getEvalState() +{ + if (!evalState) + evalState = std::make_shared<EvalState>(Strings{}, getStore()); + return ref<EvalState>(evalState); +} + +void InstallablesCommand::prepare() +{ + installables = parseInstallables(getStore(), _installables); } } diff --git a/src/nix/installables.hh b/src/nix/installables.hh deleted file mode 100644 index a58f7dc59bb4..000000000000 --- a/src/nix/installables.hh +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "args.hh" - -namespace nix { - -struct UserEnvElem -{ - Strings attrPath; - - // FIXME: should use boost::variant or so. - bool isDrv; - - // Derivation case: - Path drvPath; - StringSet outputNames; - - // Non-derivation case: - PathSet outPaths; -}; - -typedef std::vector<UserEnvElem> UserEnvElems; - -struct Value; -class EvalState; - -struct MixInstallables : virtual Args -{ - Strings installables; - Path file; - - MixInstallables() - { - mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); - expectArgs("installables", &installables); - } - - UserEnvElems evalInstallables(ref<Store> store); - - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * buildSourceExpr(EvalState & state); - -}; - -} diff --git a/src/nix/local.mk b/src/nix/local.mk index f6e7073b6e7d..c7d2d328aab5 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -2,7 +2,7 @@ programs += nix nix_DIR := $(d) -nix_SOURCES := $(wildcard $(d)/*.cc) +nix_SOURCES := $(wildcard $(d)/*.cc) src/linenoise/linenoise.c nix_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix/log.cc b/src/nix/log.cc index d8a3830e91c8..ed610261d1ca 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,12 +1,11 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" using namespace nix; -struct CmdLog : StoreCommand, MixInstallables +struct CmdLog : InstallablesCommand { CmdLog() { @@ -24,32 +23,23 @@ struct CmdLog : StoreCommand, MixInstallables void run(ref<Store> store) override { - auto elems = evalInstallables(store); - - PathSet paths; - - for (auto & elem : elems) { - if (elem.isDrv) - paths.insert(elem.drvPath); - else - paths.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - auto subs = getDefaultSubstituters(); subs.push_front(store); - for (auto & path : paths) { - bool found = false; - for (auto & sub : subs) { - auto log = sub->getBuildLog(path); - if (!log) continue; - std::cout << *log; - found = true; - break; + for (auto & inst : installables) { + for (auto & path : inst->toBuildable()) { + bool found = false; + for (auto & sub : subs) { + auto log = sub->getBuildLog(path); + if (!log) continue; + std::cout << *log; + found = true; + break; + } + if (!found) + throw Error("build log of path ‘%s’ is not available", path); } - if (!found) - throw Error("build log of path ‘%s’ is not available", path); } } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3476dfb05287..417b7b421b1c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -61,6 +61,10 @@ struct MixLs : virtual Args showFile(curPath, relPath); }; + if (path == "/") { + path = ""; + } + auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) throw Error(format("path ‘%1%’ does not exist") % path); diff --git a/src/nix/main.cc b/src/nix/main.cc index 440ced97dfcc..216f0bccef11 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -27,6 +27,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { + verbosity = lvlError; settings.verboseBuild = false; initNix(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 0f9a1125f2e9..f16209238610 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,25 +2,24 @@ #include "shared.hh" #include "store-api.hh" #include "json.hh" +#include "common-args.hh" #include <iomanip> #include <algorithm> using namespace nix; -struct CmdPathInfo : StorePathsCommand +struct CmdPathInfo : StorePathsCommand, MixJSON { bool showSize = false; bool showClosureSize = false; bool showSigs = false; - bool json = false; CmdPathInfo() { mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); mkFlag(0, "sigs", "show signatures", &showSigs); - mkFlag(0, "json", "produce JSON output", &json); } std::string name() override @@ -100,7 +99,6 @@ struct CmdPathInfo : StorePathsCommand } } - } }; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 69811b282804..24e435f81e8b 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -1,8 +1,12 @@ #include "progress-bar.hh" #include "util.hh" #include "sync.hh" +#include "store-api.hh" #include <map> +#include <atomic> + +#include <sys/ioctl.h> namespace nix { @@ -12,31 +16,47 @@ private: struct ActInfo { - Activity * activity; - Verbosity lvl; - std::string s; + std::string s, s2; }; - struct Progress + struct DownloadInfo { - uint64_t expected = 0, progress = 0; + std::string uri; + uint64_t current = 0; + uint64_t expected = 0; }; struct State { + std::map<Activity::t, Path> builds; + std::set<Activity::t> runningBuilds; + uint64_t succeededBuilds = 0; + uint64_t failedBuilds = 0; + std::map<Activity::t, Path> substitutions; + std::set<Activity::t> runningSubstitutions; + uint64_t succeededSubstitutions = 0; + uint64_t downloadedBytes = 0; // finished downloads + std::map<Activity::t, DownloadInfo> downloads; std::list<ActInfo> activities; - std::map<Activity *, std::list<ActInfo>::iterator> its; - std::map<std::string, Progress> progress; + std::map<Activity::t, std::list<ActInfo>::iterator> its; }; Sync<State> state_; + int width = 0; + public: + ProgressBar() + { + struct winsize ws; + if (ioctl(1, TIOCGWINSZ, &ws) == 0) + width = ws.ws_col; + } + ~ProgressBar() { auto state(state_.lock()); - assert(state->activities.empty()); writeToStderr("\r\e[K"); } @@ -52,52 +72,36 @@ public: update(state); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - auto state(state_.lock()); - state->activities.emplace_back(ActInfo{&activity, lvl, fs.s}); - state->its.emplace(&activity, std::prev(state->activities.end())); - update(*state); - } - - void stopActivity(Activity & activity) override - { - auto state(state_.lock()); - auto i = state->its.find(&activity); - if (i == state->its.end()) return; - state->activities.erase(i->second); - state->its.erase(i); - update(*state); - } - - void setExpected(const std::string & label, uint64_t value) override + void createActivity(State & state, Activity::t activity, const std::string & s) { - auto state(state_.lock()); - state->progress[label].expected = value; - } - - void setProgress(const std::string & label, uint64_t value) override - { - auto state(state_.lock()); - state->progress[label].progress = value; + state.activities.emplace_back(ActInfo{s}); + state.its.emplace(activity, std::prev(state.activities.end())); } - void incExpected(const std::string & label, uint64_t value) override + void deleteActivity(State & state, Activity::t activity) { - auto state(state_.lock()); - state->progress[label].expected += value; + auto i = state.its.find(activity); + if (i != state.its.end()) { + state.activities.erase(i->second); + state.its.erase(i); + } } - void incProgress(const std::string & label, uint64_t value) override + void updateActivity(State & state, Activity::t activity, const std::string & s2) { - auto state(state_.lock()); - state->progress[label].progress += value; + auto i = state.its.find(activity); + assert(i != state.its.end()); + ActInfo info = *i->second; + state.activities.erase(i->second); + info.s2 = s2; + state.activities.emplace_back(info); + i->second = std::prev(state.activities.end()); } void update() { auto state(state_.lock()); + update(*state); } void update(State & state) @@ -113,28 +117,169 @@ public: if (!state.activities.empty()) { if (!status.empty()) line += " "; - line += state.activities.rbegin()->s; + auto i = state.activities.rbegin(); + line += i->s; + if (!i->s2.empty()) { + line += ": "; + line += i->s2; + } } line += "\e[K"; - writeToStderr(line); + writeToStderr(std::string(line, 0, width - 1)); } std::string getStatus(State & state) { std::string res; - for (auto & p : state.progress) - if (p.second.expected || p.second.progress) { - if (!res.empty()) res += ", "; - res += std::to_string(p.second.progress); - if (p.second.expected) { - res += "/"; - res += std::to_string(p.second.expected); - } - res += " "; res += p.first; + + if (state.failedBuilds) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds); + } + + if (!state.builds.empty() || state.succeededBuilds) + { + if (!res.empty()) res += ", "; + if (!state.runningBuilds.empty()) + res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningBuilds.size()); + res += fmt(ANSI_GREEN "%d/%d built" ANSI_NORMAL, + state.succeededBuilds, state.succeededBuilds + state.builds.size()); + } + + if (!state.substitutions.empty() || state.succeededSubstitutions) { + if (!res.empty()) res += ", "; + if (!state.runningSubstitutions.empty()) + res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningSubstitutions.size()); + res += fmt(ANSI_GREEN "%d/%d fetched" ANSI_NORMAL, + state.succeededSubstitutions, + state.succeededSubstitutions + state.substitutions.size()); + } + + if (!state.downloads.empty() || state.downloadedBytes) { + if (!res.empty()) res += ", "; + uint64_t expected = state.downloadedBytes, current = state.downloadedBytes; + for (auto & i : state.downloads) { + expected += i.second.expected; + current += i.second.current; } + res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0); + } + return res; } + + void event(const Event & ev) override + { + if (ev.type == evBuildCreated) { + auto state(state_.lock()); + state->builds[ev.getI(0)] = ev.getS(1); + update(*state); + } + + if (ev.type == evBuildStarted) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + state->runningBuilds.insert(act); + auto name = storePathToName(state->builds[act]); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name)); + update(*state); + } + + if (ev.type == evBuildFinished) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + if (ev.getI(1)) { + if (state->runningBuilds.count(act)) + state->succeededBuilds++; + } else + state->failedBuilds++; + state->runningBuilds.erase(act); + state->builds.erase(act); + deleteActivity(*state, act); + update(*state); + } + + if (ev.type == evBuildOutput) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + assert(state->runningBuilds.count(act)); + updateActivity(*state, act, ev.getS(1)); + update(*state); + } + + if (ev.type == evSubstitutionCreated) { + auto state(state_.lock()); + state->substitutions[ev.getI(0)] = ev.getS(1); + update(*state); + } + + if (ev.type == evSubstitutionStarted) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + state->runningSubstitutions.insert(act); + auto name = storePathToName(state->substitutions[act]); + createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name)); + update(*state); + } + + if (ev.type == evSubstitutionFinished) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + if (ev.getI(1)) { + if (state->runningSubstitutions.count(act)) + state->succeededSubstitutions++; + } + state->runningSubstitutions.erase(act); + state->substitutions.erase(act); + deleteActivity(*state, act); + update(*state); + } + + if (ev.type == evDownloadCreated) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + std::string uri = ev.getS(1); + state->downloads.emplace(act, DownloadInfo{uri}); + if (state->runningSubstitutions.empty()) // FIXME: hack + createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri)); + update(*state); + } + + if (ev.type == evDownloadProgress) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto i = state->downloads.find(act); + assert(i != state->downloads.end()); + i->second.expected = ev.getI(1); + i->second.current = ev.getI(2); + update(*state); + } + + if (ev.type == evDownloadSucceeded) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto i = state->downloads.find(act); + assert(i != state->downloads.end()); + state->downloadedBytes += ev.getI(1); + state->downloads.erase(i); + deleteActivity(*state, act); + update(*state); + } + + if (ev.type == evDownloadDestroyed) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto i = state->downloads.find(act); + if (i != state->downloads.end()) { + state->downloads.erase(i); + deleteActivity(*state, act); + update(*state); + } + } + } }; StartProgressBar::StartProgressBar() diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 000000000000..437c7903ed40 --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,689 @@ +#include <iostream> +#include <cstdlib> + +#include <setjmp.h> + +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "common-opts.hh" +#include "get-drvs.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "globals.hh" +#include "command.hh" +#include "finally.hh" + +#include "src/linenoise/linenoise.h" + +namespace nix { + +#define ESC_RED "\033[31m" +#define ESC_GRE "\033[32m" +#define ESC_YEL "\033[33m" +#define ESC_BLU "\033[34;1m" +#define ESC_MAG "\033[35m" +#define ESC_CYA "\033[36m" +#define ESC_END "\033[0m" + +struct NixRepl +{ + string curDir; + EvalState state; + + Strings loadedFiles; + + const static int envSize = 32768; + StaticEnv staticEnv; + Env * env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings & searchPath, nix::ref<Store> store); + ~NixRepl(); + void mainLoop(const Strings & files); + StringSet completePrefix(string prefix); + bool getLine(string & input, const std::string &prompt); + Path getDerivationPath(Value & v); + bool processLine(string line); + void loadFile(const Path & path); + void initEnv(); + void reloadFiles(); + void addAttrsToScope(Value & attrs); + void addVarToScope(const Symbol & name, Value & v); + Expr * parseString(string s); + void evalString(string s, Value & v); + + typedef set<Value *> ValuesSeen; + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); +}; + + +void printHelp() +{ + std::cout + << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" + << "\n" + << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" + << "\n" + << "Options:\n" + << " --help\n" + << " Prints out a summary of the command syntax and exits.\n" + << "\n" + << " --version\n" + << " Prints out the Nix version number on standard output and exits.\n" + << "\n" + << " -I path\n" + << " Add a path to the Nix expression search path. This option may be given\n" + << " multiple times. See the NIX_PATH environment variable for information on\n" + << " the semantics of the Nix search path. Paths added through -I take\n" + << " precedence over NIX_PATH.\n" + << "\n" + << " paths...\n" + << " A list of paths to files containing Nix expressions which nix-repl will\n" + << " load and add to its scope.\n" + << "\n" + << " A path surrounded in < and > will be looked up in the Nix expression search\n" + << " path, as in the Nix language itself.\n" + << "\n" + << " If an element of paths starts with http:// or https://, it is interpreted\n" + << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" + << " location. The tarball must include a single top-level directory containing\n" + << " at least a file named default.nix.\n"; +} + + +string removeWhitespace(string s) +{ + s = chomp(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != string::npos) s = string(s, n); + return s; +} + + +NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store) + : state(searchPath, store) + , staticEnv(false, &state.staticBaseEnv) + , historyFile(getDataDir() + "/nix/repl-history") +{ + curDir = absPath("."); +} + + +NixRepl::~NixRepl() +{ + linenoiseHistorySave(historyFile.c_str()); +} + + +static NixRepl * curRepl; // ugly + +static void completionCallback(const char * s, linenoiseCompletions *lc) +{ + /* Otherwise, return all symbols that start with the prefix. */ + for (auto & c : curRepl->completePrefix(s)) + linenoiseAddCompletion(lc, c.c_str()); +} + + +void NixRepl::mainLoop(const Strings & files) +{ + string error = ANSI_RED "error:" ANSI_NORMAL " "; + std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; + + for (auto & i : files) + loadedFiles.push_back(i); + + reloadFiles(); + if (!loadedFiles.empty()) std::cout << std::endl; + + createDirs(dirOf(historyFile)); + linenoiseHistorySetMaxLen(1000); + linenoiseHistoryLoad(historyFile.c_str()); + + curRepl = this; + linenoiseSetCompletionCallback(completionCallback); + + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just align to the same + // number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) + break; + + try { + if (!removeWhitespace(input).empty() && !processLine(input)) return; + } catch (ParseError & e) { + if (e.msg().find("unexpected $end") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the next line of + // input without clearing the input so far. + continue; + } else { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } + } catch (Error & e) { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } catch (Interrupted & e) { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } + + // We handled the current input fully, so we should clear it + // and read brand new input. + linenoiseHistoryAdd(input.c_str()); + input.clear(); + std::cout << std::endl; + } +} + + +bool NixRepl::getLine(string & input, const std::string &prompt) +{ + char * s = linenoise(prompt.c_str()); + Finally doFree([&]() { linenoiseFree(s); }); + if (!s) return false; + input += s; + return true; +} + + +StringSet NixRepl::completePrefix(string prefix) +{ + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev, cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } + + size_t slash, dot; + + if ((slash = cur.rfind('/')) != string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { + if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) + completions.insert(prev + dir + "/" + entry.name); + } + } catch (Error &) { + } + } else if ((dot = cur.rfind('.')) == string::npos) { + /* This is a variable name; look it up in the current scope. */ + StringSet::iterator i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (string(*i, 0, cur.size()) != cur) break; + completions.insert(prev + *i); + i++; + } + } else { + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + string expr(cur, 0, dot); + string cur2 = string(cur, dot + 1); + + Expr * e = parseString(expr); + Value v; + e->eval(state, *env, v); + state.forceAttrs(v); + + for (auto & i : *v.attrs) { + string name = i.name; + if (string(name, 0, cur2.size()) != cur2) continue; + completions.insert(prev + expr + "." + name); + } + + } catch (ParseError & e) { + // Quietly ignore parse errors. + } catch (EvalError & e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError & e) { + // Quietly ignore undefined variable errors. + } + } + + return completions; +} + + +static int runProgram(const string & program, const Strings & args) +{ + Strings args2(args); + args2.push_front(program); + + Pid pid; + pid = fork(); + if (pid == -1) throw SysError("forking"); + if (pid == 0) { + restoreAffinity(); + execvp(program.c_str(), stringsToCharPtrs(args2).data()); + _exit(1); + } + + return pid.wait(); +} + + +bool isVarName(const string & s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + + +Path NixRepl::getDerivationPath(Value & v) { + DrvInfo drvInfo(state); + if (!getDerivation(state, v, drvInfo, false)) + throw Error("expression does not evaluate to a derivation, so I can't build it"); + Path drvPath = drvInfo.queryDrvPath(); + if (drvPath == "" || !state.store->isValidPath(drvPath)) + throw Error("expression did not evaluate to a valid derivation"); + return drvPath; +} + + +bool NixRepl::processLine(string line) +{ + if (line == "") return true; + + string command, arg; + + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = string(line, 0, p); + if (p != string::npos) arg = removeWhitespace(string(line, p)); + } else { + arg = line; + } + + if (command == ":?" || command == ":help") { + std::cout + << "The following commands are available:\n" + << "\n" + << " <expr> Evaluate and print expression\n" + << " <x> = <expr> Bind expression to variable\n" + << " :a <expr> Add attributes from resulting set to scope\n" + << " :b <expr> Build derivation\n" + << " :i <expr> Build derivation, then install result into current profile\n" + << " :l <path> Load Nix expression and add it to scope\n" + << " :p <expr> Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :s <expr> Build dependencies of derivation, then start nix-shell\n" + << " :t <expr> Describe result of evaluation\n" + << " :u <expr> Build derivation, then start nix-shell\n"; + } + + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } + + else if (command == ":l" || command == ":load") { + state.resetFileCache(); + loadFile(arg); + } + + else if (command == ":r" || command == ":reload") { + state.resetFileCache(); + reloadFiles(); + } + + else if (command == ":t") { + Value v; + evalString(arg, v); + std::cout << showType(v) << std::endl; + + } else if (command == ":u") { + Value v, f, result; + evalString(arg, v); + evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); + state.callFunction(f, v, result, Pos()); + + Path drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + + else if (command == ":b" || command == ":i" || command == ":s") { + Value v; + evalString(arg, v); + Path drvPath = getDerivationPath(v); + + if (command == ":b") { + /* We could do the build in this process using buildPaths(), + but doing it in a child makes it easier to recover from + problems / SIGINT. */ + if (runProgram(settings.nixBinDir + "/nix-store", Strings{"-r", drvPath}) == 0) { + Derivation drv = readDerivation(drvPath); + std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; + for (auto & i : drv.outputs) + std::cout << format(" %1% -> %2%") % i.first % i.second.path << std::endl; + } + } else if (command == ":i") { + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + } else { + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + } + + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } + + else if (command == ":q" || command == ":quit") + return false; + + else if (command != "") + throw Error(format("unknown command ‘%1%’") % command); + + else { + size_t p = line.find('='); + string name; + if (p != string::npos && + p < line.size() && + line[p + 1] != '=' && + isVarName(name = removeWhitespace(string(line, 0, p)))) + { + Expr * e = parseString(string(line, p + 1)); + Value & v(*state.allocValue()); + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = e; + addVarToScope(state.symbols.create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; + } + } + + return true; +} + + +void NixRepl::loadFile(const Path & path) +{ + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v, v2; + state.evalFile(lookupFileArg(state, path), v); + Bindings & bindings(*state.allocBindings(0)); + state.autoCallFunction(bindings, v, v2); + addAttrsToScope(v2); +} + + +void NixRepl::initEnv() +{ + env = &state.allocEnv(envSize); + env->up = &state.baseEnv; + displ = 0; + staticEnv.vars.clear(); + + varNames.clear(); + for (auto & i : state.staticBaseEnv.vars) + varNames.insert(i.first); +} + + +void NixRepl::reloadFiles() +{ + initEnv(); + + Strings old = loadedFiles; + loadedFiles.clear(); + + bool first = true; + for (auto & i : old) { + if (!first) std::cout << std::endl; + first = false; + std::cout << format("Loading ‘%1%’...") % i << std::endl; + loadFile(i); + } +} + + +void NixRepl::addAttrsToScope(Value & attrs) +{ + state.forceAttrs(attrs); + for (auto & i : *attrs.attrs) + addVarToScope(i.name, *i.value); + std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl; +} + + +void NixRepl::addVarToScope(const Symbol & name, Value & v) +{ + if (displ >= envSize) + throw Error("environment full; cannot add more variables"); + staticEnv.vars[name] = displ; + env->values[displ++] = &v; + varNames.insert((string) name); +} + + +Expr * NixRepl::parseString(string s) +{ + Expr * e = state.parseExprFromString(s, curDir, staticEnv); + return e; +} + + +void NixRepl::evalString(string s, Value & v) +{ + Expr * e = parseString(s); + e->eval(state, *env, v); + state.forceValue(v); +} + + +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) +{ + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); +} + + +std::ostream & printStringValue(std::ostream & str, const char * string) { + str << "\""; + for (const char * i = string; *i; i++) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; + return str; +} + + +// FIXME: lot of cut&paste from Nix's eval.cc. +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) +{ + str.flush(); + checkInterrupt(); + + state.forceValue(v); + + switch (v.type) { + + case tInt: + str << ESC_CYA << v.integer << ESC_END; + break; + + case tBool: + str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + break; + + case tString: + str << ESC_YEL; + printStringValue(str, v.string.s); + str << ESC_END; + break; + + case tPath: + str << ESC_GRE << v.path << ESC_END; // !!! escaping? + break; + + case tNull: + str << ESC_CYA "null" ESC_END; + break; + + case tAttrs: { + seen.insert(&v); + + bool isDrv = state.isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state.sDrvPath); + PathSet context; + Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->pos, *i->value, context) : "???"; + str << drvPath << "»"; + } + + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map<string, Value *> Sorted; + Sorted sorted; + for (auto & i : *v.attrs) + sorted[i.name] = i.value; + + /* If this is a derivation, then don't show the + self-references ("all", "out", etc.). */ + StringSet hidden; + if (isDrv) { + hidden.insert("all"); + Bindings::iterator i = v.attrs->find(state.sOutputs); + if (i == v.attrs->end()) + hidden.insert("out"); + else { + state.forceList(*i->value); + for (unsigned int j = 0; j < i->value->listSize(); ++j) + hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j])); + } + } + + for (auto & i : sorted) { + if (isVarName(i.first)) + str << i.first; + else + printStringValue(str, i.first.c_str()); + str << " = "; + if (hidden.find(i.first) != hidden.end()) + str << "«...»"; + else if (seen.find(i.second) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << "; "; + } + + str << "}"; + } else + str << "{ ... }"; + + break; + } + + case tList1: + case tList2: + case tListN: + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (seen.find(v.listElems()[n]) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *v.listElems()[n], maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << " "; + } + else + str << "... "; + str << "]"; + break; + + case tLambda: { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + break; + } + + case tPrimOp: + str << ESC_MAG "«primop»" ESC_END; + break; + + case tPrimOpApp: + str << ESC_BLU "«primop-app»" ESC_END; + break; + + default: + str << ESC_RED "«unknown»" ESC_END; + break; + } + + return str; +} + +struct CmdRepl : StoreCommand +{ + Strings files; + + CmdRepl() + { + expectArgs("files", &files); + } + + std::string name() override { return "repl"; } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + void run(ref<Store> store) override + { + // FIXME: pass searchPath + NixRepl repl({}, openStore()); + repl.mainLoop(files); + } +}; + +static RegisterCommand r1(make_ref<CmdRepl>()); + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index a30031ad07b4..bcfa74eb5f5f 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" #include "derivations.hh" @@ -13,7 +12,7 @@ using namespace nix; -struct CmdRun : StoreCommand, MixInstallables +struct CmdRun : InstallablesCommand { CmdRun() { @@ -31,20 +30,7 @@ struct CmdRun : StoreCommand, MixInstallables void run(ref<Store> store) override { - auto elems = evalInstallables(store); - - PathSet pathsToBuild; - - for (auto & elem : elems) { - if (elem.isDrv) - pathsToBuild.insert(elem.drvPath); - else - pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - - printMissing(store, pathsToBuild); - - store->buildPaths(pathsToBuild); + auto outPaths = buildInstallables(store, false); auto store2 = store.dynamic_pointer_cast<LocalStore>(); @@ -103,14 +89,6 @@ struct CmdRun : StoreCommand, MixInstallables #endif } - PathSet outPaths; - for (auto & path : pathsToBuild) - if (isDerivation(path)) { - Derivation drv = store->derivationFromPath(path); - for (auto & output : drv.outputs) - outPaths.insert(output.second.path); - } else - outPaths.insert(path); auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":"); for (auto & path : outPaths) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc new file mode 100644 index 000000000000..c628c2898d73 --- /dev/null +++ b/src/nix/show-config.cc @@ -0,0 +1,38 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "json.hh" + +using namespace nix; + +struct CmdShowConfig : Command, MixJSON +{ + CmdShowConfig() + { + } + + std::string name() override + { + return "show-config"; + } + + std::string description() override + { + return "show the Nix configuration"; + } + + void run() override + { + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + JSONObject jsonObj(std::cout, true); + settings.toJSON(jsonObj); + } else { + for (auto & s : settings.getSettings()) + std::cout << s.first + " = " + s.second + "\n"; + } + } +}; + +static RegisterCommand r1(make_ref<CmdShowConfig>()); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index d8d8c0f53df0..3dd03771619f 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -42,10 +42,10 @@ struct CmdCopySigs : StorePathsCommand std::string doneLabel = "done"; std::atomic<size_t> added{0}; - logger->setExpected(doneLabel, storePaths.size()); + //logger->setExpected(doneLabel, storePaths.size()); auto doPath = [&](const Path & storePath) { - Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); + //Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); checkInterrupt(); @@ -76,7 +76,7 @@ struct CmdCopySigs : StorePathsCommand added += newSigs.size(); } - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); }; for (auto & storePath : storePaths) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2f8d02fa060e..8facb4bef8a2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -65,7 +65,7 @@ struct CmdVerify : StorePathsCommand std::string untrustedLabel("untrusted"); std::string corruptedLabel("corrupted"); std::string failedLabel("failed"); - logger->setExpected(doneLabel, storePaths.size()); + //logger->setExpected(doneLabel, storePaths.size()); ThreadPool pool; @@ -73,7 +73,7 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); + //Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); auto info = store->queryPathInfo(storePath); @@ -85,7 +85,7 @@ struct CmdVerify : StorePathsCommand auto hash = sink.finish(); if (hash.first != info->narHash) { - logger->incProgress(corruptedLabel); + //logger->incProgress(corruptedLabel); corrupted = 1; printError( format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") @@ -137,19 +137,19 @@ struct CmdVerify : StorePathsCommand } if (!good) { - logger->incProgress(untrustedLabel); + //logger->incProgress(untrustedLabel); untrusted++; printError(format("path ‘%s’ is untrusted") % info->path); } } - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); done++; } catch (Error & e) { printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); - logger->incProgress(failedLabel); + //logger->incProgress(failedLabel); failed++; } }; diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 666cc6ef8041..8bff0fe79032 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -5,6 +5,7 @@ let input1 = mkDerivation { name = "build-hook-input-1"; builder = ./dependencies.builder1.sh; + requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { diff --git a/tests/build-hook.sh b/tests/build-hook.sh index ef77a3ae5285..2005c7cebdc4 100644 --- a/tests/build-hook.sh +++ b/tests/build-hook.sh @@ -1,8 +1,8 @@ source common.sh -export NIX_BUILD_HOOK="$(pwd)/build-hook.hook.sh" +clearStore -outPath=$(nix-build build-hook.nix --no-out-link) +outPath=$(nix-build build-hook.nix --no-out-link --option build-hook $(pwd)/build-hook.hook.sh) echo "output path is $outPath" diff --git a/tests/build-remote.sh b/tests/build-remote.sh new file mode 100644 index 000000000000..927a217f3761 --- /dev/null +++ b/tests/build-remote.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore + +if [[ $(uname) != Linux ]]; then exit; fi +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +chmod -R u+w $TEST_ROOT/store1 || true +rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 + +# FIXME: --option is not passed to build-remote, so have to create a config file. +export NIX_CONF_DIR=$TEST_ROOT/etc2 +mkdir -p $NIX_CONF_DIR +echo " +build-sandbox-paths = /nix/store +sandbox-build-dir = /build-tmp +" > $NIX_CONF_DIR/nix.conf + +outPath=$(nix-build build-hook.nix --no-out-link -j0 \ + --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo") + +cat $outPath/foobar | grep FOOBAR + +# Ensure that input1 was built on store1 due to the required feature. +p=$(readlink -f $outPath/input-2) +(! nix path-info --store local?root=$TEST_ROOT/store0 --all | grep dependencies.builder1.sh) +nix path-info --store local?root=$TEST_ROOT/store1 --all | grep dependencies.builder1.sh diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/lang/eval-okay-ind-string.exp index 886219dcf652..9cf4bd2ee78a 100644 --- a/tests/lang/eval-okay-ind-string.exp +++ b/tests/lang/eval-okay-ind-string.exp @@ -1 +1 @@ -"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\n" +"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n" diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/lang/eval-okay-ind-string.nix index 1556aae9f54f..1669dc0648ea 100644 --- a/tests/lang/eval-okay-ind-string.nix +++ b/tests/lang/eval-okay-ind-string.nix @@ -117,4 +117,12 @@ let bar ''; -in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + # Regression test: accept $'. + s16 = '' + cut -d $'\t' -f 1 + ''; + + # Accept dollars at end of strings + s17 = ''ending dollar $'' + ''$'' + "\n"; + +in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/lang/eval-okay-regex-match.nix index ae6501532d11..273e2590713e 100644 --- a/tests/lang/eval-okay-regex-match.nix +++ b/tests/lang/eval-okay-regex-match.nix @@ -17,8 +17,11 @@ assert matches "fo+" "foo"; assert matches "fo{1,2}" "foo"; assert !matches "fo{1,2}" "fooo"; assert !matches "fo*" "foobar"; +assert matches "[[:space:]]+([^[:space:]]+)[[:space:]]+" " foo "; +assert !matches "[[:space:]]+([[:upper:]]+)[[:space:]]+" " foo "; assert match "(.*)\\.nix" "foobar.nix" == [ "foobar" ]; +assert match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " == [ "FOO" ]; assert splitFN "/path/to/foobar.nix" == [ "/path/to/" "/path/to" "foobar" "nix" ]; assert splitFN "foobar.cc" == [ null null "foobar" "cc" ]; diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh new file mode 100644 index 000000000000..54cdef8178ea --- /dev/null +++ b/tests/linux-sandbox.sh @@ -0,0 +1,27 @@ +source common.sh + +clearStore + +if [[ $(uname) != Linux ]]; then exit; fi + +# Note: we need to bind-mount $SHELL into the chroot. Currently we +# only support the case where $SHELL is in the Nix store, because +# otherwise things get complicated (e.g. if it's in /bin, do we need +# /lib as well?). +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +export NIX_STORE_DIR=/my/store +export NIX_REMOTE="local?root=$TEST_ROOT/store0" + +outPath=$(nix-build dependencies.nix --no-out-link --option build-sandbox-paths /nix/store) + +[[ $outPath =~ /my/store/.*-dependencies ]] + +nix path-info -r $outPath | grep input-2 + +nix ls-store -R -l $outPath | grep foobar + +nix cat-store $outPath/foobar | grep FOOBAR diff --git a/tests/local.mk b/tests/local.mk index b3ce39cda806..7d99a0fc7675 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -11,7 +11,10 @@ nix_tests = \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ - placeholders.sh nix-shell.sh + placeholders.sh nix-shell.sh \ + linux-sandbox.sh \ + build-remote.sh \ + nar-index.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/nar-index.nix b/tests/nar-index.nix new file mode 100644 index 000000000000..0e2a7f721135 --- /dev/null +++ b/tests/nar-index.nix @@ -0,0 +1,23 @@ +with import ./config.nix; + +rec { + a = mkDerivation { + name = "nar-index-a"; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + mkdir $out/foo + touch $out/foo-x + touch $out/foo/bar + touch $out/foo/baz + touch $out/qux + mkdir $out/zyx + + cat >$out/foo/data <<EOF + lasjdöaxnasd +asdom 12398 +ä"§Æẞ¢«»”alsd +EOF + ''; + }; +} \ No newline at end of file diff --git a/tests/nar-index.sh b/tests/nar-index.sh new file mode 100644 index 000000000000..51369346c88a --- /dev/null +++ b/tests/nar-index.sh @@ -0,0 +1,23 @@ +source common.sh + +echo "building test path" +storePath="$(nix-build nar-index.nix -A a --no-out-link)" + +cd "$TEST_ROOT" + +echo "dumping path to nar" +narFile="$TEST_ROOT/path.nar" +nix-store --dump $storePath > $narFile + +echo "check that find and ls-nar match" +( cd $storePath; find . | sort ) > files.find +nix ls-nar -R -d $narFile "" | sort > files.ls-nar +diff -u files.find files.ls-nar + +echo "check that file contents of data match" +nix cat-nar $narFile /foo/data > data.cat-nar +diff -u data.cat-nar $storePath/foo/data + +echo "check that file contents of baz match" +nix cat-nar $narFile /foo/baz > baz.cat-nar +diff -u baz.cat-nar $storePath/foo/baz \ No newline at end of file diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 63aaa4d88f56..39bd090e43e7 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -43,7 +43,6 @@ in { config, pkgs, ... }: { nix.maxJobs = 0; # force remote building nix.distributedBuilds = true; - nix.envVars = pkgs.lib.mkAfter { NIX_BUILD_HOOK = "${nix}/libexec/nix/build-remote"; }; nix.buildMachines = [ { hostName = "slave1"; sshUser = "root"; diff --git a/tests/shell.nix b/tests/shell.nix index 1a092913b3b2..5845d36fc161 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -2,7 +2,7 @@ with import ./config.nix; -rec { +let pkgs = rec { setupSh = builtins.toFile "setup" '' export VAR_FROM_STDENV_SETUP=foo for pkg in $buildInputs; do @@ -44,4 +44,6 @@ rec { ''; bash = shell; -} + + inherit pkgs; +}; in pkgs diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh index 3dadd591572d..c9a83aaf83dc 100755 --- a/tests/shell.shebang.sh +++ b/tests/shell.shebang.sh @@ -1,4 +1,4 @@ #! @ENV_PROG@ nix-shell -#! nix-shell -I nixpkgs=shell.nix --option use-binary-caches false +#! nix-shell -I nixpkgs=shell.nix --option build-use-substitutes false #! nix-shell --pure -i bash -p foo bar echo "$(foo) $(bar) $@" diff --git a/tests/timeout.nix b/tests/timeout.nix index 540fba934ff6..e18d717eff1f 100644 --- a/tests/timeout.nix +++ b/tests/timeout.nix @@ -5,6 +5,7 @@ with import ./config.nix; infiniteLoop = mkDerivation { name = "timeout"; buildCommand = '' + touch $out echo "‘timeout’ builder entering an infinite loop" while true ; do echo -n .; done ''; @@ -13,6 +14,7 @@ with import ./config.nix; silent = mkDerivation { name = "silent"; buildCommand = '' + touch $out sleep 60 ''; }; @@ -20,6 +22,7 @@ with import ./config.nix; closeLog = mkDerivation { name = "silent"; buildCommand = '' + touch $out exec > /dev/null 2>&1 sleep 1000000000 ''; diff --git a/tests/timeout.sh b/tests/timeout.sh index ce1ae7d674a1..77b227e89ba5 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -29,3 +29,8 @@ if nix-build timeout.nix -A closeLog; then echo "build should have failed" exit 1 fi + +if nix build -f timeout.nix silent --option build-max-silent-time 2; then + echo "build should have failed" + exit 1 +fi |