diff options
106 files changed, 1934 insertions, 900 deletions
diff --git a/.gitignore b/.gitignore index e420fbf0144f..94067256b8bc 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,8 @@ Makefile.config *.a *.o *.so +*.dll +*.exe *.dep *~ *.pc diff --git a/Makefile b/Makefile index b41a820275f4..08e4012f99b2 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ makefiles = \ scripts/local.mk \ corepkgs/local.mk \ misc/systemd/local.mk \ + misc/launchd/local.mk \ misc/upstart/local.mk \ misc/emacs/local.mk \ doc/manual/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index 6e12afbce39b..797b99f80504 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -14,8 +14,6 @@ curl = @curl@ datadir = @datadir@ datarootdir = @datarootdir@ dblatex = @dblatex@ -docbookrng = @docbookrng@ -docbookxsl = @docbookxsl@ docdir = @docdir@ exec_prefix = @exec_prefix@ includedir = @includedir@ diff --git a/configure.ac b/configure.ac index 2806b37b0f81..622cf1e2045d 100644 --- a/configure.ac +++ b/configure.ac @@ -176,17 +176,6 @@ AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], AC_SUBST(coreutils) -AC_ARG_WITH(docbook-rng, AC_HELP_STRING([--with-docbook-rng=PATH], - [path of the DocBook RelaxNG schema]), - docbookrng=$withval, docbookrng=/docbook-rng-missing) -AC_SUBST(docbookrng) - -AC_ARG_WITH(docbook-xsl, AC_HELP_STRING([--with-docbook-xsl=PATH], - [path of the DocBook XSL stylesheets]), - docbookxsl=$withval, docbookxsl=/docbook-xsl-missing) -AC_SUBST(docbookxsl) - - AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], [path of the Nix store (defaults to /nix/store)]), storedir=$withval, storedir='/nix/store') diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml new file mode 100644 index 000000000000..338aa6f3a238 --- /dev/null +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -0,0 +1,10 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0"> + +<title>Advanced Topics</title> + +<xi:include href="distributed-builds.xml" /> + +</part> diff --git a/doc/manual/builds/enabling-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index 4b45812ee918..70f396f81cdb 100644 --- a/doc/manual/builds/enabling-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -2,9 +2,18 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" version="5.0" - xml:id="ch-enabling-builds"> + xml:id='chap-distributed-builds'> -<title>Enabling Distributed Builds</title> +<title>Distributed Builds</title> + +<para>Nix supports distributed builds, where a local Nix installation can +forward Nix builds to other machines over the network. This allows +multiple builds to be performed in parallel (thus improving +performance) and allows Nix to perform multi-platform builds in a +semi-transparent way. For instance, if you perform a build for a +<literal>powerpc-darwin</literal> on an <literal>i686-linux</literal> +machine, Nix can automatically forward the build to a +<literal>powerpc-darwin</literal> machine, if available.</para> <para>You can enable distributed builds by setting the environment variable <envar>NIX_BUILD_HOOK</envar> to point to a program that Nix @@ -41,7 +50,7 @@ example configuration is shown in <xref linkend='ex-remote-systems' bits of information: <orderedlist> - + <listitem><para>The name of the remote machine, with optionally the user under which the remote build should be performed. This is actually passed as an argument to <command>ssh</command>, so it can @@ -73,9 +82,9 @@ bits of information: <filename>build-remote.pl</filename> will only perform the derivation on a machine that has the specified features. For instance, the attribute - + <programlisting> -requiredSystemFeatures = [ "kvm" ]; +requiredSystemFeatures = [ "kvm" ]; </programlisting> will cause the build to be performed on a machine that has the @@ -103,4 +112,4 @@ running, they should use the same <envar>NIX_CURRENT_LOAD</envar> file. Maybe in the future <filename>build-remote.pl</filename> will look at the actual remote load.</para> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/builds/build-farm.xml b/doc/manual/builds/build-farm.xml deleted file mode 100644 index e0e9f10f1173..000000000000 --- a/doc/manual/builds/build-farm.xml +++ /dev/null @@ -1,22 +0,0 @@ -<part xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id='chap-distributed-builds'> - -<title>Distributed Builds</title> - -<partintro> -<para>Nix supports distributed builds, where a local Nix installation can -forward Nix builds to other machines over the network. This allows -multiple builds to be performed in parallel (thus improving -performance) and allows Nix to perform multi-platform builds in a -semi-transparent way. For instance, if you perform a build for a -<literal>powerpc-darwin</literal> on an <literal>i686-linux</literal> -machine, Nix can automatically forward the build to a -<literal>powerpc-darwin</literal> machine, if available.</para> -</partintro> - -<xi:include href="enabling-builds.xml" /> - -</part> diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c3ef184e7368..053f4d43cb0c 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -66,7 +66,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> <para>Keeping derivation around is useful for querying and traceability (e.g., it allows you to ask with what dependencies or options a store path was built), so by default this option is on. - Turn it off to safe a bit of disk space (or a lot if + Turn it off to save a bit of disk space (or a lot if <literal>gc-keep-outputs</literal> is also turned on).</para></listitem> </varlistentry> @@ -297,7 +297,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>build-cache-failures</literal></term> + <varlistentry><term><literal>build-cache-failure</literal></term> <listitem><para>If set to <literal>true</literal>, Nix will “cache” build failures, meaning that it will remember (in its @@ -348,7 +348,7 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> <listitem><para>A list of URLs of binary caches, separated by whitespace. The default is - <literal>http://cache.nixos.org</literal>.</para></listitem> + <literal>https://cache.nixos.org</literal>.</para></listitem> </varlistentry> @@ -402,6 +402,15 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> + <varlistentry><term><literal>verify-https-binary-caches</literal></term> + + <listitem><para>Whether HTTPS binary caches are required to have a + certificate that can be verified. Defaults to + <literal>true</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><literal>force-manifest</literal></term> <listitem><para>If this option is set to <literal>false</literal> diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index c4cc04ce24d8..531b41fc0902 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -21,7 +21,7 @@ <command>nix-channel</command> <group choice='req'> <arg choice='plain'><option>--add</option> <replaceable>url</replaceable> <arg choice='opt'><replaceable>name</replaceable></arg></arg> - <arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg> + <arg choice='plain'><option>--remove</option> <replaceable>name</replaceable></arg> <arg choice='plain'><option>--list</option></arg> <arg choice='plain'><option>--update</option> <arg rep='repeat'><replaceable>names</replaceable></arg></arg> <arg choice='plain'><option>--rollback</option> <arg choice='opt'><replaceable>generation</replaceable></arg></arg> @@ -33,8 +33,8 @@ <para>A Nix channel is mechanism that allows you to automatically stay up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing a set of Nix expressions -and a <command>nix-push</command> manifest. <phrase +just a URL that points to a place containing both a set of Nix +expressions and a pointer to a binary cache. <phrase condition="manual">See also <xref linkend="sec-channels" />.</phrase></para> @@ -99,13 +99,6 @@ an update.</para> <para>The list of subscribed channels is stored in <filename>~/.nix-channels</filename>.</para> -<para>A channel consists of two elements: a bzipped Tar archive -containing the Nix expressions, and a manifest created by -<command>nix-push</command>. These must be stored under -<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal> and -<literal><replaceable>url</replaceable>/MANIFEST</literal>, -respectively.</para> - </refsection> <refsection><title>Examples</title> @@ -163,4 +156,49 @@ $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' </refsection> +<refsection><title>Channel format</title> + +<para>A channel URL should point to a directory containing the +following files:</para> + +<variablelist> + + <varlistentry><term><filename>nixexprs.tar.xz</filename></term> + + <listitem><para>A tarball containing Nix expressions and files + referenced by them (such as build scripts and patches). At + top-level, the tarball should contain a single directory. That + directory must contain a file <filename>default.nix</filename> + that serves as the channel’s “entry point”.</para></listitem> + + </varlistentry> + + <varlistentry><term><filename>binary-cache-url</filename></term> + + <listitem><para>A file containing the URL to a binary cache (such + as <uri>https://cache.nixos.org</uri>. Nix will automatically + check this cache for pre-built binaries, if the user has + sufficient rights to add binary caches. For instance, in a + multi-user Nix setup, the binary caches provided by the channels + of the root user are used automatically, but caches corresponding + to the channels of non-root users are ignored. Binary caches can + be created and maintained using + <command>nix-push</command>.</para></listitem> + + </varlistentry> + + <varlistentry><term><filename>MANIFEST.bz2</filename></term> + + <listitem><para>(Deprecated in favour of binary caches.) A + manifest as created by <command>nix-push</command>. Only used if + <filename>binary-cache-url</filename> is not present or if the + <filename>nix.conf</filename> option + <option>force-manifest</option> is set.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + </refentry> diff --git a/doc/manual/command-ref/nix-install-package.xml b/doc/manual/command-ref/nix-install-package.xml index 7d5cd996e44a..f7802a95d55e 100644 --- a/doc/manual/command-ref/nix-install-package.xml +++ b/doc/manual/command-ref/nix-install-package.xml @@ -3,7 +3,7 @@ xmlns:xi="http://www.w3.org/2001/XInclude" version="5.0" xml:id="sec-nix-install-package"> - + <refmeta> <refentrytitle>nix-install-package</refentrytitle> <manvolnum>1</manvolnum> @@ -47,8 +47,7 @@ <para>The command <command>nix-install-package</command> interactively installs a Nix Package file (<filename>*.nixpkg</filename>), which is a small file that contains a store path to be installed along with the -URL of a <link linkend="sec-nix-push"><command>nix-push</command> -manifest</link>. The Nix Package file is either +URL of a binary cache. The Nix Package file is either <replaceable>file</replaceable>, or automatically downloaded from <replaceable>url</replaceable> if the <option>--url</option> switch is used.</para> @@ -76,7 +75,7 @@ to restart itself with <command>xterm</command>, <refsection><title>Options</title> <variablelist> - + <varlistentry><term><option>--non-interactive</option></term> <listitem><para>Do not open a new terminal window and do not ask @@ -139,14 +138,14 @@ The elements are as follows: <variablelist> <varlistentry><term><literal>NIXPKG1</literal></term> - + <listitem><para>The version of the Nix Package file.</para></listitem> </varlistentry> <varlistentry><term><replaceable>manifestURL</replaceable></term> - + <listitem><para>The manifest to be pulled by <command>nix-pull</command>. The manifest must contain <replaceable>outPath</replaceable>.</para></listitem> @@ -154,21 +153,21 @@ The elements are as follows: </varlistentry> <varlistentry><term><replaceable>name</replaceable></term> - + <listitem><para>The symbolic name and version of the package.</para></listitem> </varlistentry> <varlistentry><term><replaceable>system</replaceable></term> - + <listitem><para>The platform identifier of the platform for which this binary package is intended.</para></listitem> </varlistentry> <varlistentry><term><replaceable>drvPath</replaceable></term> - + <listitem><para>The path in the Nix store of the derivation from which <replaceable>outPath</replaceable> was built. Not currently used.</para></listitem> @@ -176,17 +175,21 @@ The elements are as follows: </varlistentry> <varlistentry><term><replaceable>outPath</replaceable></term> - - <listitem><para>The path in the Nix store of the package. After - <command>nix-install-package</command> has obtained the manifest - from <replaceable>manifestURL</replaceable>, it performs a - <literal>nix-env -i</literal> <replaceable>outPath</replaceable> - to install the binary package.</para></listitem> + + <listitem><para>The path in the Nix store of the + package.</para></listitem> + + </varlistentry> + + <varlistentry><term><replaceable>binaryCacheURL</replaceable></term> + + <listitem><para>The URL of a binary cache containing the closure + of <replaceable>outPath</replaceable>.</para></listitem> </varlistentry> </variablelist> - + </para> <para>An example follows: diff --git a/doc/manual/command-ref/nix-instantiate.xml b/doc/manual/command-ref/nix-instantiate.xml index d4966579ae59..1e556c7ed7c4 100644 --- a/doc/manual/command-ref/nix-instantiate.xml +++ b/doc/manual/command-ref/nix-instantiate.xml @@ -45,7 +45,7 @@ <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg> <sbr/> <command>nix-instantiate</command> - <arg choice='plain'><option>--file-file</option></arg> + <arg choice='plain'><option>--find-file</option></arg> <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg> </cmdsynopsis> </refsynopsisdiv> diff --git a/doc/manual/command-ref/nix-pull.xml b/doc/manual/command-ref/nix-pull.xml index 598c651b5091..eb471677b63f 100644 --- a/doc/manual/command-ref/nix-pull.xml +++ b/doc/manual/command-ref/nix-pull.xml @@ -13,7 +13,7 @@ <refnamediv> <refname>nix-pull</refname> - <refpurpose>pull substitutes from a network cache</refpurpose> + <refpurpose>register availability of pre-built binaries (deprecated)</refpurpose> </refnamediv> <refsynopsisdiv> @@ -26,6 +26,9 @@ <refsection><title>Description</title> +<note><para>This command and the use of manifests is deprecated. It is +better to use binary caches.</para></note> + <para>The command <command>nix-pull</command> obtains a list of pre-built store paths from the URL <replaceable>url</replaceable>, and for each of these store paths, registers a substitute derivation that @@ -43,7 +46,7 @@ with the files created by <replaceable>nix-push</replaceable>.</para> <refsection><title>Examples</title> <screen> -$ nix-pull http://nix.cs.uu.nl/dist/nix/nixpkgs-0.5pre753/MANIFEST</screen> +$ nix-pull https://nixos.org/releases/nixpkgs/nixpkgs-15.05pre54468.69858d7/MANIFEST</screen> </refsection> diff --git a/doc/manual/command-ref/nix-push.xml b/doc/manual/command-ref/nix-push.xml index c19f44a615a8..a3a3c9623e3c 100644 --- a/doc/manual/command-ref/nix-push.xml +++ b/doc/manual/command-ref/nix-push.xml @@ -260,7 +260,7 @@ The properties that are currently supported are: <listitem><para>Each binary cache has a priority (defaulting to 50). Binary caches are checked for binaries in order of ascending priority; thus a higher number denotes a lower priority. The - binary cache <uri>http://cache.nixos.org</uri> has priority + binary cache <uri>https://cache.nixos.org</uri> has priority 40.</para></listitem> </varlistentry> @@ -278,14 +278,14 @@ URL <replaceable>url</replaceable> has a binary for <replaceable>p</replaceable>, Nix fetches <replaceable>url/h</replaceable>, where <replaceable>h</replaceable> is the hash part of <replaceable>p</replaceable>. Thus, if we have a -cache <uri>http://cache.nixos.org</uri> and we want to obtain -the store path +cache <uri>https://cache.nixos.org</uri> and we want to obtain the +store path <screen> /nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7 </screen> then Nix will attempt to fetch <screen> -http://cache.nixos.org/a8922c0h87iilxzzvwn2hmv8x210aqb9.narinfo +https://cache.nixos.org/a8922c0h87iilxzzvwn2hmv8x210aqb9.narinfo </screen> (Commands such as <command>nix-env -qas</command> will issue an HTTP HEAD request, since it only needs to know if the @@ -389,7 +389,7 @@ The fields are as follows: references exist (e.g., <filename>/nix/store/2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16</filename>), Nix will fetch <screen> -http://cache.nixos.org/nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2 +https://cache.nixos.org/nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2 </screen> and decompress and unpack it to <filename>/nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7</filename>.</para> diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index 1f03117636c9..c1b172b70380 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -29,6 +29,7 @@ <replaceable>attrPath</replaceable> </arg> <arg><option>--command</option> <replaceable>cmd</replaceable></arg> + <arg><option>--run</option> <replaceable>cmd</replaceable></arg> <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg> <arg><option>--pure</option></arg> <group choice='req'> @@ -92,11 +93,24 @@ also <xref linkend="sec-common-options" />.</phrase></para> <varlistentry><term><option>--command</option> <replaceable>cmd</replaceable></term> <listitem><para>In the environment of the derivation, run the - shell command <replaceable>cmd</replaceable> instead of starting - an interactive shell. However, if you end the shell command with - <literal>return</literal>, you still get an interactive shell. - This can be useful for doing any additional - initialisation.</para></listitem> + shell command <replaceable>cmd</replaceable>. This command is + executed in an interactive shell. (Use <option>--run</option> to + use a non-interactive shell instead.) However, a call to + <literal>exit</literal> is implicitly added to the command, so the + shell will exit after running the command. To prevent this, add + <literal>return</literal> at the end; e.g. <literal>--command + "echo Hello; return"</literal> will print <literal>Hello</literal> + and then drop you into the interactive shell. This can be useful + for doing any additional initialisation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--run</option> <replaceable>cmd</replaceable></term> + + <listitem><para>Like <option>--command</option>, but executes the + command in a non-interactive shell. This means (among other + things) that if you hit Ctrl-C while the command is running, the + shell exits.</para></listitem> </varlistentry> diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index e1e6c36e0c1d..a2faeaeba422 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1277,7 +1277,7 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil <refsection><title>Description</title> <para>If build failure caching is enabled through the -<literal>build-cache-failures</literal> configuration option, the +<literal>build-cache-failure</literal> configuration option, the operation <option>--query-failed-paths</option> will print out all store paths that have failed to build.</para> @@ -1314,7 +1314,7 @@ $ nix-store --query-failed-paths <refsection><title>Description</title> <para>If build failure caching is enabled through the -<literal>build-cache-failures</literal> configuration option, the +<literal>build-cache-failure</literal> configuration option, the operation <option>--clear-failed-paths</option> clears the “failed” state of the given store paths, allowing them to be built again. This is useful if the failure was actually transient (e.g. because the disk diff --git a/doc/manual/command-ref/utilities.xml b/doc/manual/command-ref/utilities.xml index d5650fd38f43..be2fe6e2d235 100644 --- a/doc/manual/command-ref/utilities.xml +++ b/doc/manual/command-ref/utilities.xml @@ -13,7 +13,9 @@ work with Nix.</para> <xi:include href="nix-collect-garbage.xml" /> <xi:include href="nix-copy-closure.xml" /> <xi:include href="nix-daemon.xml" /> +<!-- <xi:include href="nix-generate-patches.xml" /> +--> <xi:include href="nix-hash.xml" /> <xi:include href="nix-install-package.xml" /> <xi:include href="nix-instantiate.xml" /> @@ -21,4 +23,4 @@ work with Nix.</para> <xi:include href="nix-pull.xml" /> <xi:include href="nix-push.xml" /> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index f8b84b98cb82..d476d613e462 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -40,7 +40,7 @@ allowedReferences = []; recursively. For example, <programlisting> -allowedReferences = [ foobar ]; +allowedRequisites = [ foobar ]; </programlisting> enforces that the output of a derivation cannot have any other diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 4edb3a1a76fb..3cdf4e28fe6a 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -45,14 +45,17 @@ available as <function>builtins.derivation</function>.</para> <listitem><para>Return the names of the attributes in the set <replaceable>set</replaceable> in a sorted list. For instance, <literal>builtins.attrNames { y = 1; x = "foo"; }</literal> - evaluates to <literal>[ "x" "y" ]</literal>. There is no built-in - function <function>attrValues</function>, but you can easily - define it yourself: + evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem> -<programlisting> -attrValues = set: map (name: builtins.getAttr name set) (builtins.attrNames set);</programlisting> + </varlistentry> - </para></listitem> + + <varlistentry><term><function>builtins.attrValues</function> + <replaceable>set</replaceable></term> + + <listitem><para>Return the values of the attributes in the set + <replaceable>set</replaceable> in the order corresponding to the + sorted attribute names.</para></listitem> </varlistentry> @@ -144,6 +147,19 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> --> + <varlistentry><term><function>builtins.deepSeq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>This is like <literal>seq + <replaceable>e1</replaceable> + <replaceable>e2</replaceable></literal>, except that + <replaceable>e1</replaceable> is evaluated + <emphasis>deeply</emphasis>: if it’s a list or set, its elements + or attributes are also evaluated recursively.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>derivation</function> <replaceable>attrs</replaceable></term> @@ -596,13 +612,26 @@ in config.someSetting</programlisting> </varlistentry> - <!-- - <varlistentry><term><function>relativise</function></term> + <varlistentry><term><function>builtins.readDir</function> + <replaceable>path</replaceable></term> - <listitem><para>TODO</para></listitem> + <listitem><para>Return the contents of the directory + <replaceable>path</replaceable> as a set mapping directory entries + to the corresponding file type. For instance, if directory + <filename>A</filename> contains a regular file + <filename>B</filename> and another directory + <filename>C</filename>, then <literal>builtins.readDir + ./A</literal> will return the set + +<programlisting> +{ A = "regular"; B = "directory"; }</programlisting> + + The possible values for the file type are + <literal>"regular"</literal>, <literal>"directory"</literal>, + <literal>"symlink"</literal> and + <literal>"unknown"</literal>.</para></listitem> </varlistentry> - --> <varlistentry><term><function>builtins.readFile</function> @@ -622,14 +651,25 @@ in config.someSetting</programlisting> <replaceable>set</replaceable>. The attributes don’t have to exist in <replaceable>set</replaceable>. For instance, -<screen> -removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</screen> +<programlisting> +removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> evaluates to <literal>{ y = 2; }</literal>.</para></listitem> </varlistentry> + <varlistentry><term><function>builtins.seq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Evaluate <replaceable>e1</replaceable>, then + evaluate and return <replaceable>e2</replaceable>. This ensures + that a computation is strict in the value of + <replaceable>e1</replaceable>.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.stringLength</function> <replaceable>e</replaceable></term> diff --git a/doc/manual/expressions/custom-builder.xml b/doc/manual/expressions/custom-builder.xml deleted file mode 100644 index c26deac40f4a..000000000000 --- a/doc/manual/expressions/custom-builder.xml +++ /dev/null @@ -1,26 +0,0 @@ -<section xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-custom-builder"> - -<title>Customizing the Generic Builder</title> - -<para>The operation of the generic builder can be modified in many -places by setting certain variables. These <emphasis>hook -variables</emphasis> are typically set to the name of some shell -function defined by you. For instance, to perform some additional -steps after <command>make install</command> you would set the -<varname>postInstall</varname> variable: - -<programlisting> -postInstall=myPostInstall - -myPostInstall() { - mkdir $out/share/extra - cp extrafiles/* $out/share/extra -}</programlisting> - -</para> - -</section> \ No newline at end of file diff --git a/doc/manual/expressions/debug-build.xml b/doc/manual/expressions/debug-build.xml index 508cb2c1930e..0c1f4e6719b2 100644 --- a/doc/manual/expressions/debug-build.xml +++ b/doc/manual/expressions/debug-build.xml @@ -6,13 +6,14 @@ <title>Debugging Build Failures</title> -<para>At the beginning of each phase, the set of all shell variables -is written to the file <filename>env-vars</filename> at the top-level -build directory. This is useful for debugging: it allows you to -recreate the environment in which a build was performed. For -instance, if a build fails, then assuming you used the -<option>-K</option> flag, you can go to the output directory and -<quote>switch</quote> to the environment of the builder: +<para>At the beginning of each phase of the build (such as unpacking, +building or installing), the set of all shell variables is written to +the file <filename>env-vars</filename> at the top-level build +directory. This is useful for debugging: it allows you to recreate +the environment in which a build was performed. For instance, if a +build fails, then assuming you used the <option>-K</option> flag, you +can go to the output directory and <quote>switch</quote> to the +environment of the builder: <screen> $ nix-build -K ./foo.nix @@ -30,4 +31,4 @@ $ make </para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/expressions/derivations.xml b/doc/manual/expressions/derivations.xml index b57c33f4e3a9..90e2786faaab 100644 --- a/doc/manual/expressions/derivations.xml +++ b/doc/manual/expressions/derivations.xml @@ -110,12 +110,12 @@ buildInputs = [ pkg pkg.headers ]; </itemizedlist> -<para>The function <function>mkDerivation</function> in the standard -environment is a wrapper around <function>derivation</function> that -adds a default value for <varname>system</varname> and always uses -Bash as the builder, to which the supplied builder is passed as a -command-line argument. See <xref linkend='sec-standard-environment' -/>.</para> +<para>The function <function>mkDerivation</function> in the Nixpkgs +standard environment is a wrapper around +<function>derivation</function> that adds a default value for +<varname>system</varname> and always uses Bash as the builder, to +which the supplied builder is passed as a command-line argument. See +the Nixpkgs manual for details.</para> <para>The builder is executed as follows: @@ -208,4 +208,4 @@ command-line argument. See <xref linkend='sec-standard-environment' <xi:include href="advanced-attributes.xml" /> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/expressions/generic-builder.xml b/doc/manual/expressions/generic-builder.xml index f8567a042d47..db7ff405d8b1 100644 --- a/doc/manual/expressions/generic-builder.xml +++ b/doc/manual/expressions/generic-builder.xml @@ -71,7 +71,7 @@ genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting> generic builder is smart enough to figure out whether to unpack the sources using <command>gzip</command>, <command>bzip2</command>, etc. It can be customised in many ways; - see <xref linkend='sec-standard-environment' />.</para> + see the Nixpkgs manual for details.</para> </callout> @@ -95,4 +95,4 @@ In fact, <varname>mkDerivation</varname> provides a default builder that looks exactly like that, so it is actually possible to omit the builder for Hello entirely.</para> -</section> \ No newline at end of file +</section> diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index ddb3498946b7..74809bb412d3 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -196,6 +196,24 @@ in concat { x = "foo"; y = "bar"; }</programlisting> </para> +<para>A set that has a <literal>__functor</literal> attribute whose value +is callable (i.e. is itself a function or a set with a +<literal>__functor</literal> attribute whose value is callable) can be +applied as if it were a function, with the set itself passed in first +, e.g., + +<programlisting> +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +</programlisting> + +evaluates to <literal>2</literal>. This can be used to attach metadata to a +function without the caller needing to treat it specially, or to implement +a form of object-oriented programming, for example. + +</para> + </simplesect> diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index cc90409b5e93..e0dd98b7e67e 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -83,4 +83,6 @@ Just pass the option <link linkend='opt-max-jobs'><option>-j in parallel, or set. Typically this should be the number of CPUs.</para> -</section> \ No newline at end of file +<xi:include href="debug-build.xml" /> + +</section> diff --git a/doc/manual/expressions/simple-expression.xml b/doc/manual/expressions/simple-expression.xml index a8eb96f5a8e2..29fd872eea19 100644 --- a/doc/manual/expressions/simple-expression.xml +++ b/doc/manual/expressions/simple-expression.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-simple-expression"> -<title>Simple Nix Expression Use-Case</title> +<title>A Simple Nix Expression</title> <para>This section shows how to add and test the <link xlink:href='http://www.gnu.org/software/hello/hello.html'>GNU Hello @@ -44,4 +44,4 @@ need to do three things: <xi:include href="simple-building-testing.xml" /> <xi:include href="generic-builder.xml" /> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/expressions/standard-env.xml b/doc/manual/expressions/standard-env.xml deleted file mode 100644 index 2571f43fccba..000000000000 --- a/doc/manual/expressions/standard-env.xml +++ /dev/null @@ -1,60 +0,0 @@ -<chapter xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id='sec-standard-environment'> - -<title>The Standard Environment</title> - - -<para>The standard environment is used by passing it as an input -called <envar>stdenv</envar> to the derivation, and then doing - -<programlisting> -source $stdenv/setup</programlisting> - -at the top of the builder.</para> - -<para>Apart from adding the aforementioned commands to the -<envar>PATH</envar>, <filename>setup</filename> also does the -following: - -<itemizedlist> - - <listitem><para>All input packages specified in the - <envar>buildInputs</envar> environment variable have their - <filename>/bin</filename> subdirectory added to <envar>PATH</envar>, - their <filename>/include</filename> subdirectory added to the C/C++ - header file search path, and their <filename>/lib</filename> - subdirectory added to the linker search path. This can be extended. - For instance, when the <command>pkgconfig</command> package is - used, the subdirectory <filename>/lib/pkgconfig</filename> of each - input is added to the <envar>PKG_CONFIG_PATH</envar> environment - variable.</para></listitem> - - <listitem><para>The environment variable - <envar>NIX_CFLAGS_STRIP</envar> is set so that the compiler strips - debug information from object files. This can be disabled by - setting <envar>NIX_STRIP_DEBUG</envar> to - <literal>0</literal>.</para></listitem> - -</itemizedlist> - -</para> - -<para>The <filename>setup</filename> script also exports a function -called <function>genericBuild</function> that knows how to build -typical Autoconf-style packages. It can be customised to perform -builds for any type of package. It is advisable to use -<function>genericBuild</function> since it provides facilities that -are almost always useful such as unpacking of sources, patching of -sources, nested logging, etc.</para> - -<para>The definitive, up-to-date documentation of the generic builder -is the source itself, which resides in -<filename>pkgs/stdenv/generic/setup.sh</filename>.</para> - -<xi:include href="custom-builder.xml" /> -<xi:include href="debug-build.xml" /> - -</chapter> \ No newline at end of file diff --git a/doc/manual/expressions/writing-nix-expressions.xml b/doc/manual/expressions/writing-nix-expressions.xml index 6b797c200992..6646dddf0842 100644 --- a/doc/manual/expressions/writing-nix-expressions.xml +++ b/doc/manual/expressions/writing-nix-expressions.xml @@ -22,6 +22,5 @@ manual</link>.</para></note> <xi:include href="simple-expression.xml" /> <xi:include href="expression-language.xml" /> -<xi:include href="standard-env.xml" /> </part> diff --git a/doc/manual/installation/building-source.xml b/doc/manual/installation/building-source.xml index 2202ec73febe..772cda9cc36c 100644 --- a/doc/manual/installation/building-source.xml +++ b/doc/manual/installation/building-source.xml @@ -46,12 +46,4 @@ packages will need to be built from source.</para></warning> <filename>/nix/var</filename> by default. This can be changed using <option>--localstatedir=<replaceable>path</replaceable></option>.</para> -<para>If you want to rebuild the documentation, pass the full path to -the DocBook RELAX NG schemas and to the DocBook XSL stylesheets using -the -<option>--with-docbook-rng=<replaceable>path</replaceable></option> -and -<option>--with-docbook-xsl=<replaceable>path</replaceable></option> -options.</para> - -</section> \ No newline at end of file +</section> diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index a5f9ac844e09..f9ee98c726d2 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -6,7 +6,8 @@ <title>Installing a Binary Distribution</title> -<para>The easiest way to install Nix is to run the following command: +<para>If you are using Linux or Mac OS X, the easiest way to install +Nix is to run the following command: <screen> $ bash <(curl https://nixos.org/nix/install) @@ -18,7 +19,7 @@ run this under your usual user account, <emphasis>not</emphasis> as root. The script will invoke <command>sudo</command> to create <filename>/nix</filename> if it doesn’t already exist. If you don’t have <command>sudo</command>, you should manually create -<command>/nix</command> first as root: +<command>/nix</command> first as root, e.g.: <screen> $ mkdir /nix @@ -40,7 +41,7 @@ build system</link>.</para> or upgraded using <command>rpm -U</command>. For example, <screen> -$ rpm -U nix-1.7-1.i386.rpm</screen> +$ rpm -U nix-1.8-1.i386.rpm</screen> </para> @@ -48,21 +49,21 @@ $ rpm -U nix-1.7-1.i386.rpm</screen> install it like this: <screen> -$ dpkg -i nix_1.7-1_amd64.deb</screen> +$ dpkg -i nix_1.8-1_amd64.deb</screen> </para> -<para>For other platforms, including Mac OS X (Darwin), FreeBSD and -other Linux distributions, you can download a binary tarball that -contains Nix and all its dependencies. (This is what the install -script at <uri>https://nixos.org/nix/install</uri> uses.) You should -unpack it somewhere (e.g. in <filename>/tmp</filename>), and then run -the script named <command>install</command> inside the binary tarball: +<para>For other platforms, including Mac OS X and other Linux +distributions, you can download a binary tarball that contains Nix and +all its dependencies. (This is what the install script at +<uri>https://nixos.org/nix/install</uri> uses.) You should unpack it +somewhere (e.g. in <filename>/tmp</filename>), and then run the script +named <command>install</command> inside the binary tarball: <screen> alice$ cd /tmp -alice$ tar xfj nix-1.7-x86_64-darwin.tar.bz2 -alice$ cd nix-1.7-x86_64-darwin +alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +alice$ cd nix-1.8-x86_64-darwin alice$ ./install </screen> @@ -78,4 +79,4 @@ $ rm -rf /nix</screen> </para> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/installation/multi-user.xml b/doc/manual/installation/multi-user.xml index 312f5966452d..49c4f723597e 100644 --- a/doc/manual/installation/multi-user.xml +++ b/doc/manual/installation/multi-user.xml @@ -23,11 +23,11 @@ daemon</emphasis> running under the owner of the Nix store/database that performs the operation.</para> <note><para>Multi-user mode has one important limitation: only -<systemitem class="username">root</systemitem> can run <command -linkend="sec-nix-pull">nix-pull</command> to register the availability -of pre-built binaries. However, those registrations are shared by all -users, so they still get the benefit from <command>nix-pull</command>s -done by <systemitem class="username">root</systemitem>.</para></note> +<systemitem class="username">root</systemitem> and a set of trusted +users specified in <filename>nix.conf</filename> can specify arbitrary +binary caches. So while unprivileged users may install packages from +arbitrary Nix expressions, they may not get pre-built +binaries.</para></note> <simplesect> @@ -52,6 +52,34 @@ This creates 10 build users. There can never be more concurrent builds than the number of build users, so you may want to increase this if you expect to do many builds at the same time.</para> +<para>On Mac OS X, you can create the required group and users by +running the following script: + +<programlisting> +#! /bin/bash -e + +dseditgroup -o create nixbld -q + +gid=$(dscl . -read /Groups/nixbld | awk '($1 == "PrimaryGroupID:") {print $2 }') + +echo "created nixbld group with gid $gid" + +for i in $(seq 1 10); do + user=/Users/nixbld$i + uid="$((30000 + $i))" + dscl . create $user + dscl . create $user RealName "Nix build user $i" + dscl . create $user PrimaryGroupID "$gid" + dscl . create $user UserShell /usr/bin/false + dscl . create $user NFSHomeDirectory /var/empty + dscl . create $user UniqueID "$uid" + dseditgroup -o edit -a nixbld$i -t user nixbld + echo "created nixbld$i user with uid $uid" +done +</programlisting> + +</para> + </simplesect> diff --git a/doc/manual/installation/supported-platforms.xml b/doc/manual/installation/supported-platforms.xml index a31c6431f002..cbe528690cd9 100644 --- a/doc/manual/installation/supported-platforms.xml +++ b/doc/manual/installation/supported-platforms.xml @@ -10,12 +10,13 @@ <itemizedlist> - <listitem><para>Linux (particularly on x86, x86_64, and - PowerPC).</para></listitem> + <listitem><para>Linux (i686, x86_64).</para></listitem> - <listitem><para>Mac OS X.</para></listitem> + <listitem><para>Mac OS X (x86_64).</para></listitem> + <!-- <listitem><para>FreeBSD (only tested on Intel).</para></listitem> + --> <!-- <listitem><para>Windows through <link @@ -32,7 +33,7 @@ </para> -<para>Nix is pretty portable, so it should work on most other Unix -platforms as well.</para> +<para>Nix is fairly portable, so it should work on most platforms that +support POSIX threads and have a C++11 compiler.</para> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml index 4860b3d38fab..efd6cf2bb347 100644 --- a/doc/manual/introduction/about-nix.xml +++ b/doc/manual/introduction/about-nix.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ch-about-nix"> -<title>Introduction</title> +<title>About Nix</title> <para>Nix is a <emphasis>purely functional package manager</emphasis>. This means that it treats packages like values in purely functional @@ -16,10 +16,10 @@ store</emphasis>, usually the directory subdirectory such as <programlisting> -/nix/store/nlc4z5y1hm8w9s8vm6m1f5hy962xjmp5-firefox-12.0 +/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ </programlisting> -where <literal>nlc4z5…</literal> is a unique identifier for the +where <literal>b6gvzjyb2pg0…</literal> is a unique identifier for the package that captures all its dependencies (it’s a cryptographic hash of the package’s build dependency graph). This enables many powerful features.</para> @@ -161,19 +161,19 @@ library and the compiler) would have to built, at least if they are not already in the Nix store. This is a <emphasis>source deployment model</emphasis>. For most users, building from source is not very pleasant as it takes far too long. However, Nix can automatically -skip building from source and download a pre-built binary instead if -it knows about it. <emphasis>Nix channels</emphasis> provide Nix -expressions along with pre-built binaries.</para> - -<!-- -<para>source deployment model (like <a -href="http://www.gentoo.org/">Gentoo</a>) and a binary model (like -RPM)</para> ---> +skip building from source and instead use a <emphasis>binary +cache</emphasis>, a web server that provides pre-built binaries. For +instance, when asked to build +<literal>/nix/store/b6gvzjyb2pg0…-firefox-33.1</literal> from source, +Nix would first check if the file +<uri>https://cache.nixos.org/b6gvzjyb2pg0….narinfo</uri> exists, and +if so, fetch the pre-built binary referenced from there; otherwise, it +would fall back to building from source.</para> </simplesect> +<!-- <simplesect><title>Binary patching</title> <para>In addition to downloading binaries automatically if they’re @@ -182,6 +182,7 @@ package in the Nix store into a new version. This speeds up upgrades.</para> </simplesect> +--> <simplesect><title>Nix Packages collection</title> @@ -193,10 +194,48 @@ collection</emphasis> (Nixpkgs).</para> </simplesect> +<simplesect><title>Managing build environments</title> + +<para>Nix is extremely useful for developers as it makes it easy to +automatically set up the the build environment for a package. Given a +Nix expression that describes the dependencies of your package, the +command <command>nix-shell</command> will build or download those +dependencies if they’re not already in your Nix store, and then start +a Bash shell in which all necessary environment variables (such as +compiler search paths) are set.</para> + +<para>For example, the following command gets all dependencies of the +Pan newsreader, as described by <link +xlink:href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix">its +Nix expression</link>:</para> + +<screen> +$ nix-shell '<nixpkgs>' -A pan +</screen> + +<para>You’re then dropped into a shell where you can edit, build and test +the package:</para> + +<screen> +[nix-shell]$ tar xf $src +[nix-shell]$ cd pan-* +[nix-shell]$ ./configure +[nix-shell]$ make +[nix-shell]$ ./pan/gui/pan +</screen> + +<!-- +<para>Since Nix packages are reproducible and have complete dependency +specifications, Nix makes an excellent basis for <a +href="[%root%]hydra">a continuous build system</a>.</para> +--> + +</simplesect> + + <simplesect><title>Portability</title> -<para>Nix should run on most Unix systems, including Linux and Mac OS -X.</para> +<para>Nix runs on Linux and Mac OS X.</para> </simplesect> @@ -206,20 +245,22 @@ X.</para> <para>NixOS is a Linux distribution based on Nix. It uses Nix not just for package management but also to manage the system configuration (e.g., to build configuration files in -<filename>/etc</filename>). This means, among other things, that it’s -possible to easily roll back the entire configuration of the system to -an earlier state. Also, users can install software without root +<filename>/etc</filename>). This means, among other things, that it +is easy to roll back the entire configuration of the system to an +earlier state. Also, users can install software without root privileges. For more information and downloads, see the <link xlink:href="http://nixos.org/">NixOS homepage</link>.</para> </simplesect> -<!-- other features: +<simplesect><title>License</title> -- build farms -- reproducibility (Nix expressions allows whole configuration to be rebuilt) +<para>Nix is released under the terms of the <link +xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU +LGPLv2.1 or (at your option) any later version</link>.</para> + +</simplesect> ---> </chapter> diff --git a/doc/manual/introduction/introduction.xml b/doc/manual/introduction/introduction.xml index 665fef2fb990..12b2cc761063 100644 --- a/doc/manual/introduction/introduction.xml +++ b/doc/manual/introduction/introduction.xml @@ -7,5 +7,6 @@ <title>Introduction</title> <xi:include href="about-nix.xml" /> +<xi:include href="quick-start.xml" /> </part> diff --git a/doc/manual/quick-start/getting-started.xml b/doc/manual/introduction/quick-start.xml index 3dc79b9b52d5..0d13651e0ab3 100644 --- a/doc/manual/quick-start/getting-started.xml +++ b/doc/manual/introduction/quick-start.xml @@ -2,11 +2,14 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" version="5.0" - xml:id="ch-getting-started-nix"> + xml:id="chap-quick-start"> -<title>Getting Started with Nix</title> +<title>Quick Start</title> + +<para>This chapter is for impatient people who don't like reading +documentation. For more in-depth information you are kindly referred +to subsequent chapters.</para> -<para>This tutorial takes you through the basic tasks you might perform when you start using Nix.</para> <procedure> <step><para>Install single-user Nix by running the following: @@ -25,10 +28,11 @@ in the channel: <screen> $ nix-env -qa -docbook-xml-4.2 -firefox-1.0pre-PR-0.10.1 -hello-2.1.1 -libxslt-1.1.0 +docbook-xml-4.3 +docbook-xml-4.5 +firefox-33.0.2 +hello-2.9 +libxslt-1.1.28 <replaceable>...</replaceable></screen> </para></step> @@ -59,6 +63,28 @@ $ nix-env -e hello</screen> </para></step> +<step><para>You can also test a package without installing it: + +<screen> +$ nix-shell -p hello +</screen> + +This builds or downloads GNU Hello and its dependencies, then drops +you into a Bash shell where the <command>hello</command> command is +present, all without affecting your normal environment: + +<screen> +[nix-shell:~]$ hello +Hello, world! + +[nix-shell:~]$ exit + +$ hello +hello: command not found +</screen> + +</para></step> + <step><para>To keep up-to-date with the channel, do: <screen> @@ -69,6 +95,7 @@ The latter command will upgrade each installed package for which there is a “newer” version (as determined by comparing the version numbers).</para></step> +<!-- <step><para>You can also install specific packages directly from your web browser. For instance, you can go to <link xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest" @@ -78,6 +105,7 @@ the program <command>nix-install-package</command>. A window should appear asking you whether it’s okay to install the package. Say <literal>Y</literal>. The package and all its dependencies will be installed.</para></step> +--> <step><para>If you're unhappy with the result of a <command>nix-env</command> action (e.g., an upgraded package turned diff --git a/doc/manual/local.mk b/doc/manual/local.mk index a4df921b466a..3d7e7fed9631 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -7,7 +7,11 @@ XSLTPROC = $(xsltproc) --nonet $(xmlflags) \ --param admon.style \'\' \ --param callout.graphics.extension \'.gif\' \ --param contrib.inline.enabled 0 \ - --stringparam generate.toc "book toc" + --stringparam generate.toc "book toc" \ + --param keep.relative.image.uris 0 + +docbookxsl = http://docbook.sourceforge.net/release/xsl-ns/1.78.1 +docbookrng = http://docbook.org/xml/5.0/rng/docbook.rng MANUAL_SRCS := $(call rwildcard, $(d), *.xml) @@ -24,7 +28,7 @@ $(d)/version.txt: $(d)/manual.is-valid: $(d)/manual.xmli $(trace-gen) $(XSLTPROC) --novalid --stringparam profile.condition manual \ $(docbookxsl)/profiling/profile.xsl $< 2> /dev/null | \ - $(xmllint) --nonet --noout --relaxng $(docbookrng)/docbook.rng - + $(xmllint) --nonet --noout --relaxng $(docbookrng) - @touch $@ clean-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid @@ -36,7 +40,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ nix-collect-garbage.1 nix-push.1 nix-pull.1 \ - nix-prefetch-url.1 nix-channel.1 nix-generate-patches.1 \ + nix-prefetch-url.1 nix-channel.1 \ nix-install-package.1 nix-hash.1 nix-copy-closure.1 \ nix.conf.5 nix-daemon.8, \ $(d)/$(n)) diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml index 4b2088cffda7..61205d916993 100644 --- a/doc/manual/manual.xml +++ b/doc/manual/manual.xml @@ -4,10 +4,8 @@ version="5.0"> <info> - <title>Nix Package Manager Guide</title> - - <edition>Version <xi:include href="version.txt" parse="text" /></edition> + <subtitle>Version <xi:include href="version.txt" parse="text" /></subtitle> <author> <personname> @@ -25,22 +23,23 @@ <holder>Eelco Dolstra</holder> </copyright> - <date>September 2014</date> + <date>November 2014</date> </info> + <!-- <preface> <title>Preface</title> <para>This manual describes how to set up and use the Nix package manager.</para> </preface> + --> <xi:include href="introduction/introduction.xml" /> - <xi:include href="quick-start/quick-start.xml" /> <xi:include href="installation/installation.xml" /> <xi:include href="packages/package-management.xml" /> <xi:include href="expressions/writing-nix-expressions.xml" /> - <xi:include href="builds/build-farm.xml" /> + <xi:include href="advanced-topics/advanced-topics.xml" /> <xi:include href="command-ref/command-ref.xml" /> <xi:include href="troubleshooting/troubleshooting.xml" /> <xi:include href="glossary/glossary.xml" /> diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml index 69c955c1dd11..e8d1419da093 100644 --- a/doc/manual/packages/basic-package-mgmt.xml +++ b/doc/manual/packages/basic-package-mgmt.xml @@ -28,40 +28,71 @@ Nix expressions called the Nix Package collection that contains packages ranging from basic development stuff such as GCC and Glibc, to end-user applications like Mozilla Firefox. (Nix is however not tied to the Nix Package collection; you could write your own Nix -expressions based on it, or completely new ones.) You can download -the latest version from <link -xlink:href='http://nixos.org/nixpkgs/download.html' />.</para> +expressions based on it, or completely new ones.)</para> + +<para>You can manually download the latest version of Nixpkgs from +<link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However, +it’s much more convenient to use the Nixpkgs +<emphasis>channel</emphasis>, since it makes it easy to stay up to +date with new versions of Nixpkgs. (Channels are described in more +detail in <xref linkend="sec-channels"/>.) Nixpkgs is automatically +added to your list of “subscribed” channels when you install +Nix. If this is not the case for some reason, you can add it as +follows: -<para>Assuming that you have downloaded and unpacked a release of Nix -Packages, you can view the set of available packages in the release: +<screen> +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --update +</screen> + +</para> + +<note><para>On NixOS, you’re automatically subscribed to a NixOS +channel corresponding to your NixOS major release +(e.g. <uri>http://nixos.org/channels/nixos-14.12</uri>). A NixOS +channel is identical to the Nixpkgs channel, except that it contains +only Linux binaries and is updated only if a set of regression tests +succeed.</para></note> + +<para>You can view the set of available packages in Nixpkgs: <screen> -$ nix-env -qaf nixpkgs-<replaceable>version</replaceable> '*' -ant-blackdown-1.4.2 +$ nix-env -qa aterm-2.2 bash-3.0 binutils-2.15 bison-1.875d blackdown-1.4.2 bzip2-1.0.2 -...</screen> - -where <literal>nixpkgs-<replaceable>version</replaceable></literal> is -where you’ve unpacked the release. The flag <option>-q</option> -specifies a query operation; <option>-a</option> means that you want -to show the “available” (i.e., installable) packages, as opposed to -the installed packages; and <option>-f</option> -<filename>nixpkgs-<replaceable>version</replaceable></filename> -specifies the source of the packages. The argument -<literal>'*'</literal> shows all installable packages. (The quotes are -necessary to prevent shell expansion.) You can also select specific -packages by name: +…</screen> + +The flag <option>-q</option> specifies a query operation, and +<option>-a</option> means that you want to show the “available” (i.e., +installable) packages, as opposed to the installed packages. If you +downloaded Nixpkgs yourself, or if you checked it out from GitHub, +then you need to pass the path to your Nixpkgs tree using the +<option>-f</option> flag: + +<screen> +$ nix-env -qaf <replaceable>/path/to/nixpkgs</replaceable> +</screen> + +where <replaceable>/path/to/nixpkgs</replaceable> is where you’ve +unpacked or checked out Nixpkgs.</para> + +<para>You can select specific packages by name: + +<screen> +$ nix-env -qa firefox +firefox-34.0.5 +firefox-with-plugins-34.0.5 +</screen> + +and using regular expressions: <screen> -$ nix-env -qaf nixpkgs-<replaceable>version</replaceable> gcc -gcc-3.4.6 -gcc-4.0.3 -gcc-4.1.1</screen> +$ nix-env -qa 'firefox.*' +</screen> </para> @@ -70,12 +101,12 @@ available packages, i.e., whether they are installed into the user environment and/or present in the system: <screen> -$ nix-env -qasf nixpkgs-<replaceable>version</replaceable> '*' -... +$ nix-env -qas +… -PS bash-3.0 --S binutils-2.15 IPS bison-1.875d -...</screen> +…</screen> The first character (<literal>I</literal>) indicates whether the package is installed in your current user environment. The second @@ -88,40 +119,33 @@ just means that Nix knows that it can fetch a pre-built package from somewhere (typically a network server) instead of building it locally.</para> -<para>So now that we have a set of Nix expressions we can build the -packages contained in them. This is done using <literal>nix-env --i</literal>. For instance, +<para>You can install a package using <literal>nix-env -i</literal>. +For instance, <screen> -$ nix-env -f nixpkgs-<replaceable>version</replaceable> -i subversion</screen> +$ nix-env -i subversion</screen> will install the package called <literal>subversion</literal> (which is, of course, the <link xlink:href='http://subversion.tigris.org/'>Subversion version management system</link>).</para> -<para>When you do this for the first time, Nix will start building -Subversion and all its dependencies. This will take quite a while — -typically an hour or two on modern machines. Fortunately, there is a -faster way (so do a Ctrl-C on that install operation!): you just need -to tell Nix that pre-built binaries of all those packages are -available somewhere. This is done using the -<command>nix-pull</command> command, which must be supplied with a URL -containing a <emphasis>manifest</emphasis> describing what binaries -are available. This URL should correspond to the Nix Packages release -that you’re using. For instance, if you obtained a release from <link -xlink:href='http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x' -/>, then you should do: - -<screen> -$ nix-pull http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x/MANIFEST</screen> - -If you then issue the installation command, it should start -downloading binaries from <systemitem -class='fqdomainname'>nixos.org</systemitem>, instead of building -them from source. This might still take a while since all -dependencies must be downloaded, but on a reasonably fast connection -such as a DSL line it’s on the order of a few minutes.</para> +<note><para>When you ask Nix to install a package, it will first try +to get it in pre-compiled form from a <emphasis>binary +cache</emphasis>. By default, Nix will use the binary cache +<uri>https://cache.nixos.org</uri>; it contains binaries for most +packages in Nixpkgs. Only if no binary is available in the binary +cache, Nix will build the package from source. So if <literal>nix-env +-i subversion</literal> results in Nix building stuff from source, +then either the package is not built for your platform by the Nixpkgs +build servers, or your version of Nixpkgs is too old or too new. For +instance, if you have a very recent checkout of Nixpkgs, then the +Nixpkgs build servers may not have had a chance to build everything +and upload the resulting binaries to +<uri>https://cache.nixos.org</uri>. The Nixpkgs channel is only +updated after all binaries have been uploaded to the cache, so if you +stick to the Nixpkgs channel (rather than using a Git checkout of the +Nixpkgs tree), you will get binaries for most packages.</para></note> <para>Naturally, packages can also be uninstalled: @@ -134,7 +158,7 @@ $ nix-env -e subversion</screen> release of Nix Packages, you can do: <screen> -$ nix-env -f nixpkgs-<replaceable>version</replaceable> -u subversion</screen> +$ nix-env -u subversion</screen> This will <emphasis>only</emphasis> upgrade Subversion if there is a “newer” version in the new set of Nix expressions, as @@ -149,17 +173,17 @@ whatever version is already installed.</para> versions: <screen> -$ nix-env -f nixpkgs-<replaceable>version</replaceable> -u '*'</screen> +$ nix-env -u</screen> </para> <para>Sometimes it’s useful to be able to ask what <command>nix-env</command> would do, without actually doing it. For instance, to find out what packages would be upgraded by -<literal>nix-env -u '*'</literal>, you can do +<literal>nix-env -u</literal>, you can do <screen> -$ nix-env ... -u '*' --dry-run +$ nix-env -u --dry-run (dry run; not doing anything) upgrading `libxslt-1.1.0' to `libxslt-1.1.10' upgrading `graphviz-1.10' to `graphviz-1.12' @@ -167,4 +191,4 @@ upgrading `coreutils-5.0' to `coreutils-5.2.1'</screen> </para> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/packages/binary-cache-substituter.xml b/doc/manual/packages/binary-cache-substituter.xml new file mode 100644 index 000000000000..c6ceb9c80610 --- /dev/null +++ b/doc/manual/packages/binary-cache-substituter.xml @@ -0,0 +1,70 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-binary-cache-substituter"> + +<title>Serving a Nix store via HTTP</title> + +<para>You can easily share the Nix store of a machine via HTTP. This +allows other machines to fetch store paths from that machine to speed +up installations. It uses the same <emphasis>binary cache</emphasis> +mechanism that Nix usually uses to fetch pre-built binaries from +<uri>https://cache.nixos.org</uri>.</para> + +<para>The daemon that handles binary cache requests via HTTP, +<command>nix-serve</command>, is not part of the Nix distribution, but +you can install it from Nixpkgs: + +<screen> +$ nix-env -i nix-serve +</screen> + +You can then start the server, listening for HTTP connections on +whatever port you like: + +<screen> +$ nix-serve -p 8080 +</screen> + +To check whether it works, try the following on the client: + +<screen> +$ curl http://avalon:8080/nix-cache-info +</screen> + +which should print something like: + +<screen> +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 30 +</screen> + +</para> + +<para>On the client side, you can tell Nix to use your binary cache +using <option>--option extra-binary-caches</option>, e.g.: + +<screen> +$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/ +</screen> + +The option <option>extra-binary-caches</option> tells Nix to use this +binary cache in addition to your default caches, such as +<uri>https://cache.nixos.org</uri>. Thus, for any path in the closure +of Firefox, Nix will first check if the path is available on the +server <literal>avalon</literal> or another binary caches. If not, it +will fall back to building from source.</para> + +<para>You can also tell Nix to always use your binary cache by adding +a line to the <filename linkend="sec-conf-file">nix.conf</filename> +configuration file like this: + +<programlisting> +binary-caches = http://avalon:8080/ https://cache.nixos.org/ +</programlisting> + +</para> + +</section> diff --git a/doc/manual/packages/channels.xml b/doc/manual/packages/channels.xml index 094e11fe3b15..15c119fcb1f9 100644 --- a/doc/manual/packages/channels.xml +++ b/doc/manual/packages/channels.xml @@ -8,10 +8,9 @@ <para>If you want to stay up to date with a set of packages, it’s not very convenient to manually download the latest set of Nix expressions -for those packages, use <command>nix-pull</command> to register -pre-built binaries (if available), and upgrade using -<command>nix-env</command>. Fortunately, there’s a better way: -<emphasis>Nix channels</emphasis>.</para> +for those packages and upgrade using <command>nix-env</command>. +Fortunately, there’s a better way: <emphasis>Nix +channels</emphasis>.</para> <para>A Nix channel is just a URL that points to a place that contains a set of Nix expressions and a manifest. Using the command <link @@ -23,35 +22,36 @@ URL.</para> <command>nix-channel --add</command>, e.g., <screen> -$ nix-channel --add http://nixos.org/channels/nixpkgs-unstable</screen> +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable</screen> subscribes you to a channel that always contains that latest version -of the Nix Packages collection. (Instead of -<literal>nixpkgs-unstable</literal> you could also subscribe to -<literal>nixpkgs-stable</literal>, which should have a higher level of -stability, but right now is just outdated.) Subscribing really just -means that the URL is added to the file -<filename>~/.nix-channels</filename>. Right now there is no command -to “unsubscribe”; you should just edit that file manually -and delete the offending URL.</para> +of the Nix Packages collection. (Subscribing really just means that +the URL is added to the file <filename>~/.nix-channels</filename>, +where it is read by subsequent calls to <command>nix-channel +--update</command>.) You can “unsubscribe” using <command>nix-channel +--remove</command>: + +<screen> +$ nix-channel --remove nixpkgs +</screen> +</para> <para>To obtain the latest Nix expressions available in a channel, do <screen> $ nix-channel --update</screen> -This downloads the Nix expressions in every channel (downloaded from -<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal>) -and registers any available pre-built binaries in every channel -(by <command>nix-pull</command>ing -<literal><replaceable>url</replaceable>/MANIFEST</literal>). It also -makes the union of each channel’s Nix expressions the default for -<command>nix-env</command> operations. Consequently, you can then say +This downloads and unpacks the Nix expressions in every channel +(downloaded from <literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal>). +It also makes the union of each channel’s Nix expressions available by +default to <command>nix-env</command> operations (via the symlink +<filename>~/.nix-defexpr/channels</filename>). Consequently, you can +then say <screen> -$ nix-env -u '*'</screen> +$ nix-env -u</screen> to upgrade all packages in your profile to the latest versions available in the subscribed channels.</para> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/packages/copy-closure.xml b/doc/manual/packages/copy-closure.xml new file mode 100644 index 000000000000..012030e3eb67 --- /dev/null +++ b/doc/manual/packages/copy-closure.xml @@ -0,0 +1,50 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-copy-closure"> + +<title>Copying Closures Via SSH</title> + +<para>The command <command +linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix +store path along with all its dependencies to or from another machine +via the SSH protocol. It doesn’t copy store paths that are already +present on the target machine. For example, the following command +copies Firefox with all its dependencies: + +<screen> +$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen> + +See <xref linkend='sec-nix-copy-closure' /> for details.</para> + +<para>With <command linkend='refsec-nix-store-export'>nix-store +--export</command> and <command +linkend='refsec-nix-store-import'>nix-store --import</command> you can +write the closure of a store path (that is, the path and all its +dependencies) to a file, and then unpack that file into another Nix +store. For example, + +<screen> +$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure</screen> + +writes the closure of Firefox to a file. You can then copy this file +to another machine and install the closure: + +<screen> +$ nix-store --import < firefox.closure</screen> + +Any store paths in the closure that are already present in the target +store are ignored. It is also possible to pipe the export into +another command, e.g. to copy and install a closure directly to/on +another machine: + +<screen> +$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ + ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen> + +However, <command>nix-copy-closure</command> is generally more +efficient because it only copies paths that are not already present in +the target Nix store.</para> + +</section> diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml index ae28c485f076..03b8e4c976c1 100644 --- a/doc/manual/packages/garbage-collection.xml +++ b/doc/manual/packages/garbage-collection.xml @@ -37,7 +37,14 @@ generations, e.g., <screen> $ nix-env --delete-generations 10 11 14</screen> -</para> +To delete all generations older than a specified number of days +(except the current generation), use the <literal>d</literal> +suffix. For example, + +<screen> +$ nix-env --delete-generations 14d</screen> + +deletes all generations older than two weeks.</para> <para>After removing appropriate old generations you can run the garbage collector as follows: @@ -67,4 +74,4 @@ is a quick and easy way to clean up your system.</para> <xi:include href="garbage-collector-roots.xml" /> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/packages/profiles.xml b/doc/manual/packages/profiles.xml index ad5e92aeb64b..4d10319abe1c 100644 --- a/doc/manual/packages/profiles.xml +++ b/doc/manual/packages/profiles.xml @@ -30,7 +30,7 @@ store.</para> <figure xml:id='fig-user-environments'><title>User environments</title> <mediaobject> <imageobject> - <imagedata fileref='figures/user-environments.png' format='PNG' /> + <imagedata fileref='../figures/user-environments.png' format='PNG' /> </imageobject> </mediaobject> </figure> @@ -73,9 +73,9 @@ generated based on the current one. For instance, generation 43 was created from generation 42 when we did <screen> -$ nix-env -i subversion mozilla</screen> +$ nix-env -i subversion firefox</screen> -on a set of Nix expressions that contained Mozilla and a new version +on a set of Nix expressions that contained Firefox and a new version of Subversion.</para> <para>Generations are grouped together into @@ -120,8 +120,7 @@ can also see all available generations: <screen> $ nix-env --list-generations</screen></para> -<para>Actually, there is another level of indirection not shown in the -figure above. You generally wouldn’t have +<para>You generally wouldn’t have <filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename> in your <envar>PATH</envar>. Rather, there is a symlink <filename>~/.nix-profile</filename> that points to your current @@ -156,4 +155,4 @@ $ nix-env -p /nix/var/nix/profiles/other-profile -i subversion</screen> This will <emphasis>not</emphasis> change the <command>~/.nix-profile</command> symlink.</para> -</chapter> \ No newline at end of file +</chapter> diff --git a/doc/manual/packages/sharing-packages.xml b/doc/manual/packages/sharing-packages.xml index 8fab15f7ef2d..8465c182ee72 100644 --- a/doc/manual/packages/sharing-packages.xml +++ b/doc/manual/packages/sharing-packages.xml @@ -12,46 +12,8 @@ another machine already has some or all of those packages or their dependencies. In that case there are mechanisms to quickly copy packages between machines.</para> -<para>The command <command -linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix -store path along with all its dependencies to or from another machine -via the SSH protocol. It doesn’t copy store paths that are already -present on the target machine. For example, the following command -copies Firefox with all its dependencies: - -<screen> -$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen> - -See <xref linkend='sec-nix-copy-closure' /> for details.</para> - -<para>With <command linkend='refsec-nix-store-export'>nix-store ---export</command> and <command -linkend='refsec-nix-store-import'>nix-store --import</command> you can -write the closure of a store path (that is, the path and all its -dependencies) to a file, and then unpack that file into another Nix -store. For example, - -<screen> -$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure</screen> - -writes the closure of Firefox to a file. You can then copy this file -to another machine and install the closure: - -<screen> -$ nix-store --import < firefox.closure</screen> - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into -another command, e.g. to copy and install a closure directly to/on -another machine: - -<screen> -$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen> - -But note that <command>nix-copy-closure</command> is generally more -efficient in this example because it only copies paths that are not -already present in the target Nix store.</para> - +<xi:include href="binary-cache-substituter.xml" /> +<xi:include href="copy-closure.xml" /> +<xi:include href="ssh-substituter.xml" /> </chapter> diff --git a/doc/manual/packages/ssh-substituter.xml b/doc/manual/packages/ssh-substituter.xml new file mode 100644 index 000000000000..f24f354c4c39 --- /dev/null +++ b/doc/manual/packages/ssh-substituter.xml @@ -0,0 +1,73 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-ssh-substituter"> + +<title>Serving a Nix store via SSH</title> + +<para>You can tell Nix to automatically fetch needed binaries from a +remote Nix store via SSH. For example, the following installs Firefox, +automatically fetching any store paths in Firefox’s closure if they +are available on the server <literal>avalon</literal>: + +<screen> +$ nix-env -i firefox --option ssh-substituter-hosts alice@avalon +</screen> + +This works similar to the binary cache substituter that Nix usually +uses, only using SSH instead of HTTP: if a store path +<literal>P</literal> is needed, Nix will first check if it’s available +in the Nix store on <literal>avalon</literal>. If not, it will fall +back to using the binary cache substituter, and then to building from +source.</para> + +<note><para>The SSH substituter currently does not allow you to enter +an SSH passphrase interactively. Therefore, you should use +<command>ssh-add</command> to load the decrypted private key into +<command>ssh-agent</command>.</para></note> + +<para>You can also copy the closure of some store path, without +installing it into your profile, e.g. + +<screen> +$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --option ssh-substituter-hosts alice@avalon +</screen> + +This is essentially equivalent to doing + +<screen> +$ nix-copy-closure --from alice@avalon /nix/store/m85bxg…-firefox-34.0.5 +</screen> + +</para> + +<para>You can use SSH’s <emphasis>forced command</emphasis> feature to +set up a restricted user account for SSH substituter access, allowing +read-only access to the local Nix store, but nothing more. For +example, add the following lines to <filename>sshd_config</filename> +to restrict the user <literal>nix-ssh</literal>: + +<programlisting> +Match User nix-ssh + AllowAgentForwarding no + AllowTcpForwarding no + PermitTTY no + PermitTunnel no + X11Forwarding no + ForceCommand nix-store --serve +Match All +</programlisting> + +On NixOS, you can accomplish the same by adding the following to your +<filename>configuration.nix</filename>: + +<programlisting> +nix.sshServe.enable = true; +nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; +</programlisting> + +where the latter line lists the public keys of users that are allowed +to connect.</para> + +</section> diff --git a/doc/manual/quick-start/quick-start.xml b/doc/manual/quick-start/quick-start.xml deleted file mode 100644 index b4757cb22043..000000000000 --- a/doc/manual/quick-start/quick-start.xml +++ /dev/null @@ -1,17 +0,0 @@ -<part xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="chap-quick-start"> - -<title>Quick-Start</title> - -<partintro> -<para>This section is for impatient people who don't like reading -documentation. For more in-depth information you are kindly referred -to subsequent chapters.</para> -</partintro> - -<xi:include href="getting-started.xml" /> - -</part> diff --git a/doc/manual/release-notes/rl-1.8.xml b/doc/manual/release-notes/rl-1.8.xml index 0e6150ccf5c7..e551ee06055f 100644 --- a/doc/manual/release-notes/rl-1.8.xml +++ b/doc/manual/release-notes/rl-1.8.xml @@ -4,10 +4,19 @@ version="5.0" xml:id="ssec-relnotes-1.8"> -<title>Release 1.8 (TBA)</title> +<title>Release 1.8 (December 14, 2014)</title> <itemizedlist> + <listitem><para>Breaking change: to address a race condition, the + remote build hook mechanism now uses <command>nix-store + --serve</command> on the remote machine. This requires build slaves + to be updated to Nix 1.8.</para></listitem> + + <listitem><para>Nix now uses HTTPS instead of HTTP to access the + default binary cache, + <literal>cache.nixos.org</literal>.</para></listitem> + <listitem><para><command>nix-env</command> selectors are now regular expressions. For instance, you can do @@ -18,6 +27,42 @@ $ nix-env -qa '.*zip.*' to query all packages with a name containing <literal>zip</literal>.</para></listitem> + <listitem><para><command>nix-store --read-log</command> can now + fetch remote build logs. If a build log is not available locally, + then ‘nix-store -l’ will now try to download it from the servers + listed in the ‘log-servers’ option in nix.conf. For instance, if you + have the configuration option + +<programlisting> +log-servers = http://hydra.nixos.org/log +</programlisting> + +then it will try to get logs from +<literal>http://hydra.nixos.org/log/<replaceable>base name of the +store path</replaceable></literal>. This allows you to do things like: + +<screen> +$ nix-store -l $(which xterm) +</screen> + + and get a log even if <command>xterm</command> wasn't built + locally.</para></listitem> + + <listitem><para>New builtin functions: + <function>attrValues</function>, <function>deepSeq</function>, + <function>fromJSON</function>, <function>readDir</function>, + <function>seq</function>.</para></listitem> + + <listitem><para><command>nix-instantiate --eval</command> now has a + <option>--json</option> flag to print the resulting value in JSON + format.</para></listitem> + + <listitem><para><command>nix-copy-closure</command> now uses + <command>nix-store --serve</command> on the remote side to send or + receive closures. This fixes a race condition between + <command>nix-copy-closureE</command> and the garbage + collector.</para></listitem> + <listitem><para>Derivations can specify the new special attribute <varname>allowedRequisites</varname>, which has a similar meaning to <varname>allowedReferences</varname>. But instead of only enforcing @@ -26,6 +71,53 @@ $ nix-env -qa '.*zip.*' name, requisites) that are used by the resulting output.</para></listitem> + <listitem><para>On Mac OS X, Nix now handles case collisions when + importing closures from case-sensitive file systems. This is mostly + useful for running NixOps on Mac OS X.</para></listitem> + + <listitem><para>The Nix daemon has new configuration options + <option>allowed-users</option> (specifying the users and groups that + are allowed to connect to the daemon) and + <option>trusted-users</option> (specifying the users and groups that + can perform privileged operations like specifying untrusted binary + caches).</para></listitem> + + <listitem><para>The configuration option + <option>build-max-jobs</option> now defaults to the number of + available CPU cores.</para></listitem> + + <listitem><para>Build users are now used by default when Nix is + invoked as root. This prevents builds from accidentally running as + root.</para></listitem> + + <listitem><para>Nix now includes systemd units and Upstart + jobs.</para></listitem> + + <listitem><para>Speed improvements to <command>nix-store + --optimise</command>.</para></listitem> + + <listitem><para>Language change: the <literal>==</literal> operator + now ignores string contexts (the “dependencies” of a + string).</para></listitem> + + <listitem><para>Nix now filters out Nix-specific ANSI escape + sequences on standard error. They are supposed to be invisible, but + some terminals show them anyway.</para></listitem> + + <listitem><para>Various commands now automatically pipe their output + into the pager as specified by the <envar>PAGER</envar> environment + variable.</para></listitem> + + <listitem><para>Several improvements to reduce memory consumption in + the evaluator.</para></listitem> + </itemizedlist> +<para>This release has contributions from Adam Szkoda, Aristid +Breitkreuz, Bob van der Linden, Charles Strahan, darealshinji, Eelco +Dolstra, Gergely Risko, Joel Taylor, Ludovic Courtès, Marko Durkovic, +Mikey Ariel, Paul Colomiets, Ricardo M. Correia, Ricky Elrod, Robert +Helgesson, Rob Vermaas, Russell O'Connor, Shea Levy, Shell Turner, +Sönke Hahn, Steve Purcell, Vladimír Čunát and Wout Mertens.</para> + </section> diff --git a/doc/manual/style.css b/doc/manual/style.css index f805aeab076c..53fd9d5709c3 100644 --- a/doc/manual/style.css +++ b/doc/manual/style.css @@ -23,6 +23,11 @@ h1 /* title */ font-size: 200%; } +div.part h1 +{ + font-size: 240%; +} + h2 /* chapters, appendices, subtitle */ { font-size: 180%; @@ -30,7 +35,7 @@ h2 /* chapters, appendices, subtitle */ div.part { - margin-top: 2em; + margin-top: 4em; } /* Extra space between chapters, appendices. */ @@ -61,6 +66,12 @@ div.appendix h3 margin-top: 1.5em; } +div.refentry\.separator +{ + margin-top: 2.5em; + margin-bottom: 2em; +} + div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */ { margin-top: 1.4em; diff --git a/doc/manual/troubleshooting/links-nix-store.xml b/doc/manual/troubleshooting/links-nix-store.xml index c81477bd4f27..c768889567d0 100644 --- a/doc/manual/troubleshooting/links-nix-store.xml +++ b/doc/manual/troubleshooting/links-nix-store.xml @@ -20,7 +20,7 @@ in <filename>/nix/store</filename>, as can be seen using <command>ls -l</command>: <screen> -$ ls -l /nix/store +$ ls -ld /nix/store drwxrwxrwt 32000 nix nix 4620288 Sep 8 15:08 store</screen> The <literal>ext2</literal> file system is limited to an inode link @@ -35,9 +35,9 @@ machines).</para> the <option>--max-links</option> option.</para> <para>Real solution: put the Nix store on a file system that supports -more than 32,000 subdirectories per directory, such as ReiserFS. -(This doesn’t solve the <literal>st_nlink</literal> limit, but -ReiserFS lies to the kernel by reporting a link count of 1 if it -exceeds the limit.)</para> +more than 32,000 subdirectories per directory, such as ext4. (This +doesn’t solve the <literal>st_nlink</literal> limit, but ext4 lies to +the kernel by reporting a link count of 1 if it exceeds the +limit.)</para> </section> diff --git a/doc/manual/troubleshooting/troubleshooting.xml b/doc/manual/troubleshooting/troubleshooting.xml index 3618d907e596..1e973a192b18 100644 --- a/doc/manual/troubleshooting/troubleshooting.xml +++ b/doc/manual/troubleshooting/troubleshooting.xml @@ -7,8 +7,8 @@ <title>Troubleshooting</title> <para>This section provides solutions for some common problems. See -the <link xlink:href="http://bugs.strategoxt.org/browse/NIX">Nix -bug tracker</link> for a list of currently known issues.</para> +the <link xlink:href="https://github.com/NixOS/nix/issues">Nix bug +tracker</link> for a list of currently known issues.</para> <xi:include href="collisions-nixenv.xml" /> <xi:include href="links-nix-store.xml" /> diff --git a/doc/signing.txt b/doc/signing.txt index 1d042e95e220..7403cac470b2 100644 --- a/doc/signing.txt +++ b/doc/signing.txt @@ -1,6 +1,6 @@ Generate a private key: -$ (umask 277 && openssl genrsa -out /nix/etc/nix/signing-key.sec 2048) +$ (umask 277 && openssl genrsa -out /etc/nix/signing-key.sec 2048) The private key should be kept secret (only readable to the Nix daemon user). @@ -8,7 +8,7 @@ user). Generate the corresponding public key: -$ openssl rsa -in /nix/etc/nix/signing-key.sec -pubout > /nix/etc/nix/signing-key.pub +$ openssl rsa -in /etc/nix/signing-key.sec -pubout > /etc/nix/signing-key.pub The public key should be copied to all machines to which you want to export store paths. diff --git a/misc/launchd/local.mk b/misc/launchd/local.mk new file mode 100644 index 000000000000..0ba722efbf19 --- /dev/null +++ b/misc/launchd/local.mk @@ -0,0 +1,5 @@ +ifeq ($(OS), Darwin) + + $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons)) + +endif diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in new file mode 100644 index 000000000000..66fcd155ee9b --- /dev/null +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> + <dict> + <key>Label</key> + <string>org.nixos.nix-daemon</string> + <key>RunAtLoad</key> + <true/> + <key>Program</key> + <string>@bindir@/nix-daemon</string> + <key>StandardErrorPath</key> + <string>/var/log/nix-daemon.log</string> + <key>StandardOutPath</key> + <string>/dev/null</string> + </dict> +</plist> diff --git a/mk/functions.mk b/mk/functions.mk index 45d917399391..c48775db8c3b 100644 --- a/mk/functions.mk +++ b/mk/functions.mk @@ -10,5 +10,5 @@ filename-to-dep = $(dir $1).$(notdir $1).dep # empty string if not found. find-program = $(shell for i in $$(IFS=: ; echo $$PATH); do p=$$i/$(strip $1); if [ -e $$p ]; then echo $$p; break; fi; done) -# Remove trailing slash. +# Ensure that the given string ends in a single slash. add-trailing-slash = $(patsubst %/,%,$(1))/ diff --git a/mk/jars.mk b/mk/jars.mk index 99470f37435b..c8513e664ed5 100644 --- a/mk/jars.mk +++ b/mk/jars.mk @@ -1,4 +1,5 @@ define build-jar + $(1)_NAME ?= $(1) _d := $$(strip $$($(1)_DIR)) @@ -7,14 +8,20 @@ define build-jar $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp - $$($(1)_PATH): $$($(1)_SOURCES) + _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH)) + + $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER) @rm -rf $$($(1)_TMPDIR) @mkdir -p $$($(1)_TMPDIR) - $$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) $$($(1)_SOURCES) - $$(trace-jar) jar cf $$($(1)_PATH) -C $$($(1)_TMPDIR) . + $$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \ + $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \ + -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH" + @echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest + $$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) . + @rm $$($(1)_PATH).manifest @rm -rf $$($(1)_TMPDIR) - $(1)_INSTALL_DIR ?= $$(libdir)/java + $(1)_INSTALL_DIR ?= $$(jardir) $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar diff --git a/mk/lib.mk b/mk/lib.mk index 56e162d5007c..4ad5c636c8d4 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -14,23 +14,51 @@ dist-files := OS = $(shell uname -s) +# Hack to define a literal space. +space := +space += + + +# Hack to define a literal newline. +define newline + + +endef + + # Default installation paths. prefix ?= /usr/local libdir ?= $(prefix)/lib bindir ?= $(prefix)/bin libexecdir ?= $(prefix)/libexec datadir ?= $(prefix)/share +jardir ?= $(datadir)/java localstatedir ?= $(prefix)/var sysconfdir ?= $(prefix)/etc mandir ?= $(prefix)/share/man +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ +else + buildprefix = +endif + + # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 ifeq ($(BUILD_SHARED_LIBS), 1) - GLOBAL_CFLAGS += -fPIC - GLOBAL_CXXFLAGS += -fPIC + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + GLOBAL_CFLAGS += -U__STRICT_ANSI__ + GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ + else + GLOBAL_CFLAGS += -fPIC + GLOBAL_CXXFLAGS += -fPIC + endif ifneq ($(OS), Darwin) ifneq ($(OS), SunOS) GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries @@ -39,7 +67,6 @@ ifeq ($(BUILD_SHARED_LIBS), 1) SET_RPATH_TO_LIBS ?= 1 endif - # Pass -g if we want debug info. BUILD_DEBUG ?= 1 diff --git a/mk/libraries.mk b/mk/libraries.mk index 3b91c699e65a..3cd7a53107bd 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -3,7 +3,11 @@ libs-list := ifeq ($(OS), Darwin) SO_EXT = dylib else - SO_EXT = so + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + SO_EXT = dll + else + SO_EXT = so + endif endif # Build a library with symbolic name $(1). The library is defined by @@ -45,12 +49,16 @@ endif # built, otherwise a static library. define build-library $(1)_NAME ?= $(1) - _d := $$(strip $$($(1)_DIR)) + _d := $(buildprefix)$$(strip $$($(1)_DIR)) _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) - $(1)_OBJS := $$(addsuffix .o, $$(basename $$(_srcs))) + $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - $(1)_INSTALL_DIR ?= $$(libdir) + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + $(1)_INSTALL_DIR ?= $$(bindir) + else + $(1)_INSTALL_DIR ?= $$(libdir) + endif $(1)_LDFLAGS_USE := $(1)_LDFLAGS_USE_INSTALLED := @@ -65,7 +73,9 @@ define build-library endif else ifneq ($(OS), Darwin) - $(1)_LDFLAGS += -Wl,-z,defs + ifneq (CYGWIN,$(findstring CYGWIN,$(OS))) + $(1)_LDFLAGS += -Wl,-z,defs + endif endif endif @@ -76,9 +86,12 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) - $(1)_LDFLAGS_USE += -L$$(_d) -Wl,-rpath,$$(abspath $$(_d)) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) + ifneq ($(OS), Darwin) + $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) + endif + $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) $(1)_INSTALL_PATH := $(DESTDIR)$$($(1)_INSTALL_DIR)/$$($(1)_NAME).$(SO_EXT) @@ -90,10 +103,12 @@ define build-library $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) - ifeq ($(SET_RPATH_TO_LIBS), 1) - $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR) - else - $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath-link,$$($(1)_INSTALL_DIR) + ifneq ($(OS), Darwin) + ifeq ($(SET_RPATH_TO_LIBS), 1) + $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR) + else + $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath-link,$$($(1)_INSTALL_DIR) + endif endif ifdef $(1)_FORCE_INSTALL diff --git a/mk/patterns.mk b/mk/patterns.mk index 6b2cfd017050..3219d9629fbf 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,8 +1,11 @@ -%.o: %.cc +$(buildprefix)%.o: %.cc + @mkdir -p "$(dir $@)" $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP -%.o: %.cpp +$(buildprefix)%.o: %.cpp + @mkdir -p "$(dir $@)" $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP -%.o: %.c +$(buildprefix)%.o: %.c + @mkdir -p "$(dir $@)" $(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP diff --git a/mk/programs.mk b/mk/programs.mk index 72afdf95251b..3ac64494e3a5 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -23,9 +23,9 @@ programs-list := # - $(1)_INSTALL_DIR: the directory where the program will be # installed; defaults to $(bindir). define build-program - _d := $$($(1)_DIR) + _d := $(buildprefix)$$($(1)_DIR) _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) - $(1)_OBJS := $$(addsuffix .o, $$(basename $$(_srcs))) + $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) $(1)_PATH := $$(_d)/$(1) diff --git a/mk/templates.mk b/mk/templates.mk index ab99168bb7a5..c7ac7afbff27 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -8,8 +8,12 @@ define instantiate-template endef +ifneq ($(MAKECMDGOALS), clean) + %.h: %.h.in $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ %: %.in $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ + +endif diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 1520f1012c7d..800feb3b413a 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -57,6 +57,7 @@ sub copyTo { if ($@) { chomp $@; warn "$@; falling back to old closure copying method\n"; + $@ = ""; return oldCopyTo(@_); } diff --git a/perl/lib/Nix/SSH.pm b/perl/lib/Nix/SSH.pm index 9e0c10beb0e0..95393d881450 100644 --- a/perl/lib/Nix/SSH.pm +++ b/perl/lib/Nix/SSH.pm @@ -1,5 +1,6 @@ package Nix::SSH; +use utf8; use strict; use File::Temp qw(tempdir); use IPC::Open2; @@ -89,10 +90,16 @@ sub connectToRemoteNix { my $pid = open2($from, $to, "exec ssh -x -a $sshHost @globalSshOpts @{$sshOpts} nix-store --serve --write $extraFlags"); # Do the handshake. - my $SERVE_MAGIC_1 = 0x390c9deb; # FIXME - my $clientVersion = 0x200; - syswrite($to, pack("L<x4L<x4", $SERVE_MAGIC_1, $clientVersion)) or die; - die "did not get valid handshake from remote host\n" if readInt($from) != 0x5452eecb; + my $magic; + eval { + my $SERVE_MAGIC_1 = 0x390c9deb; # FIXME + my $clientVersion = 0x200; + syswrite($to, pack("L<x4L<x4", $SERVE_MAGIC_1, $clientVersion)) or die; + $magic = readInt($from); + }; + die "unable to connect to ‘$sshHost’\n" if $@; + die "did not get valid handshake from remote host\n" if $magic != 0x5452eecb; + my $serverVersion = readInt($from); die "unsupported server version\n" if $serverVersion < 0x200 || $serverVersion >= 0x300; diff --git a/perl/lib/Nix/Utils.pm b/perl/lib/Nix/Utils.pm index 32fb0aafb808..392c45f2fffb 100644 --- a/perl/lib/Nix/Utils.pm +++ b/perl/lib/Nix/Utils.pm @@ -43,5 +43,5 @@ sub readFile { sub mkTempDir { my ($name) = @_; return tempdir("$name.XXXXXX", CLEANUP => 1, DIR => $ENV{"TMPDIR"} // $ENV{"XDG_RUNTIME_DIR"} // "/tmp") - or die "cannot create a temporary directory"; + || die "cannot create a temporary directory"; } diff --git a/perl/local.mk b/perl/local.mk index 8ff60e9ce3c7..73d8a7c9526b 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -29,6 +29,12 @@ ifeq ($(perlbindings), yes) -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \ -D_FILE_OFFSET_BITS=64 -Wno-unused-variable -Wno-literal-suffix -Wno-reserved-user-defined-literal + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + archlib = $(shell perl -E 'use Config; print $$Config{archlib};') + libperl = $(shell perl -E 'use Config; print $$Config{libperl};') + Store_LDFLAGS = $(shell find ${archlib} -name ${libperl}) + endif + Store_ALLOW_UNDEFINED = 1 Store_FORCE_INSTALL = 1 diff --git a/release.nix b/release.nix index e84f49634051..a08cf7a96c81 100644 --- a/release.nix +++ b/release.nix @@ -24,12 +24,11 @@ let buildInputs = [ curl bison flex perl libxml2 libxslt bzip2 - tetex dblatex nukeReferences pkgconfig sqlite git - ]; + tetex dblatex nukeReferences pkgconfig sqlite + docbook5 docbook5_xsl + ] ++ lib.optional (!lib.inNixShell) git; configureFlags = '' - --with-docbook-rng=${docbook5}/xml/rng/docbook - --with-docbook-xsl=${docbook5_xsl}/xml/xsl/docbook --with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix} @@ -123,15 +122,16 @@ let in runCommand "nix-binary-tarball-${version}" - { exportReferencesGraph = [ "closure" toplevel ]; + { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ]; buildInputs = [ perl ]; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' - storePaths=$(perl ${pathsFromGraph} ./closure) - printRegistration=1 perl ${pathsFromGraph} ./closure > $TMPDIR/reginfo + storePaths=$(perl ${pathsFromGraph} ./closure1 ./closure2) + printRegistration=1 perl ${pathsFromGraph} ./closure1 ./closure2 > $TMPDIR/reginfo substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${toplevel} + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} chmod +x $TMPDIR/install dir=nix-${version}-${system} fn=$out/$dir.tar.bz2 @@ -181,44 +181,51 @@ let }; - rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50; - rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50; - rpm_fedora18i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora18i386) 60; - rpm_fedora18x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora18x86_64) 60; - rpm_fedora19i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora19i386) 70; - rpm_fedora19x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora19x86_64) 70; - rpm_fedora20i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora20i386) 70; - rpm_fedora20x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora20x86_64) 70; + rpm_fedora20i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora20i386); + rpm_fedora20x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora20x86_64); - #deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50; - #deb_debian60x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian60x86_64) 50; - deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386) 60; - deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64) 60; + deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386); + deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64); - deb_ubuntu1110i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1110i386) 60; - deb_ubuntu1110x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1110x86_64) 60; - deb_ubuntu1204i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1204i386) 60; - deb_ubuntu1204x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1204x86_64) 60; - deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386) 70; - deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64) 70; - deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386) 80; - deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64) 80; - deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) 90; - deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) 90; - deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386) 90; - deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64) 90; + deb_ubuntu1210i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1210i386); + deb_ubuntu1210x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1210x86_64); + deb_ubuntu1304i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1304i386); + deb_ubuntu1304x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1304x86_64); + deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386); + deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64); + deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386); + deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64); + deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386); + deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64); # System tests. - tests.remote_builds = (import ./tests/remote-builds.nix rec { + tests.remoteBuilds = (import ./tests/remote-builds.nix rec { nix = build.x86_64-linux; system = "x86_64-linux"; }); - tests.nix_copy_closure = (import ./tests/nix-copy-closure.nix rec { + tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec { nix = build.x86_64-linux; system = "x86_64-linux"; }); + tests.binaryTarball = + with import <nixpkgs> { system = "x86_64-linux"; }; + vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" + { diskImage = vmTools.diskImages.ubuntu1204x86_64; + } + '' + useradd -m alice + su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' + mount -t tmpfs none /nix # Provide a writable /nix. + chown alice /nix + su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' + su - alice -c 'nix-store --verify' + su - alice -c 'nix-store -qR ${build.x86_64-linux}' + mkdir -p $out/nix-support + touch $out/nix-support/hydra-build-products + ''); # */ + # Aggregate job containing the release-critical jobs. release = pkgs.releaseTools.aggregate { @@ -238,18 +245,15 @@ let binaryTarball.x86_64-linux deb_debian7i386 deb_debian7x86_64 - deb_ubuntu1304i386 - deb_ubuntu1304x86_64 - deb_ubuntu1310i386 - deb_ubuntu1310x86_64 deb_ubuntu1404i386 deb_ubuntu1404x86_64 - rpm_fedora19i386 - rpm_fedora19x86_64 + deb_ubuntu1410i386 + deb_ubuntu1410x86_64 rpm_fedora20i386 rpm_fedora20x86_64 - tests.remote_builds - tests.nix_copy_closure + tests.remoteBuilds + tests.nix-copy-closure + tests.binaryTarball ]; }; @@ -260,7 +264,7 @@ let makeRPM_x86_64 = makeRPM "x86_64-linux"; makeRPM = - system: diskImageFun: prio: + system: diskImageFun: with import <nixpkgs> { inherit system; }; @@ -270,7 +274,7 @@ let diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" ]; }; memSize = 1024; - meta.schedulingPriority = prio; + meta.schedulingPriority = 50; postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck"; }; @@ -279,7 +283,7 @@ let makeDeb_x86_64 = makeDeb "x86_64-linux"; makeDeb = - system: diskImageFun: prio: + system: diskImageFun: with import <nixpkgs> { inherit system; }; @@ -289,7 +293,7 @@ let diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = [ "libdbd-sqlite3-perl" "libsqlite3-dev" "libbz2-dev" "libwww-curl-perl" ]; }; memSize = 1024; - meta.schedulingPriority = prio; + meta.schedulingPriority = 50; configureFlags = "--sysconfdir=/etc"; debRequires = [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" ]; debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>"; diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 5f0c72b663b8..dcf245d7d55f 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -259,11 +259,13 @@ close UPLOADLOCK; # Perform the build. print STDERR "building ‘$drvPath’ on ‘$hostName’\n"; +print STDERR "@ build-remote-start $drvPath $hostName\n" if $printBuildTrace; writeInt(6, $to) or die; # == cmdBuildPaths writeStrings([$drvPath], $to); writeInt($maxSilentTime, $to); writeInt($buildTimeout, $to); my $res = readInt($from); +print STDERR "@ build-remote-done $drvPath $hostName\n" if $printBuildTrace; if ($res != 0) { my $msg = decode("utf-8", readString($from)); print STDERR "error: $msg on ‘$hostName’\n"; diff --git a/scripts/copy-from-other-stores.pl.in b/scripts/copy-from-other-stores.pl.in index cf36bae9e803..8ce5a9d4049f 100755 --- a/scripts/copy-from-other-stores.pl.in +++ b/scripts/copy-from-other-stores.pl.in @@ -9,6 +9,7 @@ my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; STDOUT->autoflush(1); +binmode STDERR, ":encoding(utf8)"; my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or ""); diff --git a/scripts/download-from-binary-cache.pl.in b/scripts/download-from-binary-cache.pl.in index cdce8eb74a41..4655f9ac9a0e 100644 --- a/scripts/download-from-binary-cache.pl.in +++ b/scripts/download-from-binary-cache.pl.in @@ -13,6 +13,7 @@ use WWW::Curl::Easy; use WWW::Curl::Multi; use strict; +STDERR->autoflush(1); binmode STDERR, ":encoding(utf8)"; Nix::Config::readConfig; @@ -41,9 +42,18 @@ my $curlIdCount = 1; my %requests; my %scheduled; my $caBundle = $ENV{"SSL_CERT_FILE"} // $ENV{"CURL_CA_BUNDLE"} // $ENV{"OPENSSL_X509_CERT_FILE"}; +$caBundle = "/etc/ssl/certs/ca-bundle.crt" if !$caBundle && -f "/etc/ssl/certs/ca-bundle.crt"; +$caBundle = "/etc/ssl/certs/ca-certificates.crt" if !$caBundle && -f "/etc/ssl/certs/ca-certificates.crt"; my $userName = getpwuid($<) || $ENV{"USER"} or die "cannot figure out user name"; +my $userAgent = "Nix/$Nix::Config::version"; + +sub isTrue { + my ($x) = @_; + return $x eq "true" || $x eq "1"; +} + my $requireSignedBinaryCaches = ($Nix::Config::config{"signed-binary-caches"} // "0") ne "0"; my $curlConnectTimeout = int( @@ -66,7 +76,8 @@ sub addRequest { $curl->setopt(CURLOPT_WRITEDATA, $fh); $curl->setopt(CURLOPT_FOLLOWLOCATION, 1); $curl->setopt(CURLOPT_CAINFO, $caBundle) if defined $caBundle; - $curl->setopt(CURLOPT_USERAGENT, "Nix/$Nix::Config::version"); + $curl->setopt(CURLOPT_SSL_VERIFYPEER, 0) unless isTrue($Nix::Config::config{"verify-https-binary-caches"} // "1"); + $curl->setopt(CURLOPT_USERAGENT, $userAgent); $curl->setopt(CURLOPT_NOBODY, 1) if $head; $curl->setopt(CURLOPT_FAILONERROR, 1); $curl->setopt(CURLOPT_CONNECTTIMEOUT, $curlConnectTimeout); @@ -217,7 +228,7 @@ sub getAvailableCaches { } my @urls = strToList($Nix::Config::config{"binary-caches"} // - ($Nix::Config::storeDir eq "/nix/store" ? "http://cache.nixos.org" : "")); + ($Nix::Config::storeDir eq "/nix/store" ? "https://cache.nixos.org" : "")); my $urlsFiles = $Nix::Config::config{"binary-cache-files"} // "$Nix::Config::stateDir/profiles/per-user/$userName/channels/binary-caches/*"; @@ -546,7 +557,7 @@ sub downloadBinary { die if $requireSignedBinaryCaches && !defined $info->{signedBy}; print STDERR "\n*** Downloading ‘$url’ ", ($requireSignedBinaryCaches ? "(signed by ‘$info->{signedBy}’) " : ""), "to ‘$storePath’...\n"; checkURL $url; - if (system("$Nix::Config::curl --fail --location --insecure --connect-timeout $curlConnectTimeout '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { + if (system("$Nix::Config::curl --fail --location --insecure --connect-timeout $curlConnectTimeout -A '$userAgent' '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { warn "download of ‘$url’ failed" . ($! ? ": $!" : "") . "\n"; next; } diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index ef56149915b1..c9ba9a2a280a 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -5,6 +5,7 @@ set -e dest="/nix" self="$(dirname "$0")" nix="@nix@" +cacert="@cacert@" if ! [ -e $self/.reginfo ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 @@ -17,7 +18,7 @@ if [ -z "$USER" ]; then fi if [ "$(id -u)" -eq 0 ]; then - echo "warning: installing Nix as root is not recommended" >&2 + printf '\e[1;31mwarning: installing Nix as root is not supported by this script!\e[0m\n' fi echo "performing a single-user installation of Nix..." >&2 @@ -66,23 +67,24 @@ fi . $nix/etc/profile.d/nix.sh -if ! $nix/bin/nix-env -i $nix; then +if ! $nix/bin/nix-env -i "$nix"; then echo "$0: unable to install Nix into your default profile" >&2 exit 1 fi +# Install an SSL certificate bundle. +if [ -z "$SSL_CERT_FILE" -o ! -f "$SSL_CERT_FILE" ]; then + $nix/bin/nix-env -i "$cacert" + export SSL_CERT_FILE="$HOME/.nix-profile/etc/ca-bundle.crt" +fi + # Subscribe the user to the Nixpkgs channel and fetch it. if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - if [ -n "$SSL_CERT_FILE" ]; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable - else - $nix/bin/nix-channel --add http://nixos.org/channels/nixpkgs-unstable - fi + $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable +fi +if [ -z "$_NIX_INSTALLER_TEST" ]; then + $nix/bin/nix-channel --update nixpkgs fi -$nix/bin/nix-channel --update nixpkgs - -# Install an SSL certificate bundle. -$nix/bin/nix-env -iA nixpkgs.cacert || true # Make the shell source nix.sh during login. p=$NIX_LINK/etc/profile.d/nix.sh diff --git a/scripts/nix-build.in b/scripts/nix-build.in index f8cf318ff07a..ca43041b77b0 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -5,6 +5,8 @@ use strict; use Nix::Config; use Nix::Store; use Nix::Utils; +use File::Basename; +use Cwd; binmode STDERR, ":encoding(utf8)"; @@ -14,6 +16,7 @@ my $runEnv = $0 =~ /nix-shell$/; my $pure = 0; my $fromArgs = 0; my $packages = 0; +my $interactive = 1; my @instArgs = (); my @buildArgs = (); @@ -25,6 +28,8 @@ my @envExclude = (); my $myName = $runEnv ? "nix-shell" : "nix-build"; +my $inShebang = 0; +my $script; my $tmpDir = mkTempDir($myName); @@ -35,6 +40,29 @@ my $drvLink = "$tmpDir/derivation"; $SIG{'INT'} = sub { exit 1 }; +# Heuristic to see if we're invoked as a shebang script, namely, if we +# have a single argument, it's the name of an executable file, and it +# starts with "#!". +if ($runEnv && scalar @ARGV == 1) { + $script = $ARGV[0]; + if (-f $script && -x $script) { + open SCRIPT, "<$script" or die "$0: cannot open ‘$script’: $!\n"; + my $first = <SCRIPT>; + if ($first =~ /^\#\!/) { + $inShebang = 1; + @ARGV = (); + while (<SCRIPT>) { + chomp; + if (/^\#\!\s*nix-shell (.*)$/) { + @ARGV = split / /, $1; + } + } + } + close SCRIPT; + } +} + + for (my $n = 0; $n < scalar @ARGV; $n++) { my $arg = $ARGV[$n]; @@ -131,10 +159,11 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $runEnv = 1; } - elsif ($arg eq "--command") { + elsif ($arg eq "--command" || $arg eq "--run") { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - $envCommand = "$ARGV[$n]\nexit $!"; + $envCommand = "$ARGV[$n]\nexit"; + $interactive = 0 if $arg eq "--run"; } elsif ($arg eq "--exclude") { @@ -155,6 +184,18 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $packages = 1; } + elsif ($inShebang && $arg eq "-i") { + $n++; + die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; + my $interpreter = $ARGV[$n]; + # Überhack to support Perl. Perl examines the shebang and + # executes it unless it contains the string "perl" or "indir", + # or (undocumented) argv[0] does not contain "perl". Exploit + # the latter by doing "exec -a". + my $execArgs = $interpreter =~ /perl/ ? "-a PERL" : ""; + $envCommand = "exec $execArgs $interpreter $script"; + } + elsif (substr($arg, 0, 1) eq "-") { push @buildArgs, $arg; } @@ -182,6 +223,11 @@ foreach my $expr (@exprs) { # Instantiate. my @drvPaths; if ($expr !~ /^\/.*\.drv$/) { + # If we're in a #! script, interpret filenames relative to the + # script. + $expr = dirname(Cwd::abs_path($script)) . "/" . $expr + if $inShebang && $expr !~ /^\//; + # !!! would prefer the perl 5.8.0 pipe open feature here. my $pid = open(DRVPATHS, "-|") || exec "$Nix::Config::binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr; while (<DRVPATHS>) {chomp; push @drvPaths, $_;} @@ -232,17 +278,20 @@ foreach my $expr (@exprs) { ($pure ? '' : 'p=$PATH; ' ) . 'dontAddDisableDepTrack=1; ' . '[ -e $stdenv/setup ] && source $stdenv/setup; ' . - 'if [ "$(type -t runHook)" = function ]; then runHook shellHook; fi; ' . ($pure ? '' : 'PATH=$PATH:$p; unset p; ') . 'set +e; ' . '[ -n "$PS1" ] && PS1="\n\[\033[1;32m\][nix-shell:\w]$\[\033[0m\] "; ' . + 'if [ "$(type -t runHook)" = function ]; then runHook shellHook; fi; ' . 'unset NIX_ENFORCE_PURITY; ' . 'unset NIX_INDENT_MAKE; ' . 'shopt -u nullglob; ' . 'unset TZ; ' . (defined $ENV{'TZ'} ? "export TZ='${ENV{'TZ'}}'; " : '') . $envCommand); $ENV{BASH_ENV} = $rcfile; - exec($ENV{NIX_BUILD_SHELL} // "bash", "--rcfile", $rcfile); + my @args = ($ENV{NIX_BUILD_SHELL} // "bash"); + push @args, "--rcfile" if $interactive; + push @args, $rcfile; + exec @args; die; } @@ -276,7 +325,7 @@ foreach my $expr (@exprs) { while (<OUTPATHS>) {chomp; push @outPaths, $_;} if (!close OUTPATHS) { die "nix-store killed by signal " . ($? & 127) . "\n" if ($? & 127); - exit $? >> 8 || 1; + exit ($? >> 8 || 1); } next if $dryRun; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 7dd7968c314a..a91b54bd589f 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -10,9 +10,9 @@ if [ -n "$HOME" ]; then export PATH=$NIX_LINK/bin:$NIX_LINK/sbin:$PATH - # Subscribe the root user to the Nixpkgs channel by default. + # Subscribe the user to the Nixpkgs channel by default. if [ ! -e $HOME/.nix-channels ]; then - echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels fi # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index b64455eb1724..f71cf56507b8 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -9,6 +9,7 @@ #include "store-api.hh" #include <iostream> +#include <cstdlib> #include <unistd.h> using namespace nix; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5b9db6eeaa6e..298f6a3a60e3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -104,6 +104,9 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con case tPrimOpApp: str << "<PRIMOP-APP>"; break; + case tExternal: + str << *v.external; + break; default: throw Error("invalid value"); } @@ -136,6 +139,7 @@ string showType(const Value & v) case tBlackhole: return "a black hole"; case tPrimOp: return "a built-in function"; case tPrimOpApp: return "a partially applied built-in function"; + case tExternal: return v.external->showType(); } abort(); } @@ -180,6 +184,7 @@ EvalState::EvalState(const Strings & _searchPath) , sFile(symbols.create("file")) , sLine(symbols.create("line")) , sColumn(symbols.create("column")) + , sFunctor(symbols.create("__functor")) , repair(false) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) @@ -212,9 +217,9 @@ EvalState::EvalState(const Strings & _searchPath) allocated. This might be a problem on systems that don't overcommit. */ if (!getenv("GC_INITIAL_HEAP_SIZE")) { - size_t maxSize = 384 * 1024 * 1024; size_t size = 32 * 1024 * 1024; #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + size_t maxSize = 384 * 1024 * 1024; long pageSize = sysconf(_SC_PAGESIZE); long pages = sysconf(_SC_PHYS_PAGES); if (pageSize != -1) @@ -242,7 +247,6 @@ EvalState::EvalState(const Strings & _searchPath) EvalState::~EvalState() { fileEvalCache.clear(); - printCanaries(); } @@ -312,11 +316,6 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, co throw EvalError(format(s) % sym % p1 % p2); } -LocalNoInlineNoReturn(void throwTypeError(const char * s)) -{ - throw TypeError(s); -} - LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) { throw TypeError(format(s) % pos); @@ -327,11 +326,6 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) throw TypeError(format(s) % s1); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1, const string & s2)) -{ - throw TypeError(format(s) % s1 % s2); -} - LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) { throw TypeError(format(s) % fun.showNamePos() % s2 % pos); @@ -895,6 +889,17 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po return; } + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + forceValue(*found->value); + Value * v2 = allocValue(); + callFunction(*found->value, fun, *v2, pos); + forceValue(*v2); + return callFunction(*v2, arg, v, pos); + } + } + if (fun.type != tLambda) throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); @@ -1255,9 +1260,9 @@ void copyContext(const Value & v, PathSet & context) } -string EvalState::forceString(Value & v, PathSet & context) +string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { - string s = forceString(v); + string s = forceString(v, pos); copyContext(v, context); return s; } @@ -1312,6 +1317,9 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } + if (v.type == tExternal) + return v.external->coerceToString(pos, context, coerceMore, copyToStore); + if (coerceMore) { /* Note that `false' is represented as an empty string for @@ -1432,6 +1440,9 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tPrimOpApp: return false; + case tExternal: + return *v1.external == *v2.external; + default: throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); } @@ -1502,26 +1513,6 @@ void EvalState::printStats() } -void EvalState::printCanaries() -{ -#if HAVE_BOEHMGC - if (!settings.get("debug-gc", false)) return; - - GC_gcollect(); - - if (gcCanaries.empty()) { - printMsg(lvlError, "all canaries have been garbage-collected"); - return; - } - - printMsg(lvlError, "the following canaries have not been garbage-collected:"); - - for (auto i : gcCanaries) - printMsg(lvlError, format(" %1%") % i->string.s); -#endif -} - - size_t valueSize(Value & v) { std::set<const void *> seen; @@ -1573,6 +1564,11 @@ size_t valueSize(Value & v) sz += doValue(*v.primOpApp.left); sz += doValue(*v.primOpApp.right); break; + case tExternal: + if (seen.find(v.external) != seen.end()) break; + seen.insert(v.external); + sz += v.external->valueSize(seen); + break; default: ; } @@ -1599,4 +1595,22 @@ size_t valueSize(Value & v) } +string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +{ + throw TypeError(format("cannot coerce %1% to a string, at %2%") % + showType() % pos); +} + + +bool ExternalValueBase::operator==(const ExternalValueBase & b) const +{ + return false; +} + + +std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { + return v.print(str); +} + + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index daf53846ffd5..f7415fb78dfd 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -39,10 +39,10 @@ public: typedef uint32_t size_t; private: - size_t size_, capacity; + size_t size_, capacity_; Attr attrs[0]; - Bindings(uint32_t capacity) : size_(0), capacity(capacity) { } + Bindings(size_t capacity) : size_(0), capacity_(capacity) { } Bindings(const Bindings & bindings) = delete; public: @@ -54,7 +54,7 @@ public: void push_back(const Attr & attr) { - assert(size_ < capacity); + assert(size_ < capacity_); attrs[size_++] = attr; } @@ -76,6 +76,8 @@ public: void sort(); + size_t capacity() { return capacity_; } + friend class EvalState; }; @@ -126,7 +128,7 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, - sFile, sLine, sColumn; + sFile, sLine, sColumn, sFunctor; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -169,7 +171,7 @@ public: /* Look up a file in the search path. */ Path findFile(const string & path); - Path findFile(SearchPath & searchPath, const string & path); + Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); /* Evaluate an expression to normal form, storing the result in value `v'. */ @@ -200,7 +202,7 @@ public: inline void forceList(Value & v, const Pos & pos); void forceFunction(Value & v, const Pos & pos); // either lambda or primop string forceString(Value & v, const Pos & pos = noPos); - string forceString(Value & v, PathSet & context); + string forceString(Value & v, PathSet & context, const Pos & pos = noPos); string forceStringNoCtx(Value & v, const Pos & pos = noPos); /* Return true iff the value `v' denotes a derivation (i.e. a @@ -287,8 +289,6 @@ public: /* Print statistics. */ void printStats(); - void printCanaries(); - private: unsigned long nrEnvs; @@ -320,12 +320,6 @@ private: friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); - -#if HAVE_BOEHMGC - std::set<Value *> gcCanaries; - friend void canaryFinalizer(GC_PTR obj, GC_PTR client_data); - friend void prim_gcCanary(EvalState & state, const Pos & pos, Value * * args, Value & v); -#endif }; @@ -340,6 +334,9 @@ struct InvalidPathError : EvalError { Path path; InvalidPathError(const Path & path); +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~InvalidPathError() throw () { }; +#endif }; /* Realise all paths in `context' */ diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index c8521718b82a..50997e096fd1 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -16,6 +16,47 @@ std::ostream & operator << (std::ostream & str, Expr & e) return str; } +static void showString(std::ostream & str, const string & s) +{ + str << '"'; + for (auto c : (string) s) + if (c == '"' || c == '\\' || c == '$') str << "\\" << c; + else if (c == '\n') str << "\\n"; + else if (c == '\r') str << "\\r"; + else if (c == '\t') str << "\\t"; + else str << c; + str << '"'; +} + +static void showId(std::ostream & str, const string & s) +{ + assert(!s.empty()); + if (s == "if") + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + showString(str, s); + return; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + showString(str, s); + return; + } + str << s; + } +} + +std::ostream & operator << (std::ostream & str, const Symbol & sym) +{ + showId(str, *sym.s); + return str; +} + void Expr::show(std::ostream & str) { abort(); @@ -28,7 +69,7 @@ void ExprInt::show(std::ostream & str) void ExprString::show(std::ostream & str) { - str << "\"" << s << "\""; // !!! escaping + showString(str, s); } void ExprPath::show(std::ostream & str) @@ -44,12 +85,12 @@ void ExprVar::show(std::ostream & str) void ExprSelect::show(std::ostream & str) { str << "(" << *e << ")." << showAttrPath(attrPath); - if (def) str << " or " << *def; + if (def) str << " or (" << *def << ")"; } void ExprOpHasAttr::show(std::ostream & str) { - str << "(" << *e << ") ? " << showAttrPath(attrPath); + str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; } void ExprAttrs::show(std::ostream & str) @@ -85,6 +126,10 @@ void ExprLambda::show(std::ostream & str) str << i->name; if (i->def) str << " ? " << *i->def; } + if (formals->ellipsis) { + if (!first) str << ", "; + str << "..."; + } str << " }"; if (!arg.empty()) str << " @ "; } @@ -94,23 +139,24 @@ void ExprLambda::show(std::ostream & str) void ExprLet::show(std::ostream & str) { - str << "let "; + str << "(let "; foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs) - if (i->second.inherited) + if (i->second.inherited) { str << "inherit " << i->first << "; "; + } else str << i->first << " = " << *i->second.e << "; "; - str << "in " << *body; + str << "in " << *body << ")"; } void ExprWith::show(std::ostream & str) { - str << "with " << *attrs << "; " << *body; + str << "(with " << *attrs << "; " << *body << ")"; } void ExprIf::show(std::ostream & str) { - str << "if " << *cond << " then " << *then << " else " << *else_; + str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; } void ExprAssert::show(std::ostream & str) @@ -120,16 +166,18 @@ void ExprAssert::show(std::ostream & str) void ExprOpNot::show(std::ostream & str) { - str << "! " << *e; + str << "(! " << *e << ")"; } void ExprConcatStrings::show(std::ostream & str) { bool first = true; + str << "("; foreach (vector<Expr *>::iterator, i, *es) { if (first) first = false; else str << " + "; str << **i; } + str << ")"; } void ExprPos::show(std::ostream & str) @@ -143,7 +191,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos) if (!pos) str << "undefined position"; else - str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % pos.file % pos.line % pos.column).str(); + str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str(); return str; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 0eaa362fd68b..121dc58f25f7 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -280,7 +280,7 @@ struct ExprOpNot : Expr Expr##name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ void show(std::ostream & str) \ { \ - str << *e1 << " " s " " << *e2; \ + str << "(" << *e1 << " " s " " << *e2 << ")"; \ } \ void bindVars(const StaticEnv & env) \ { \ diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index dcb270b862a3..7d877cd67862 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -53,10 +53,6 @@ namespace nix { #include "parser-tab.hh" #include "lexer-tab.hh" -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - YY_DECL; using namespace nix; @@ -630,7 +626,7 @@ Path EvalState::findFile(const string & path) } -Path EvalState::findFile(SearchPath & searchPath, const string & path) +Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) { foreach (SearchPath::iterator, i, searchPath) { Path res; @@ -645,7 +641,11 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path) } if (pathExists(res)) return canonPath(res); } - throw ThrownError(format("file ‘%1%’ was not found in the Nix search path (add it using $NIX_PATH or -I)") % path); + format f = format( + "file ‘%1%’ was not found in the Nix search path (add it using $NIX_PATH or -I)" + + string(pos ? ", at %2%" : "")); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + throw ThrownError(f % path % pos); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a7884cb85521..6c30e6d549ec 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,7 +35,7 @@ std::pair<string, string> decodeContext(const string & s) size_t index = s.find("!", 1); return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); } else - return std::pair<string, string>(s, ""); + return std::pair<string, string>(s.at(0) == '/' ? s: string(s, 1), ""); } @@ -51,7 +51,7 @@ void realiseContext(const PathSet & context) assert(isStorePath(ctx)); if (!store->isValidPath(ctx)) throw InvalidPathError(ctx); - if (isDerivation(ctx)) + if (!decoded.second.empty() && isDerivation(ctx)) drvs.insert(decoded.first + "!" + decoded.second); } if (!drvs.empty()) { @@ -84,16 +84,19 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Derivation drv = readDerivation(path); Value & w = *state.allocValue(); state.mkAttrs(w, 2 + drv.outputs.size()); - mkString(*state.allocAttr(w, state.sDrvPath), path, singleton<PathSet>("=" + path)); - state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); + Value * v2 = state.allocAttr(w, state.sDrvPath); + mkString(*v2, path, singleton<PathSet>("=" + path)); + Value * outputsVal = + state.allocAttr(w, state.symbols.create("outputs")); + state.mkList(*outputsVal, drv.outputs.size()); unsigned int outputs_index = 0; - Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; - foreach (DerivationOutputs::iterator, i, drv.outputs) { - mkString(*state.allocAttr(w, state.symbols.create(i->first)), - i->second.path, singleton<PathSet>("!" + i->first + "!" + path)); - mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), - i->first); + for (const auto & o : drv.outputs) { + v2 = state.allocAttr(w, state.symbols.create(o.first)); + mkString(*v2, o.second.path, + singleton<PathSet>("!" + o.first + "!" + path)); + outputsVal->list.elems[outputs_index] = state.allocValue(); + mkString(*(outputsVal->list.elems[outputs_index++]), o.first); } w.attrs->sort(); Value fun; @@ -184,6 +187,9 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu case tPrimOpApp: t = "lambda"; break; + case tExternal: + t = args[0]->external->typeOf(); + break; default: abort(); } mkString(v, state.symbols.create(t)); @@ -411,32 +417,6 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value } -#if HAVE_BOEHMGC -void canaryFinalizer(GC_PTR obj, GC_PTR client_data) -{ - Value * v = (Value *) obj; - EvalState & state(* (EvalState *) client_data); - printMsg(lvlError, format("canary ‘%1%’ garbage-collected") % v->string.s); - auto i = state.gcCanaries.find(v); - assert(i != state.gcCanaries.end()); - state.gcCanaries.erase(i); -} -#endif - - -void prim_gcCanary(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - string s = state.forceStringNoCtx(*args[0], pos); - state.mkList(v, 1); - Value * canary = v.list.elems[0] = state.allocValue(); -#if HAVE_BOEHMGC - state.gcCanaries.insert(canary); - GC_register_finalizer(canary, canaryFinalizer, &state, 0, 0); -#endif - mkString(*canary, s); -} - - void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v) { /* We're not forcing the argument on purpose. */ @@ -750,8 +730,12 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string ‘%1%’ cannot refer to other paths, at %2%") % path % pos); + try { + realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot read ‘%1%’, since path ‘%2%’ is not valid, at %3%") + % path % e.path % pos); + } mkString(v, readFile(path).c_str()); } @@ -791,7 +775,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va % path % e.path % pos); } - mkPath(v, state.findFile(searchPath, path).c_str()); + mkPath(v, state.findFile(searchPath, path, pos).c_str()); } /* Read a directory (without . or ..) */ @@ -812,7 +796,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val for (auto & ent : entries) { Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) - ent.type = getFileType(path); + ent.type = getFileType(path + "/" + ent.name); mkStringNoCopy(*ent_val, ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : @@ -867,7 +851,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu { PathSet context; string name = state.forceStringNoCtx(*args[0], pos); - string contents = state.forceString(*args[1], context); + string contents = state.forceString(*args[1], context, pos); PathSet refs; @@ -1424,10 +1408,37 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, throw Error(format("unknown hash type ‘%1%’, at %2%") % type % pos); PathSet context; // discarded - string s = state.forceString(*args[1], context); + string s = state.forceString(*args[1], context, pos); mkString(v, printHash(hashString(ht, s)), context); -}; +} + + +/* Match a regular expression against a string and return either + ‘null’ or a list containing substring matches. */ +static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + Regex regex(state.forceStringNoCtx(*args[0], pos), true); + + PathSet context; + string s = state.forceString(*args[1], context, pos); + + Regex::Subs subs; + if (!regex.matches(s, subs)) { + mkNull(v); + return; + } + + unsigned int len = subs.empty() ? 0 : subs.rbegin()->first + 1; + state.mkList(v, len); + for (unsigned int n = 0; n < len; ++n) { + auto i = subs.find(n); + if (i == subs.end()) + mkNull(*(v.list.elems[n] = state.allocValue())); + else + mkString(*(v.list.elems[n] = state.allocValue()), i->second); + } +} /************************************************************* @@ -1523,7 +1534,6 @@ void EvalState::createBaseEnv() // Debugging addPrimOp("__trace", 2, prim_trace); - addPrimOp("__gcCanary", 1, prim_gcCanary); addPrimOp("__valueSize", 1, prim_valueSize); // Paths @@ -1581,6 +1591,7 @@ void EvalState::createBaseEnv() addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); + addPrimOp("__match", 2, prim_match); // Versions addPrimOp("__parseDrvName", 1, prim_parseDrvName); diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 140662b51501..2fdf820211c8 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -58,12 +58,6 @@ public: friend std::ostream & operator << (std::ostream & str, const Symbol & sym); }; -inline std::ostream & operator << (std::ostream & str, const Symbol & sym) -{ - str << *sym.s; - return str; -} - class SymbolTable { private: diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index d1ec9b566d66..cdb71341875a 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -80,10 +80,21 @@ void printValueAsJSON(EvalState & state, bool strict, break; } + case tExternal: + v.external->printValueAsJSON(state, strict, str, context); + break; + default: throw TypeError(format("cannot convert %1% to JSON") % showType(v)); } } +void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, + std::ostream & str, PathSet & context) const +{ + throw TypeError(format("cannot convert %1% to JSON") % showType()); +} + + } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 3934a83eec25..bbbb7039bf70 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -144,12 +144,23 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } + case tExternal: + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + break; + default: doc.writeEmptyElement("unevaluated"); } } +void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const +{ + doc.writeEmptyElement("unevaluated"); +} + + void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context) { diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 5f18f629e632..c06b5a6d1153 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -19,6 +19,7 @@ typedef enum { tBlackhole, tPrimOp, tPrimOpApp, + tExternal, } ValueType; @@ -29,10 +30,58 @@ struct ExprLambda; struct PrimOp; struct PrimOp; class Symbol; +struct Pos; +class EvalState; +class XMLWriter; typedef long NixInt; +/* External values must descend from ExternalValueBase, so that + * type-agnostic nix functions (e.g. showType) can be implemented + */ +class ExternalValueBase +{ + friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); + protected: + /* Print out the value */ + virtual std::ostream & print(std::ostream & str) const = 0; + + public: + /* Return a simple string describing the type */ + virtual string showType() const = 0; + + /* Return a string to be used in builtins.typeOf */ + virtual string typeOf() const = 0; + + /* How much space does this value take up */ + virtual size_t valueSize(std::set<const void *> & seen) const = 0; + + /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + * error + */ + virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + + /* Compare to another value of the same type. Defaults to uncomparable, + * i.e. always false. + */ + virtual bool operator==(const ExternalValueBase & b) const; + + /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + virtual void printValueAsJSON(EvalState & state, bool strict, + std::ostream & str, PathSet & context) const; + + /* Print the value as XML. Defaults to unevaluated */ + virtual void printValueAsXML(EvalState & state, bool strict, bool location, + XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + + virtual ~ExternalValueBase() + { + }; +}; + +std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); + struct Value { @@ -88,6 +137,7 @@ struct Value struct { Value * left, * right; } primOpApp; + ExternalValueBase * external; }; }; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index c4b5c210d5be..c83e997b2307 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -11,11 +11,14 @@ #include <exception> #include <algorithm> +#include <cstdlib> #include <sys/time.h> #include <sys/stat.h> #include <unistd.h> #include <signal.h> +extern char * * environ; + namespace nix { @@ -289,8 +292,10 @@ int handleExceptions(const string & programName, std::function<void()> fun) RunPager::RunPager() { - string pager = getEnv("PAGER"); - if (!isatty(STDOUT_FILENO) || pager.empty()) return; + if (!isatty(STDOUT_FILENO)) return; + char * pager = getenv("NIX_PAGER"); + if (!pager) pager = getenv("PAGER"); + if (pager && ((string) pager == "" || (string) pager == "cat")) return; /* Ignore SIGINT. The pager will handle it (and we'll get SIGPIPE). */ @@ -310,7 +315,11 @@ RunPager::RunPager() throw SysError("dupping stdin"); if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1); - execl("/bin/sh", "sh", "-c", pager.c_str(), NULL); + if (pager) + execl("/bin/sh", "sh", "-c", pager, NULL); + execlp("pager", "pager", NULL); + execlp("less", "less", NULL); + execlp("more", "more", NULL); throw SysError(format("executing ‘%1%’") % pager); }); @@ -321,10 +330,14 @@ RunPager::RunPager() RunPager::~RunPager() { - if (pid != -1) { - std::cout.flush(); - close(STDOUT_FILENO); - pid.wait(true); + try { + if (pid != -1) { + std::cout.flush(); + close(STDOUT_FILENO); + pid.wait(true); + } + } catch (...) { + ignoreException(); } } diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index 64df95547e0b..41b617d98be2 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -18,7 +18,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) the stack pointer. Unfortunately, getting the stack pointer is not portable. */ bool haveSP = true; - char * sp; + char * sp = 0; #if defined(__x86_64__) && defined(REG_RSP) sp = (char *) ((ucontext *) ctx)->uc_mcontext.gregs[REG_RSP]; #elif defined(REG_ESP) @@ -32,7 +32,7 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) if (diff < 0) diff = -diff; if (diff < 4096) { char msg[] = "error: stack overflow (possible infinite recursion)\n"; - write(2, msg, strlen(msg)); + (void) write(2, msg, strlen(msg)); _exit(1); // maybe abort instead? } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3c9db5f7a0e4..e0398e2fb4a3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -50,6 +50,15 @@ #define CHROOT_ENABLED HAVE_CHROOT && HAVE_UNSHARE && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + #define SANDBOX_ENABLED 1 + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library/Frameworks /usr/lib /dev /bin/sh" +#else + #define SANDBOX_ENABLED 0 + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/bin" "/usr/bin" +#endif + #if CHROOT_ENABLED #include <sys/socket.h> #include <sys/ioctl.h> @@ -84,8 +93,12 @@ class Goal; typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; +struct CompareGoalPtrs { + bool operator() (const GoalPtr & a, const GoalPtr & b); +}; + /* Set of goals. */ -typedef set<GoalPtr> Goals; +typedef set<GoalPtr, CompareGoalPtrs> Goals; typedef list<WeakGoalPtr> WeakGoals; /* A map of paths to goals (and the other way around). */ @@ -172,11 +185,20 @@ public: (important!), etc. */ virtual void cancel(bool timeout) = 0; + virtual string key() = 0; + protected: void amDone(ExitCode result); }; +bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) { + string s1 = a->key(); + string s2 = b->key(); + return s1 < s2; +} + + /* A mapping used to remember for each child process to what goal it belongs, and file descriptors for receiving log data and output path creation commands. */ @@ -303,6 +325,7 @@ public: void addToWeakGoals(WeakGoals & goals, GoalPtr p) { // FIXME: necessary? + // FIXME: O(n) foreach (WeakGoals::iterator, i, goals) if (i->lock() == p) return; goals.push_back(p); @@ -401,18 +424,6 @@ static void commonChildInit(Pipe & logPipe) } -/* Convert a string list to an array of char pointers. Careful: the - string list should outlive the array. */ -const char * * strings2CharPtrs(const Strings & ss) -{ - const char * * arr = new const char * [ss.size() + 1]; - const char * * p = arr; - foreach (Strings::const_iterator, i, ss) *p++ = i->c_str(); - *p = 0; - return arr; -} - - ////////////////////////////////////////////////////////////////////// @@ -767,6 +778,15 @@ public: void cancel(bool timeout); + string key() + { + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + storePathToName(drvPath) + "$" + drvPath; + } + void work(); Path getDrvPath() @@ -793,8 +813,8 @@ private: /* Start building a derivation. */ void startBuilder(); - /* Initialise the builder's process. */ - void initChild(); + /* Run the builder's process. */ + void runChild(); friend int childEntry(void *); @@ -851,13 +871,9 @@ DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { - killChild(); - deleteTmpDir(false); - closeLogFile(); - } catch (...) { - ignoreException(); - } + try { killChild(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreException(); } + try { closeLogFile(); } catch (...) { ignoreException(); } } @@ -928,6 +944,11 @@ void DerivationGoal::init() /* The first thing to do is to make sure that the derivation exists. If it doesn't, it may be created through a substitute. */ + if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + haveDerivation(); + return; + } + addWaitee(worker.makeSubstitutionGoal(drvPath)); state = &DerivationGoal::haveDerivation; @@ -1574,6 +1595,13 @@ void chmod_(const Path & path, mode_t mode) } +int childEntry(void * arg) +{ + ((DerivationGoal *) arg)->runChild(); + return 1; +} + + void DerivationGoal::startBuilder() { startNest(nest, lvlInfo, format( @@ -1717,21 +1745,6 @@ void DerivationGoal::startBuilder() /* Change ownership of the temporary build directory. */ if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1) throw SysError(format("cannot change ownership of ‘%1%’") % tmpDir); - - /* Check that the Nix store has the appropriate permissions, - i.e., owned by root and mode 1775 (sticky bit on so that - the builder can create its output but not mess with the - outputs of other processes). */ - struct stat st; - if (stat(settings.nixStore.c_str(), &st) == -1) - throw SysError(format("cannot stat ‘%1%’") % settings.nixStore); - if (!(st.st_mode & S_ISVTX) || - ((st.st_mode & S_IRWXG) != S_IRWXG) || - (st.st_gid != buildUser.getGID())) - throw Error(format( - "builder does not have write permission to ‘%2%’; " - "try ‘chgrp %1% %2%; chmod 1775 %2%’") - % buildUser.getGID() % settings.nixStore); } @@ -1748,6 +1761,47 @@ void DerivationGoal::startBuilder() if (get(drv.env, "__noChroot") == "1") useChroot = false; if (useChroot) { + /* Allow a user-configurable set of directories from the + host file system. */ + PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", string(DEFAULT_CHROOT_DIRS))); + PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string(""))); + dirs.insert(dirs2.begin(), dirs2.end()); + + for (auto & i : dirs) { + size_t p = i.find('='); + if (p == string::npos) + dirsInChroot[i] = i; + else + dirsInChroot[string(i, 0, p)] = string(i, p + 1); + } + dirsInChroot[tmpDir] = tmpDir; + + string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); + PathSet allowedPaths = tokenizeString<StringSet>(allowed); + + /* This works like the above, except on a per-derivation level */ + Strings impurePaths = tokenizeString<Strings>(get(drv.env, "__impureHostDeps")); + + for (auto & i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ + for (auto & a : allowedPaths) { + Path canonA = canonPath(a); + if (canonI == canonA || isInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) + throw Error(format("derivation '%1%' requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + + dirsInChroot[i] = i; + } + #if CHROOT_ENABLED /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store @@ -1789,20 +1843,6 @@ void DerivationGoal::startBuilder() /* Create /etc/hosts with localhost entry. */ writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); - /* Bind-mount a user-configurable set of directories from the - host file system. */ - PathSet dirs = tokenizeString<StringSet>(settings.get("build-chroot-dirs", string(DEFAULT_CHROOT_DIRS))); - PathSet dirs2 = tokenizeString<StringSet>(settings.get("build-extra-chroot-dirs", string(""))); - dirs.insert(dirs2.begin(), dirs2.end()); - for (auto & i : dirs) { - size_t p = i.find('='); - if (p == string::npos) - dirsInChroot[i] = i; - else - dirsInChroot[string(i, 0, p)] = string(i, p + 1); - } - dirsInChroot[tmpDir] = tmpDir; - /* Make the closure of the inputs available in the chroot, rather than the whole Nix store. This prevents any access to undeclared dependencies. Directories are bind-mounted, @@ -1846,6 +1886,9 @@ void DerivationGoal::startBuilder() foreach (DerivationOutputs::iterator, i, drv.outputs) dirsInChroot.erase(i->second.path); +#elif SANDBOX_ENABLED + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ #else throw Error("chroot builds are not supported on this platform"); #endif @@ -1890,9 +1933,61 @@ void DerivationGoal::startBuilder() builderOut.create(); /* Fork a child to build the package. */ - pid = startProcess([&]() { - initChild(); - }); +#if CHROOT_ENABLED + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + Pid helper = startProcess([&]() { + char stack[32 * 1024]; + pid_t child = clone(childEntry, stack + sizeof(stack) - 8, + CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD, this); + if (child == -1) throw SysError("cloning builder process"); + writeFull(builderOut.writeSide, int2String(child) + "\n"); + _exit(0); + }); + if (helper.wait(true) != 0) + throw Error("unable to start build process"); + pid_t tmp; + if (!string2Int<pid_t>(readLine(builderOut.readSide), tmp)) abort(); + pid = tmp; + } else +#endif + { + ProcessOptions options; + options.allowVfork = !buildUser.enabled(); + pid = startProcess([&]() { + runChild(); + }, options); + } /* parent */ pid.setSeparatePG(true); @@ -1908,11 +2003,10 @@ void DerivationGoal::startBuilder() printMsg(lvlError, format("@ build-started %1% - %2% %3%") % drvPath % drv.platform % logFile); } - } -void DerivationGoal::initChild() +void DerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -1924,36 +2018,6 @@ void DerivationGoal::initChild() #if CHROOT_ENABLED if (useChroot) { - /* Set up private namespaces for the build: - - - The PID namespace causes the build to start as PID 1. - Processes outside of the chroot are not visible to - those on the inside, but processes inside the chroot - are visible from the outside (though with different - PIDs). - - - The private mount namespace ensures that all the bind - mounts we do will only show up in this process and - its children, and will disappear automatically when - we're done. - - - The private network namespace ensures that the - builder cannot talk to the outside world (or vice - versa). It only has a private loopback interface. - - - The IPC namespace prevents the builder from - communicating with outside processes using SysV IPC - mechanisms (shared memory, message queues, - semaphores). It also ensures that all IPC objects - are destroyed when the builder exits. - - - The UTS namespace ensures that builders see a - hostname of localhost rather than the actual - hostname. - */ - if (unshare(CLONE_NEWNS | CLONE_NEWNET | CLONE_NEWIPC | CLONE_NEWUTS) == -1) - throw SysError("setting up private namespaces"); - /* Initialise the loopback interface. */ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); if (fd == -1) throw SysError("cannot open IP socket"); @@ -1968,9 +2032,11 @@ void DerivationGoal::initChild() /* Set the hostname etc. to fixed values. */ char hostname[] = "localhost"; - sethostname(hostname, sizeof(hostname)); + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); char domainname[] = "(none)"; // kernel default - setdomainname(domainname, sizeof(domainname)); + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); /* Make all filesystems private. This is necessary because subtrees may have been mounted as "shared" @@ -2032,8 +2098,7 @@ void DerivationGoal::initChild() throw SysError(format("bind mount from ‘%1%’ to ‘%2%’ failed") % source % target); } - /* Bind a new instance of procfs on /proc to reflect our - private PID namespace. */ + /* Bind a new instance of procfs on /proc. */ createDirs(chrootRootDir + "/proc"); if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); @@ -2084,7 +2149,7 @@ void DerivationGoal::initChild() if (drv.platform == "i686-linux" && (settings.thisSystem == "x86_64-linux" || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { - if (personality(PER_LINUX32_3GB) == -1) + if (personality(PER_LINUX32) == -1) throw SysError("cannot set i686-linux personality"); } @@ -2105,11 +2170,7 @@ void DerivationGoal::initChild() Strings envStrs; foreach (Environment::const_iterator, i, env) envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp)); - const char * * envArr = strings2CharPtrs(envStrs); - - Path program = drv.builder.c_str(); - std::vector<const char *> args; /* careful with c_str()! */ - string user; /* must be here for its c_str()! */ + auto envArr = stringsToCharPtrs(envStrs); /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop all root @@ -2118,8 +2179,6 @@ void DerivationGoal::initChild() setuid() when run as root sets the real, effective and saved UIDs. */ if (buildUser.enabled()) { - printMsg(lvlChatty, format("switching to user ‘%1%’") % buildUser.getUser()); - if (setgroups(0, 0) == -1) throw SysError("cannot clear the set of supplementary groups"); @@ -2135,29 +2194,146 @@ void DerivationGoal::initChild() } /* Fill in the arguments. */ - string builderBasename = baseNameOf(drv.builder); - args.push_back(builderBasename.c_str()); - foreach (Strings::iterator, i, drv.args) { - auto re = rewriteHashes(*i, rewritesToTmp); - auto cstr = new char[re.length()+1]; - std::strcpy(cstr, re.c_str()); - - args.push_back(cstr); + Strings args; + + const char *builder = "invalid"; + + string sandboxProfile; + if (useChroot && SANDBOX_ENABLED) { + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : dirsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } + + /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = settings.nixStore; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) + dirsInChroot[i] = i; + + + /* TODO: we should factor out the policy cleanly, so we don't have to repeat the constants every time... */ + sandboxProfile += "(version 1)\n"; + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.get("darwin-log-sandbox-violations", false)) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += "(allow file-read* file-write-data (literal \"/dev/null\"))\n"; + + sandboxProfile += "(allow file-read-metadata\n" + "\t(literal \"/var\")\n" + "\t(literal \"/tmp\")\n" + "\t(literal \"/etc\")\n" + "\t(literal \"/etc/nix\")\n" + "\t(literal \"/etc/nix/nix.conf\"))\n"; + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + + /* This is where our temp folders are and where most of the building will happen, so we want rwx on it. */ + sandboxProfile += (format("(allow file-read* file-write* process-exec (subpath \"%1%\") (subpath \"/private/tmp\"))\n") % globalTmpDir).str(); + + sandboxProfile += "(allow process-fork)\n"; + sandboxProfile += "(allow sysctl-read)\n"; + sandboxProfile += "(allow signal (target same-sandbox))\n"; + + /* Enables getpwuid (used by git and others) */ + sandboxProfile += "(allow mach-lookup (global-name \"com.apple.system.notification_center\") (global-name \"com.apple.system.opendirectoryd.libinfo\"))\n"; + + + /* Our rwx outputs */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : missingPaths) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); + } + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + Note that the sandbox profile allows file-write* even though it isn't seemingly necessary. First of all, nix's standard user permissioning + mechanism still prevents builders from writing to input directories, so no security/purity is lost. The reason we allow file-write* is that + denying it means the `access` syscall will return EPERM instead of EACCESS, which confuses a few programs that assume (understandably, since + it appears to be a violation of the POSIX spec) that `access` won't do that, and don't deal with it nicely if it does. The most notable of + these is the entire GHC Haskell ecosystem. */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : dirsInChroot) { + if (i.first != i.second) + throw SysError(format("can't map '%1%' to '%2%': mismatched impure paths not supported on darwin")); + + string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path ‘%1%’") % path); + if (S_ISDIR(st.st_mode)) + sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); + else + sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); + } + sandboxProfile += ")\n"; + + /* Our ancestry. N.B: this uses literal on folders, instead of subpath. Without that, + you open up the entire filesystem because you end up with (subpath "/") */ + sandboxProfile += "(allow file-read-metadata\n"; + for (auto & i : ancestry) { + sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); + } + sandboxProfile += ")\n"; + + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-p"); + args.push_back(sandboxProfile); + args.push_back(drv.builder); + } else { + builder = drv.builder.c_str(); + string builderBasename = baseNameOf(drv.builder); + args.push_back(builderBasename); } - args.push_back(0); + + foreach (Strings::iterator, i, drv.args) + args.push_back(rewriteHashes(*i, rewritesToTmp)); + auto argArr = stringsToCharPtrs(args); restoreSIGPIPE(); /* Indicate that we managed to set up the build environment. */ - writeToStderr("\n"); + writeFull(STDERR_FILENO, "\n"); + + /* This needs to be after that fateful '\n', and I didn't want to duplicate code */ + if (useChroot && SANDBOX_ENABLED) { + printMsg(lvlDebug, "Generated sandbox profile:"); + printMsg(lvlDebug, sandboxProfile); + } /* Execute the program. This should not return. */ - execve(program.c_str(), (char * *) &args[0], (char * *) envArr); + execve(builder, (char * *) &argArr[0], (char * *) &envArr[0]); throw SysError(format("executing ‘%1%’") % drv.builder); } catch (std::exception & e) { - writeToStderr("while setting up the build environment: " + string(e.what()) + "\n"); + writeFull(STDERR_FILENO, "while setting up the build environment: " + string(e.what()) + "\n"); _exit(1); } } @@ -2308,7 +2484,7 @@ void DerivationGoal::registerOutputs() if (buildMode == bmCheck) { ValidPathInfo info = worker.store.queryPathInfo(path); if (hash.first != info.hash) - throw Error(format("derivation ‘%2%’ may not be deterministic: hash mismatch in output ‘%1%’") % drvPath % path); + throw Error(format("derivation ‘%1%’ may not be deterministic: hash mismatch in output ‘%2%’") % drvPath % path); continue; } @@ -2470,7 +2646,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err); } else if (fdLogFile != -1) - writeFull(fdLogFile, (unsigned char *) data.data(), data.size()); + writeFull(fdLogFile, data); } if (hook && fd == hook->fromHook.readSide) @@ -2581,6 +2757,13 @@ public: void cancel(bool timeout); + string key() + { + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + storePathToName(storePath) + "$" + storePath; + } + void work(); /* The states. */ @@ -2773,7 +2956,7 @@ void SubstitutionGoal::tryToRun() args.push_back("--substitute"); args.push_back(storePath); args.push_back(destPath); - const char * * argArr = strings2CharPtrs(args); + auto argArr = stringsToCharPtrs(args); /* Fork the substitute program. */ pid = startProcess([&]() { @@ -2783,7 +2966,7 @@ void SubstitutionGoal::tryToRun() if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) throw SysError("cannot dup output pipe into stdout"); - execv(sub.c_str(), (char * *) argArr); + execv(sub.c_str(), (char * *) &argArr[0]); throw SysError(format("executing ‘%1%’") % sub); }); @@ -3091,15 +3274,19 @@ void Worker::run(const Goals & _topGoals) checkInterrupt(); - /* Call every wake goal. */ + /* Call every wake goal (in the ordering established by + CompareGoalPtrs). */ while (!awake.empty() && !topGoals.empty()) { - WeakGoals awake2(awake); + Goals awake2; + for (auto & i : awake) { + GoalPtr goal = i.lock(); + if (goal) awake2.insert(goal); + } awake.clear(); - foreach (WeakGoals::iterator, i, awake2) { + for (auto & goal : awake2) { checkInterrupt(); - GoalPtr goal = i->lock(); - if (goal) goal->work(); - if (topGoals.empty()) break; + goal->work(); + if (topGoals.empty()) break; // stuff may have been cancelled } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e9db711e7ca0..7959a592d987 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -145,11 +145,6 @@ Path addPermRoot(StoreAPI & store, const Path & _storePath, } -/* The file to which we write our temporary roots. */ -static Path fnTempRoots; -static AutoCloseFD fdTempRoots; - - void LocalStore::addTempRoot(const Path & path) { /* Create the temporary roots file for this process. */ @@ -196,7 +191,7 @@ void LocalStore::addTempRoot(const Path & path) lockFile(fdTempRoots, ltWrite, true); string s = path + '\0'; - writeFull(fdTempRoots, (const unsigned char *) s.data(), s.size()); + writeFull(fdTempRoots, s); /* Downgrade to a read lock. */ debug(format("downgrading to read lock on ‘%1%’") % fnTempRoots); @@ -204,27 +199,6 @@ void LocalStore::addTempRoot(const Path & path) } -void removeTempRoots() -{ - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); - } -} - - -/* Automatically clean up the temporary roots file when we exit. */ -struct RemoveTempRoots -{ - ~RemoveTempRoots() - { - removeTempRoots(); - } -}; - -static RemoveTempRoots autoRemoveTempRoots __attribute__((unused)); - - typedef std::shared_ptr<AutoCloseFD> FDPtr; typedef list<FDPtr> FDs; @@ -257,7 +231,7 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds) if (lockFile(*fd, ltWrite, false)) { printMsg(lvlError, format("removing stale temporary roots file ‘%1%’") % path); unlink(path.c_str()); - writeFull(*fd, (const unsigned char *) "d", 1); + writeFull(*fd, "d"); continue; } @@ -355,7 +329,8 @@ Roots LocalStore::findRoots() /* Process direct roots in {gcroots,manifests,profiles}. */ nix::findRoots(*this, settings.nixStateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); - nix::findRoots(*this, settings.nixStateDir + "/manifests", DT_UNKNOWN, roots); + if (pathExists(settings.nixStateDir + "/manifests")) + nix::findRoots(*this, settings.nixStateDir + "/manifests", DT_UNKNOWN, roots); nix::findRoots(*this, settings.nixStateDir + "/profiles", DT_UNKNOWN, roots); return roots; @@ -749,7 +724,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } /* While we're at it, vacuum the database. */ - if (options.action == GCOptions::gcDeleteDead) vacuumDB(); + //if (options.action == GCOptions::gcDeleteDead) vacuumDB(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f08c877fe3d7..bc792baf296b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -256,20 +256,23 @@ LocalStore::LocalStore(bool reserveSpace) if (chmod(perUserDir.c_str(), 01777) == -1) throw SysError(format("could not set permissions on ‘%1%’ to 1777") % perUserDir); + mode_t perm = 01735; + struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); if (!gr) - throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist") + printMsg(lvlError, format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist") % settings.buildUsersGroup); - - struct stat st; - if (stat(settings.nixStore.c_str(), &st)) - throw SysError(format("getting attributes of path ‘%1%’") % settings.nixStore); - - if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != 01775) { - if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path ‘%1%’") % settings.nixStore); - if (chmod(settings.nixStore.c_str(), 01775) == -1) - throw SysError(format("changing permissions on path ‘%1%’") % settings.nixStore); + else { + struct stat st; + if (stat(settings.nixStore.c_str(), &st)) + throw SysError(format("getting attributes of path ‘%1%’") % settings.nixStore); + + if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { + if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1) + throw SysError(format("changing ownership of path ‘%1%’") % settings.nixStore); + if (chmod(settings.nixStore.c_str(), perm) == -1) + throw SysError(format("changing permissions on path ‘%1%’") % settings.nixStore); + } } } @@ -358,7 +361,17 @@ LocalStore::~LocalStore() i->second.to.close(); i->second.from.close(); i->second.error.close(); - i->second.pid.wait(true); + if (i->second.pid != -1) + i->second.pid.wait(true); + } + } catch (...) { + ignoreException(); + } + + try { + if (fdTempRoots != -1) { + fdTempRoots.close(); + unlink(fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -489,7 +502,7 @@ void LocalStore::makeStoreWritable() if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); - if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1) + if (mount(0, settings.nixStore.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError(format("remounting %1% writable") % settings.nixStore); } #endif diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index dccdba533a0c..e0aabdba420d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -244,6 +244,10 @@ private: bool didSetSubstituterEnv; + /* The file to which we write our temporary roots. */ + Path fnTempRoots; + AutoCloseFD fdTempRoots; + int getSchema(); void openDB(bool create); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index dd18d66fa008..55c252b9b2e3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "globals.hh" +#include <cstdlib> #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index f26684afacb9..9db37e8f9aaa 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -33,7 +33,7 @@ void deleteLockFile(const Path & path, int fd) other processes waiting on this lock that the lock is stale (deleted). */ unlink(path.c_str()); - writeFull(fd, (const unsigned char *) "d", 1); + writeFull(fd, "d"); /* Note that the result of unlink() is ignored; removing the lock file is an optimisation, not a necessity. */ } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a0e9f22410f7..d08913246321 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -10,6 +10,7 @@ #include <sys/stat.h> #include <sys/socket.h> #include <sys/un.h> +#include <errno.h> #include <fcntl.h> #include <iostream> @@ -87,8 +88,7 @@ void RemoteStore::openConnection(bool reserveSpace) processStderr(); } catch (Error & e) { - throw Error(format("cannot start worker (%1%)") - % e.msg()); + throw Error(format("cannot start daemon worker: %1%") % e.msg()); } setOptions(); @@ -110,7 +110,7 @@ void RemoteStore::connectToDaemon() applications... */ AutoCloseFD fdPrevDir = open(".", O_RDONLY); if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - chdir(dirOf(socketPath).c_str()); + if (chdir(dirOf(socketPath).c_str()) == -1) throw SysError(format("couldn't change to directory of ‘%1%’") % socketPath); Path socketPathRel = "./" + baseNameOf(socketPath); struct sockaddr_un addr; diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 72d23fb6934c..6f01ccd91a43 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -3,6 +3,7 @@ #include <thread> #include <atomic> +#include <cstdlib> #include <poll.h> #include <sys/types.h> #include <unistd.h> diff --git a/src/libutil/regex.cc b/src/libutil/regex.cc index 36c8458cee08..84274b3e1da9 100644 --- a/src/libutil/regex.cc +++ b/src/libutil/regex.cc @@ -1,13 +1,16 @@ #include "regex.hh" #include "types.hh" +#include <algorithm> + namespace nix { -Regex::Regex(const string & pattern) +Regex::Regex(const string & pattern, bool subs) { /* Patterns must match the entire string. */ - int err = regcomp(&preg, ("^(" + pattern + ")$").c_str(), REG_NOSUB | REG_EXTENDED); - if (err) throw Error(format("compiling pattern ‘%1%’: %2%") % pattern % showError(err)); + int err = regcomp(&preg, ("^(" + pattern + ")$").c_str(), (subs ? 0 : REG_NOSUB) | REG_EXTENDED); + if (err) throw RegexError(format("compiling pattern ‘%1%’: %2%") % pattern % showError(err)); + nrParens = subs ? std::count(pattern.begin(), pattern.end(), '(') : 0; } Regex::~Regex() @@ -23,6 +26,20 @@ bool Regex::matches(const string & s) throw Error(format("matching string ‘%1%’: %2%") % s % showError(err)); } +bool Regex::matches(const string & s, Subs & subs) +{ + regmatch_t pmatch[nrParens + 2]; + int err = regexec(&preg, s.c_str(), nrParens + 2, pmatch, 0); + if (err == 0) { + for (unsigned int n = 2; n < nrParens + 2; ++n) + if (pmatch[n].rm_eo != -1) + subs[n - 2] = string(s, pmatch[n].rm_so, pmatch[n].rm_eo - pmatch[n].rm_so); + return true; + } + else if (err == REG_NOMATCH) return false; + throw Error(format("matching string ‘%1%’: %2%") % s % showError(err)); +} + string Regex::showError(int err) { char buf[256]; diff --git a/src/libutil/regex.hh b/src/libutil/regex.hh index aa012b721cb7..53e31f4edc4a 100644 --- a/src/libutil/regex.hh +++ b/src/libutil/regex.hh @@ -5,16 +5,23 @@ #include <sys/types.h> #include <regex.h> +#include <map> + namespace nix { +MakeError(RegexError, Error) + class Regex { public: - Regex(const string & pattern); + Regex(const string & pattern, bool subs = false); ~Regex(); bool matches(const string & s); + typedef std::map<unsigned int, string> Subs; + bool matches(const string & s, Subs & subs); private: + unsigned nrParens; regex_t preg; string showError(int err); }; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 906a959e3079..160884ee1ad7 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -8,6 +8,15 @@ #include <boost/format.hpp> +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + namespace nix { @@ -39,8 +48,12 @@ protected: public: unsigned int status; // exit status BaseError(const FormatOrString & fs, unsigned int status = 1); +#ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } +#else + const char * what() const noexcept { return err.c_str(); } +#endif const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 99d2b1e0adce..0d903f2f0d43 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -193,8 +193,12 @@ Path readLink(const Path & path) if (!S_ISLNK(st.st_mode)) throw Error(format("‘%1%’ is not a symlink") % path); char buf[st.st_size]; - if (readlink(path.c_str(), buf, st.st_size) != st.st_size) + ssize_t rlsize = readlink(path.c_str(), buf, st.st_size); + if (rlsize == -1) throw SysError(format("reading symbolic link ‘%1%’") % path); + else if (rlsize > st.st_size) + throw Error(format("symbolic link ‘%1%’ size overflow %2% > %3%") + % path % rlsize % st.st_size); return string(buf, st.st_size); } @@ -265,7 +269,7 @@ void writeFile(const Path & path, const string & s) AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); if (fd == -1) throw SysError(format("opening file ‘%1%’") % path); - writeFull(fd, (unsigned char *) s.data(), s.size()); + writeFull(fd, s); } @@ -292,7 +296,7 @@ string readLine(int fd) void writeLine(int fd, string s) { s += '\n'; - writeFull(fd, (const unsigned char *) s.data(), s.size()); + writeFull(fd, s); } @@ -483,18 +487,13 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs) } -static void defaultWriteToStderr(const unsigned char * buf, size_t count) -{ - writeFull(STDERR_FILENO, buf, count); -} - - void writeToStderr(const string & s) { try { - auto p = _writeToStderr; - if (!p) p = defaultWriteToStderr; - p((const unsigned char *) s.data(), s.size()); + if (_writeToStderr) + _writeToStderr((const unsigned char *) s.data(), s.size()); + else + writeFull(STDERR_FILENO, s); } catch (SysError & e) { /* Ignore failing writes to stderr if we're in an exception handler, otherwise throw an exception. We need to ignore @@ -506,7 +505,7 @@ void writeToStderr(const string & s) } -void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr; +void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; void readFull(int fd, unsigned char * buf, size_t count) @@ -540,6 +539,12 @@ void writeFull(int fd, const unsigned char * buf, size_t count) } +void writeFull(int fd, const string & s) +{ + writeFull(fd, (const unsigned char *) s.data(), s.size()); +} + + string drainFD(int fd) { string result; @@ -825,6 +830,9 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ + ProcessOptions options; + options.allowVfork = false; + Pid pid = startProcess([&]() { if (setuid(uid) == -1) @@ -847,7 +855,7 @@ void killUser(uid_t uid) } _exit(0); - }); + }, options); int status = pid.wait(true); if (status != 0) @@ -863,43 +871,64 @@ void killUser(uid_t uid) ////////////////////////////////////////////////////////////////////// -pid_t startProcess(std::function<void()> fun, - bool dieWithParent, const string & errorPrefix) +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function<void()> fun) { +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else pid_t pid = fork(); - if (pid == -1) throw SysError("unable to fork"); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} - if (pid == 0) { - _writeToStderr = 0; + +pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) +{ + auto wrapper = [&]() { + if (!options.allowVfork) _writeToStderr = 0; try { #if __linux__ - if (dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) throw SysError("setting death signal"); #endif restoreAffinity(); fun(); } catch (std::exception & e) { try { - std::cerr << errorPrefix << e.what() << "\n"; + std::cerr << options.errorPrefix << e.what() << "\n"; } catch (...) { } } catch (...) { } - _exit(1); - } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); return pid; } +std::vector<const char *> stringsToCharPtrs(const Strings & ss) +{ + std::vector<const char *> res; + for (auto & s : ss) res.push_back(s.c_str()); + res.push_back(0); + return res; +} + + string runProgram(Path program, bool searchPath, const Strings & args) { checkInterrupt(); - std::vector<const char *> cargs; /* careful with c_str()! */ - cargs.push_back(program.c_str()); - for (Strings::const_iterator i = args.begin(); i != args.end(); ++i) - cargs.push_back(i->c_str()); - cargs.push_back(0); - /* Create a pipe. */ Pipe pipe; pipe.create(); @@ -909,6 +938,10 @@ string runProgram(Path program, bool searchPath, const Strings & args) if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + Strings args_(args); + args_.push_front(program); + auto cargs = stringsToCharPtrs(args_); + if (searchPath) execvp(program.c_str(), (char * *) &cargs[0]); else diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b35e02dceb60..186ee71f45d0 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -171,6 +171,7 @@ extern void (*_writeToStderr) (const unsigned char * buf, size_t count); requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); void writeFull(int fd, const unsigned char * buf, size_t count); +void writeFull(int fd, const string & s); MakeError(EndOfFile, Error) @@ -269,8 +270,16 @@ void killUser(uid_t uid); /* Fork a process that runs the given function, and return the child pid to the caller. */ -pid_t startProcess(std::function<void()> fun, bool dieWithParent = true, - const string & errorPrefix = "error: "); +struct ProcessOptions +{ + string errorPrefix; + bool dieWithParent; + bool runExitHandlers; + bool allowVfork; + ProcessOptions() : errorPrefix("error: "), dieWithParent(true), runExitHandlers(false), allowVfork(true) { }; +}; + +pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); /* Run a program and return its stdout in a string (i.e., like the @@ -280,6 +289,11 @@ string runProgram(Path program, bool searchPath = false, MakeError(ExecError, Error) +/* Convert a list of strings to a null-terminated vector of char + *'s. The result must not be accessed beyond the lifetime of the + list of strings. */ +std::vector<const char *> stringsToCharPtrs(const Strings & ss); + /* Close all file descriptors except stdin, stdout, stderr, and those listed in the given set. Good practice in child processes. */ void closeMostFDs(const set<int> & exceptions); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8ec54e4580dc..bed7de0859a3 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -22,6 +22,10 @@ #include <pwd.h> #include <grp.h> +#if __APPLE__ || __FreeBSD__ +#include <sys/ucred.h> +#endif + using namespace nix; @@ -509,11 +513,11 @@ static void performOp(bool trusted, unsigned int clientVersion, } case wopOptimiseStore: - startWork(); - store->optimiseStore(); - stopWork(); - writeInt(1, to); - break; + startWork(); + store->optimiseStore(); + stopWork(); + writeInt(1, to); + break; default: throw Error(format("invalid operation %1%") % op); @@ -565,7 +569,7 @@ static void processConnection(bool trusted) to.flush(); } catch (Error & e) { - stopWork(false, e.msg()); + stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); to.flush(); return; } @@ -606,6 +610,8 @@ static void processConnection(bool trusted) assert(!canSendStderr); }; + canSendStderr = false; + _isInterrupted = false; printMsg(lvlDebug, format("%1% operations") % opCount); } @@ -649,12 +655,51 @@ bool matchUser(const string & user, const string & group, const Strings & users) } +struct PeerInfo +{ + bool pidKnown; + pid_t pid; + bool uidKnown; + uid_t uid; + bool gidKnown; + gid_t gid; +}; + + +/* Get the identity of the caller, if possible. */ +static PeerInfo getPeerInfo(int remote) +{ + PeerInfo peer = { false, 0, false, 0, false, 0 }; + +#if defined(SO_PEERCRED) + + ucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { true, cred.pid, true, cred.uid, true, cred.gid }; + +#elif defined(LOCAL_PEERCRED) + + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { false, 0, true, cred.cr_uid, false, 0 }; + +#endif + + return peer; +} + + #define SD_LISTEN_FDS_START 3 static void daemonLoop(char * * argv) { - chdir("/"); + if (chdir("/") == -1) + throw SysError("cannot change current directory"); /* Get rid of children automatically; don't let them become zombies. */ @@ -684,7 +729,8 @@ static void daemonLoop(char * * argv) /* Urgh, sockaddr_un allows path names of only 108 characters. So chdir to the socket directory so that we can pass a relative path name. */ - chdir(dirOf(socketPath).c_str()); + if (chdir(dirOf(socketPath).c_str()) == -1) + throw SysError("cannot change current directory"); Path socketPathRel = "./" + baseNameOf(socketPath); struct sockaddr_un addr; @@ -704,7 +750,8 @@ static void daemonLoop(char * * argv) if (res == -1) throw SysError(format("cannot bind to socket ‘%1%’") % socketPath); - chdir("/"); /* back to the root */ + if (chdir("/") == -1) /* back to the root */ + throw SysError("cannot change current directory"); if (listen(fdSocket, 5) == -1) throw SysError(format("cannot listen on socket ‘%1%’") % socketPath); @@ -735,22 +782,13 @@ static void daemonLoop(char * * argv) closeOnExec(remote); bool trusted = false; - pid_t clientPid = -1; - -#if defined(SO_PEERCRED) - /* Get the identity of the caller, if possible. */ - ucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) - throw SysError("getting peer credentials"); + PeerInfo peer = getPeerInfo(remote); - clientPid = cred.pid; + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + string user = pw ? pw->pw_name : int2String(peer.uid); - struct passwd * pw = getpwuid(cred.uid); - string user = pw ? pw->pw_name : int2String(cred.uid); - - struct group * gr = getgrgid(cred.gid); - string group = gr ? gr->gr_name : int2String(cred.gid); + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + string group = gr ? gr->gr_name : int2String(peer.gid); Strings trustedUsers = settings.get("trusted-users", Strings({"root"})); Strings allowedUsers = settings.get("allowed-users", Strings({"*"})); @@ -761,11 +799,16 @@ static void daemonLoop(char * * argv) if (!trusted && !matchUser(user, group, allowedUsers)) throw Error(format("user ‘%1%’ is not allowed to connect to the Nix daemon") % user); - printMsg(lvlInfo, format((string) "accepted connection from pid %1%, user %2%" - + (trusted ? " (trusted)" : "")) % clientPid % user); -#endif + printMsg(lvlInfo, format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) + % (peer.pidKnown ? int2String(peer.pid) : "<unknown>") + % (peer.uidKnown ? user : "<unknown>")); /* Fork a child to handle the connection. */ + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; startProcess([&]() { fdSocket.close(); @@ -777,8 +820,8 @@ static void daemonLoop(char * * argv) setSigChldAction(false); /* For debugging, stuff the pid into argv[1]. */ - if (clientPid != -1 && argv[1]) { - string processName = int2String(clientPid); + if (peer.pidKnown && argv[1]) { + string processName = int2String(peer.pid); strncpy(argv[1], processName.c_str(), strlen(argv[1])); } @@ -787,8 +830,8 @@ static void daemonLoop(char * * argv) to.fd = remote; processConnection(trusted); - _exit(0); - }, false, "unexpected Nix daemon error: "); + exit(0); + }, options); } catch (Interrupted & e) { throw; @@ -799,15 +842,6 @@ static void daemonLoop(char * * argv) } -void run(Strings args) -{ - for (Strings::iterator i = args.begin(); i != args.end(); ) { - string arg = *i++; - } - -} - - int main(int argc, char * * argv) { return handleExceptions(argv[0], [&]() { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f5e8ee08c42f..f3c8d3ba8953 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -132,6 +132,8 @@ static void getAllExprs(EvalState & state, Value & vArg(*state.allocValue()); state.getBuiltin("import", vFun); mkString(vArg, path2); + if (v.attrs->size() == v.attrs->capacity()) + throw Error(format("too many Nix expressions in directory ‘%1%’") % path); mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); } else if (S_ISDIR(st.st_mode)) @@ -160,7 +162,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) ~/.nix-defexpr directory that includes some system-wide directory). */ if (S_ISDIR(st.st_mode)) { - state.mkAttrs(v, 16); + state.mkAttrs(v, 1024); state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); StringSet attrs; getAllExprs(state, path, attrs, v); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 9a6e178f514e..e7214e657bd5 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -86,8 +86,6 @@ void processExpr(EvalState & state, const Strings & attrPaths, } } } - - state.printCanaries(); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c91bca97d534..87bc8c379de5 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -488,7 +488,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) if (pathExists(logPath)) { /* !!! Make this run in O(1) memory. */ string log = readFile(logPath); - writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); + writeFull(STDOUT_FILENO, log); found = true; break; } diff --git a/tests/lang/eval-okay-callable-attrs.exp b/tests/lang/eval-okay-callable-attrs.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/lang/eval-okay-callable-attrs.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-callable-attrs.nix b/tests/lang/eval-okay-callable-attrs.nix new file mode 100644 index 000000000000..310a030df004 --- /dev/null +++ b/tests/lang/eval-okay-callable-attrs.nix @@ -0,0 +1 @@ +({ __functor = self: x: self.foo && x; foo = false; } // { foo = true; }) true diff --git a/tests/lang/eval-okay-regex-match.exp b/tests/lang/eval-okay-regex-match.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/lang/eval-okay-regex-match.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/lang/eval-okay-regex-match.nix new file mode 100644 index 000000000000..ae6501532d11 --- /dev/null +++ b/tests/lang/eval-okay-regex-match.nix @@ -0,0 +1,26 @@ +with builtins; + +let + + matches = pat: s: match pat s != null; + + splitFN = match "((.*)/)?([^/]*)\\.(nix|cc)"; + +in + +assert matches "foobar" "foobar"; +assert matches "fo*" "f"; +assert !matches "fo+" "f"; +assert matches "fo*" "fo"; +assert matches "fo*" "foo"; +assert matches "fo+" "foo"; +assert matches "fo{1,2}" "foo"; +assert !matches "fo{1,2}" "fooo"; +assert !matches "fo*" "foobar"; + +assert match "(.*)\\.nix" "foobar.nix" == [ "foobar" ]; + +assert splitFN "/path/to/foobar.nix" == [ "/path/to/" "/path/to" "foobar" "nix" ]; +assert splitFN "foobar.cc" == [ null null "foobar" "cc" ]; + +true diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 5e2688d1adcf..0f16026a428a 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -13,7 +13,7 @@ let { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; - environment.nix = nix; + nix.package = nix; }; # Trivial Nix expression to build remotely. @@ -25,7 +25,7 @@ let system = "i686-linux"; PATH = "''${utils}/bin"; builder = "''${utils}/bin/sh"; - args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 5" ]; + args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 10" ]; outputs = [ "out" "foo" ]; } ''; diff --git a/version b/version index 468437494697..872765e5f285 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.8 \ No newline at end of file +1.9 \ No newline at end of file |