diff options
132 files changed, 2507 insertions, 3945 deletions
diff --git a/.gitignore b/.gitignore index 0a9599378567..0f2f3ddeec14 100644 --- a/.gitignore +++ b/.gitignore @@ -13,9 +13,6 @@ perl/Makefile.config /corepkgs/config.nix -# /corepkgs/buildenv/ -/corepkgs/buildenv/builder.pl - # /corepkgs/channels/ /corepkgs/channels/unpack.sh @@ -72,9 +69,6 @@ perl/Makefile.config # /src/nix-channel/ /src/nix-channel/nix-channel -# /src/buildenv/ -/src/buildenv/buildenv - # /src/nix-build/ /src/nix-build/nix-build diff --git a/Makefile b/Makefile index c867823fc485..834f84b286bf 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ makefiles = \ local.mk \ - src/boost/format/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ src/libmain/local.mk \ @@ -13,7 +12,6 @@ makefiles = \ src/nix-collect-garbage/local.mk \ src/nix-copy-closure/local.mk \ src/nix-prefetch-url/local.mk \ - src/buildenv/local.mk \ src/resolve-system-dependencies/local.mk \ src/nix-channel/local.mk \ src/nix-build/local.mk \ @@ -27,7 +25,7 @@ makefiles = \ tests/local.mk \ tests/plugins/local.mk -GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include config.h +GLOBAL_CXXFLAGS += -g -Wall -include config.h -include Makefile.config diff --git a/configure.ac b/configure.ac index 54322d463ae7..c41a83c97646 100644 --- a/configure.ac +++ b/configure.ac @@ -62,7 +62,7 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_11 +AX_CXX_COMPILE_STDCXX_14 # Use 64-bit file system calls so that we can support files > 2 GiB. @@ -215,7 +215,7 @@ AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) + declare -a aws_version_tokens=($(printf '#include <aws/core/VersionConfig.h>\nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) fi diff --git a/corepkgs/buildenv.nix b/corepkgs/buildenv.nix index 5e7b40eaa0cb..0bac4c44b48a 100644 --- a/corepkgs/buildenv.nix +++ b/corepkgs/buildenv.nix @@ -1,11 +1,9 @@ -with import <nix/config.nix>; - { derivations, manifest }: derivation { name = "user-environment"; - system = builtins.currentSystem; - builder = nixLibexecDir + "/nix/buildenv"; + system = "builtin"; + builder = "builtin:buildenv"; inherit manifest; @@ -24,21 +22,4 @@ derivation { # Also don't bother substituting. allowSubstitutes = false; - - __sandboxProfile = '' - (allow sysctl-read) - (allow file-read* - (literal "/usr/lib/libSystem.dylib") - (literal "/usr/lib/libSystem.B.dylib") - (literal "/usr/lib/libobjc.A.dylib") - (literal "/usr/lib/libobjc.dylib") - (literal "/usr/lib/libauto.dylib") - (literal "/usr/lib/libc++abi.dylib") - (literal "/usr/lib/libc++.1.dylib") - (literal "/usr/lib/libDiagnosticMessagesClient.dylib") - (subpath "/usr/lib/system") - (subpath "/dev")) - ''; - - inherit chrootDeps; } diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 5c21a40a11e9..1865bb37c860 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -123,7 +123,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>auto-optimise-store</literal></term> + <varlistentry xml:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term> <listitem><para>If set to <literal>true</literal>, Nix automatically detects files in the store that have identical @@ -146,7 +146,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>builders-use-substitutes</literal></term> + <varlistentry xml:id="conf-builders-use-substitutes"><term><literal>builders-use-substitutes</literal></term> <listitem><para>If set to <literal>true</literal>, Nix will instruct remote build machines to use their own binary substitutes if available. In @@ -201,7 +201,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>compress-build-log</literal></term> + <varlistentry xml:id="conf-compress-build-log"><term><literal>compress-build-log</literal></term> <listitem><para>If set to <literal>true</literal> (the default), build logs written to <filename>/nix/var/log/nix/drvs</filename> @@ -254,7 +254,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>extra-platforms</literal></term> + <varlistentry xml:id="conf-extra-platforms"><term><literal>extra-platforms</literal></term> <listitem><para>Platforms other than the native one which this machine is capable of building for. This can be useful for @@ -273,7 +273,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>extra-substituters</literal></term> + <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term> <listitem><para>Additional binary caches appended to those specified in <option>substituters</option>. When used by @@ -284,7 +284,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>fallback</literal></term> + <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term> <listitem><para>If set to <literal>true</literal>, Nix will fall back to building from source if a binary substitute fails. This @@ -294,7 +294,7 @@ false</literal>.</para> </varlistentry> - <varlistentry><term><literal>fsync-metadata</literal></term> + <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term> <listitem><para>If set to <literal>true</literal>, changes to the Nix store metadata (in <filename>/nix/var/nix/db</filename>) are @@ -333,7 +333,7 @@ builtins.fetchurl { </varlistentry> - <varlistentry><term><literal>http-connections</literal></term> + <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term> <listitem><para>The maximum number of parallel TCP connections used to fetch files from binary caches and by other downloads. It @@ -342,7 +342,7 @@ builtins.fetchurl { </varlistentry> - <varlistentry><term><literal>keep-build-log</literal></term> + <varlistentry xml:id="conf-keep-build-log"><term><literal>keep-build-log</literal></term> <listitem><para>If set to <literal>true</literal> (the default), Nix will write the build log of a derivation (i.e. the standard @@ -371,7 +371,7 @@ builtins.fetchurl { </varlistentry> - <varlistentry><term><literal>keep-env-derivations</literal></term> + <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term> <listitem><para>If <literal>false</literal> (default), derivations are not stored in Nix user environments. That is, the derivation @@ -425,6 +425,12 @@ builtins.fetchurl { </varlistentry> + <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term> + + <listitem><para>This option defines after how many free bytes to stop collecting + garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem> + + </varlistentry> <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term> @@ -459,8 +465,48 @@ builtins.fetchurl { </varlistentry> + <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term> + + <listitem> + <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix + will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk. + A value of <literal>0</literal> (the default) means that this feature is disabled.</para> + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-narinfo-cache-negative-ttl"><term><literal>narinfo-cache-negative-ttl</literal></term> + + <listitem> - <varlistentry><term><literal>netrc-file</literal></term> + <para>The TTL in seconds for negative lookups. If a store path is + queried from a substituter but was not found, there will be a + negative lookup cached in the local disk cache database for the + specified duration.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-narinfo-cache-positive-ttl"><term><literal>narinfo-cache-positive-ttl</literal></term> + + <listitem> + + <para>The TTL in seconds for positive lookups. If a store path is + queried from a substituter, the result of the query will be cached + in the local disk cache database including some of the NAR + metadata. The default TTL is a month, setting a shorter TTL for + positive lookups can be useful for binary caches that have + frequent garbage collection, in which case having a more frequent + cache invalidation would prevent trying to pull the path again and + failing with a hash mismatch if the build isn't reproducible. + </para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term> <listitem><para>If set to an absolute path to a <filename>netrc</filename> file, Nix will use the HTTP authentication credentials in this file when @@ -514,7 +560,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term> <listitem> @@ -563,7 +608,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>require-sigs</literal></term> + <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term> <listitem><para>If set to <literal>true</literal> (the default), any non-content-addressed path added or copied to the Nix store @@ -592,7 +637,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>sandbox</literal></term> + <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term> <listitem><para>If set to <literal>true</literal>, builds will be performed in a <emphasis>sandboxed environment</emphasis>, i.e., @@ -663,7 +708,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>secret-key-files</literal></term> + <varlistentry xml:id="conf-secret-key-files"><term><literal>secret-key-files</literal></term> <listitem><para>A whitespace-separated list of files containing secret (private) keys. These are used to sign locally-built @@ -684,7 +729,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>substitute</literal></term> + <varlistentry xml:id="conf-substitute"><term><literal>substitute</literal></term> <listitem><para>If set to <literal>true</literal> (default), Nix will use binary substitutes if available. This option can be @@ -693,7 +738,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>substituters</literal></term> + <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by whitespace. The default is @@ -702,7 +747,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>system</literal></term> + <varlistentry xml:id="conf-system"><term><literal>system</literal></term> <listitem><para>This option specifies the canonical Nix system name of the current installation, such as @@ -743,7 +788,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>trusted-public-keys</literal></term> + <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term> <listitem><para>A whitespace-separated list of public keys. When paths are copied from another Nix store (such as a binary cache), @@ -754,7 +799,7 @@ password <replaceable>my-password</replaceable> </varlistentry> - <varlistentry><term><literal>trusted-substituters</literal></term> + <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by whitespace. These are not used by default, but can be enabled by @@ -791,7 +836,6 @@ password <replaceable>my-password</replaceable> </varlistentry> - </variablelist> </para> diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index 9acf44e52984..ff4021a765e0 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -31,7 +31,7 @@ <refsection><title>Description</title> -<para>A Nix channel is mechanism that allows you to automatically stay +<para>A Nix channel is a mechanism that allows you to automatically stay up-to-date with a set of pre-built Nix expressions. A Nix channel is just a URL that points to a place containing both a set of Nix expressions and a pointer to a binary cache. <phrase @@ -165,8 +165,8 @@ following files:</para> <varlistentry><term><filename>nixexprs.tar.xz</filename></term> <listitem><para>A tarball containing Nix expressions and files - referenced by them (such as build scripts and patches). At - top-level, the tarball should contain a single directory. That + referenced by them (such as build scripts and patches). At the + top level, the tarball should contain a single directory. That directory must contain a file <filename>default.nix</filename> that serves as the channelโs โentry pointโ.</para></listitem> @@ -175,7 +175,7 @@ following files:</para> <varlistentry><term><filename>binary-cache-url</filename></term> <listitem><para>A file containing the URL to a binary cache (such - as <uri>https://cache.nixos.org</uri>. Nix will automatically + as <uri>https://cache.nixos.org</uri>). Nix will automatically check this cache for pre-built binaries, if the user has sufficient rights to add binary caches. For instance, in a multi-user Nix setup, the binary caches provided by the channels diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index d4563ac47551..eac7739558be 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -456,7 +456,7 @@ $ nix-env -f ~/foo.nix -i '.*'</screen> from another profile: <screen> -$ nix-env -i --from-profile /nix/var/nix/profiles/foo -i gcc</screen> +$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc</screen> </para> diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index a5f615b0c268..f2dc6ed8540d 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -501,10 +501,11 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' <arg choice='plain'><option>--referrers</option></arg> <arg choice='plain'><option>--referrers-closure</option></arg> <arg choice='plain'><option>--deriver</option></arg> - <arg choice='plain'><option>--deriver</option></arg> + <arg choice='plain'><option>-d</option></arg> <arg choice='plain'><option>--graph</option></arg> <arg choice='plain'><option>--tree</option></arg> <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg> + <arg choice='plain'><option>-b</option> <replaceable>name</replaceable></arg> <arg choice='plain'><option>--hash</option></arg> <arg choice='plain'><option>--size</option></arg> <arg choice='plain'><option>--roots</option></arg> @@ -642,6 +643,7 @@ query is applied to the target of the symlink.</para> </varlistentry> <varlistentry><term><option>--deriver</option></term> + <term><option>-d</option></term> <listitem><para>Prints the <link linkend="gloss-deriver">deriver</link> of the store paths @@ -678,6 +680,7 @@ query is applied to the target of the symlink.</para> </varlistentry> <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term> + <term><option>-b</option> <replaceable>name</replaceable></term> <listitem><para>Prints the value of the attribute <replaceable>name</replaceable> (i.e., environment variable) of diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index 168bef080f4f..b610b54b9620 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -9,6 +9,9 @@ </group> </arg> <arg> + <arg choice='plain'><option>--quiet</option></arg> +</arg> +<arg> <group choice='plain'> <arg choice='plain'><option>--no-build-output</option></arg> <arg choice='plain'><option>-Q</option></arg> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index bcb60b30125c..4c572e129445 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -75,6 +75,23 @@ </varlistentry> +<varlistentry><term><option>--quiet</option></term> + + <listitem> + + <para>Decreases the level of verbosity of diagnostic messages + printed on standard error. This is the inverse option to + <option>-v</option> / <option>--verbose</option>. + </para> + + <para>This option may be specified repeatedly. See the previous + verbosity levels list.</para> + + </listitem> + +</varlistentry> + + <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term> <listitem><para>By default, output written by builders to standard diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 8a32ed8b5c99..ac1fe7e2fafe 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -307,6 +307,61 @@ stdenv.mkDerivation { โฆ } </varlistentry> + <varlistentry> + <term> + <function>builtins.fetchGit</function> + <replaceable>args</replaceable> + </term> + + <listitem> + <para> + Fetch a path from git. <replaceable>args</replaceable> can be + a URL, in which case the HEAD of the repo at that URL is + fetched. Otherwise, it can be an attribute with the following + attributes (all except <varname>url</varname> optional): + </para> + + <variablelist> + <varlistentry> + <term>url</term> + <listitem> + <para> + The URL of the repo. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>name</term> + <listitem> + <para> + The name of the directory the repo should be exported to + in the store. Defaults to the basename of the URL. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>rev</term> + <listitem> + <para> + The git revision to fetch. Defaults to the tip of + <varname>ref</varname>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>ref</term> + <listitem> + <para> + The git ref to look for the requested revision under. + This is often a branch or tag name. Defaults to + <literal>HEAD</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + </listitem> + </varlistentry> + <varlistentry><term><function>builtins.filter</function> <replaceable>f</replaceable> <replaceable>xs</replaceable></term> @@ -1148,7 +1203,10 @@ in foo</programlisting> This is not allowed because it would cause a cyclic dependency in the computation of the cryptographic hashes for - <varname>foo</varname> and <varname>bar</varname>.</para></listitem> + <varname>foo</varname> and <varname>bar</varname>.</para> + <para>It is also not possible to reference the result of a derivation. + If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to + do that.</para></listitem> </varlistentry> diff --git a/doc/manual/glossary/glossary.xml b/doc/manual/glossary/glossary.xml index e0636044cc25..4977825578f1 100644 --- a/doc/manual/glossary/glossary.xml +++ b/doc/manual/glossary/glossary.xml @@ -85,29 +85,48 @@ <glossentry xml:id="gloss-reference"><glossterm>reference</glossterm> - <glossdef><para>A store path <varname>P</varname> is said to have a - reference to a store path <varname>Q</varname> if the store object - at <varname>P</varname> contains the path <varname>Q</varname> - somewhere. This implies than an execution involving - <varname>P</varname> potentially needs <varname>Q</varname> to be - present. The <emphasis>references</emphasis> of a store path are - the set of store paths to which it has a reference.</para></glossdef> + <glossdef> + <para>A store path <varname>P</varname> is said to have a + reference to a store path <varname>Q</varname> if the store object + at <varname>P</varname> contains the path <varname>Q</varname> + somewhere. The <emphasis>references</emphasis> of a store path are + the set of store paths to which it has a reference. + </para> + <para>A derivation can reference other derivations and sources + (but not output paths), whereas an output path only references other + output paths. + </para> + </glossdef> </glossentry> +<glossentry xml:id="gloss-reachable"><glossterm>reachable</glossterm> + + <glossdef><para>A store path <varname>Q</varname> is reachable from + another store path <varname>P</varname> if <varname>Q</varname> is in the + <link linkend="gloss-closure">closure</link> of the + <link linkend="gloss-reference">references</link> relation. + </para></glossdef> +</glossentry> <glossentry xml:id="gloss-closure"><glossterm>closure</glossterm> <glossdef><para>The closure of a store path is the set of store paths that are directly or indirectly โreachableโ from that store path; that is, itโs the closure of the path under the <link - linkend="gloss-reference">references</link> relation. For instance, - if the store object at path <varname>P</varname> contains a - reference to path <varname>Q</varname>, then <varname>Q</varname> is - in the closure of <varname>P</varname>. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command <command>nix-store - -qR</command> prints out closures of store paths.</para></glossdef> + linkend="gloss-reference">references</link> relation. For a package, the + closure of its derivation is equivalent to the build-time + dependencies, while the closure of its output path is equivalent to its + runtime dependencies. For correct deployment it is necessary to deploy whole + closures, since otherwise at runtime files could be missing. The command + <command>nix-store -qR</command> prints out closures of store paths. + </para> + <para>As an example, if the store object at path <varname>P</varname> contains + a reference to path <varname>Q</varname>, then <varname>Q</varname> is + in the closure of <varname>P</varname>. Further, if <varname>Q</varname> + references <varname>R</varname> then <varname>R</varname> is also in + the closure of <varname>P</varname>. + </para></glossdef> </glossentry> @@ -147,7 +166,7 @@ linkend="sec-profiles" />.</para> </glossdef> - + </glossentry> diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index 49660c36e397..01e9688d635f 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -9,6 +9,9 @@ <itemizedlist> <listitem><para>GNU Make.</para></listitem> + + <listitem><para>Bash Shell. The <literal>./configure</literal> script + relies on bashisms, so Bash is required.</para></listitem> <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem> @@ -28,6 +31,14 @@ distribution does not provide these, you can obtain bzip2 from <link xlink:href="http://www.bzip.org/"/>.</para></listitem> + <listitem><para><literal>liblzma</literal>, which is provided by + XZ Utils. If your distribution does not provide this, you can + get it from <link xlink:href="https://tukaani.org/xz/"/>.</para></listitem> + + <listitem><para>cURL and its library. If your distribution does not + provide it, you can get it from <link + xlink:href="https://curl.haxx.se/"/>.</para></listitem> + <listitem><para>The SQLite embedded database library, version 3.6.19 or higher. If your distribution does not provide it, please install it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem> diff --git a/doc/manual/introduction/about-nix.xml b/doc/manual/introduction/about-nix.xml index 83a2b6786ac0..e8c0a29753a1 100644 --- a/doc/manual/introduction/about-nix.xml +++ b/doc/manual/introduction/about-nix.xml @@ -60,7 +60,8 @@ This is because tools such as compilers donโt search in per-packages directories such as <filename>/nix/store/5lbfaxb722zpโฆ-openssl-0.9.8d/include</filename>, so if a package builds correctly on your system, this is because you -specified the dependency explicitly.</para> +specified the dependency explicitly. This takes care of the build-time +dependencies.</para> <para>Once a package is built, runtime dependencies are found by scanning binaries for the hash parts of Nix store paths (such as diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml index 03b8e4c976c1..a1b0ef22a11e 100644 --- a/doc/manual/packages/garbage-collection.xml +++ b/doc/manual/packages/garbage-collection.xml @@ -52,6 +52,14 @@ garbage collector as follows: <screen> $ nix-store --gc</screen> +The behaviour of the gargage collector is affected by the <literal>keep- +derivations</literal> (default: true) and <literal>keep-outputs</literal> +(default: false) options in the Nix configuration file. The defaults will ensure +that all derivations that are not build-time dependencies of garbage collector roots +will be collected but that all output paths that are not runtime dependencies +will be collected. (This is usually what you want, but while you are developing +it may make sense to keep outputs to ensure that rebuild times are quick.) + If you are feeling uncertain, you can also first view what files would be deleted: diff --git a/doc/manual/packages/ssh-substituter.xml b/doc/manual/packages/ssh-substituter.xml index f24f354c4c39..8db3f96625d3 100644 --- a/doc/manual/packages/ssh-substituter.xml +++ b/doc/manual/packages/ssh-substituter.xml @@ -12,7 +12,7 @@ automatically fetching any store paths in Firefoxโs closure if they are available on the server <literal>avalon</literal>: <screen> -$ nix-env -i firefox --option ssh-substituter-hosts alice@avalon +$ nix-env -i firefox --substituters ssh://alice@avalon </screen> This works similar to the binary cache substituter that Nix usually @@ -31,7 +31,7 @@ an SSH passphrase interactively. Therefore, you should use installing it into your profile, e.g. <screen> -$ nix-store -r /nix/store/m85bxgโฆ-firefox-34.0.5 --option ssh-substituter-hosts alice@avalon +$ nix-store -r /nix/store/m85bxgโฆ-firefox-34.0.5 --substituters ssh://alice@avalon </screen> This is essentially equivalent to doing diff --git a/local.mk b/local.mk index 40a910991a48..5d7e0fb2e428 100644 --- a/local.mk +++ b/local.mk @@ -10,5 +10,3 @@ GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) - -$(foreach i, $(call rwildcard, src/boost, *.hpp), $(eval $(call install-file-in, $(i), $(includedir)/nix/$(patsubst src/%/,%,$(dir $(i))), 0644))) diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index d6b88c7e91a5..2f8e3dd7a679 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -4,7 +4,9 @@ FROM alpine RUN apk add --update openssl # Download Nix and install it into the system. -RUN wget -O- https://nixos.org/releases/nix/nix-1.11.14/nix-1.11.14-x86_64-linux.tar.bz2 | bzcat - | tar xf - \ +RUN wget https://nixos.org/releases/nix/nix-2.0/nix-2.0-x86_64-linux.tar.bz2 \ + && echo "6312837aee33306cdbb351b75ba1638b89d21b30f0caf0346f9a742425f197ee nix-2.0-x86_64-linux.tar.bz2" | sha256sum -c \ + && tar xjf nix-*-x86_64-linux.tar.bz2 \ && addgroup -g 30000 -S nixbld \ && for i in $(seq 1 30); do adduser -S -D -h /var/empty -g "Nix build user $i" -u $((30000 + i)) -G nixbld nixbld$i ; done \ && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \ diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 66fcd155ee9b..549619a57d56 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -4,6 +4,8 @@ <dict> <key>Label</key> <string>org.nixos.nix-daemon</string> + <key>KeepAlive</key> + <true/> <key>RunAtLoad</key> <true/> <key>Program</key> diff --git a/mk/programs.mk b/mk/programs.mk index 3ac64494e3a5..2fbda12bd153 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -51,7 +51,7 @@ define build-program else $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - install -t $$($(1)_INSTALL_DIR) $$< + install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< endif diff --git a/nix.spec.in b/nix.spec.in index 4e7dd90ea68c..d962bcc857b9 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -21,10 +21,12 @@ Requires: bzip2 Requires: gzip Requires: xz Requires: libseccomp +Requires: boost-context BuildRequires: bzip2-devel BuildRequires: sqlite-devel BuildRequires: libcurl-devel BuildRequires: libseccomp-devel +BuildRequires: boost-devel # Hack to make that shitty RPM scanning hack shut up. Provides: perl(Nix::SSH) diff --git a/perl/Makefile b/perl/Makefile index 684a37e8121f..284c75022493 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,6 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -std=c++14 -g -Wall +GLOBAL_CXXFLAGS += -g -Wall -include Makefile.config diff --git a/release-common.nix b/release-common.nix index f9df6fab9ff6..0c12bc7ceb38 100644 --- a/release-common.nix +++ b/release-common.nix @@ -1,9 +1,11 @@ { pkgs }: +with pkgs; + rec { # Use "busybox-sandbox-shell" if present, # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + sh = pkgs.busybox-sandbox-shell or (busybox.override { useMusl = true; enableStatic = true; enableMinimal = true; @@ -30,7 +32,41 @@ rec { configureFlags = [ "--disable-init-state" "--enable-gc" - ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + ] ++ lib.optionals stdenv.isLinux [ "--with-sandbox-shell=${sh}/bin/busybox" ]; + + tarballDeps = + [ bison + flex + libxml2 + libxslt + docbook5 + docbook5_xsl + autoconf-archive + autoreconfHook + ]; + + buildDeps = + [ curl + bzip2 xz brotli + openssl pkgconfig sqlite boehmgc + boost + + # Tests + git + mercurial + ] + ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + (aws-sdk-cpp.override { + apis = ["s3"]; + customMemoryManagement = false; + }); + + perlDeps = + [ perl + perlPackages.DBDSQLite + ]; } diff --git a/release.nix b/release.nix index 3f8d5da4721e..37deb8e7ee38 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "nix-2.0"; } +, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: @@ -14,6 +14,8 @@ let tarball = with pkgs; + with import ./release-common.nix { inherit pkgs; }; + releaseTools.sourceTarball { name = "nix-tarball"; version = builtins.readFile ./version; @@ -21,19 +23,13 @@ let src = nix; inherit officialRelease; - buildInputs = - [ curl bison flex libxml2 libxslt - bzip2 xz brotli - pkgconfig sqlite libsodium boehmgc - docbook5 docbook5_xsl - autoconf-archive - ] ++ lib.optional stdenv.isLinux libseccomp; + buildInputs = tarballDeps ++ buildDeps; configureFlags = "--enable-gc"; postUnpack = '' - (cd source && find . -type f) | cut -c3- > source/.dist-files - cat source/.dist-files + (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files + cat $sourceRoot/.dist-files ''; preConfigure = '' @@ -59,7 +55,9 @@ let build = pkgs.lib.genAttrs systems (system: - with import nixpkgs { inherit system; }; + let pkgs = import nixpkgs { inherit system; }; in + + with pkgs; with import ./release-common.nix { inherit pkgs; }; @@ -67,22 +65,7 @@ let name = "nix"; src = tarball; - buildInputs = - [ curl - bzip2 xz brotli - openssl pkgconfig sqlite boehmgc - - # Tests - git - mercurial - ] - ++ lib.optional stdenv.isLinux libseccomp - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3"]; - customMemoryManagement = false; - }); + buildInputs = buildDeps; configureFlags = configureFlags ++ [ "--sysconfdir=/etc" ]; @@ -109,7 +92,7 @@ let src = tarball; buildInputs = - [ (builtins.getAttr system jobs.build) curl bzip2 xz pkgconfig pkgs.perl ] + [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; configureFlags = '' @@ -132,30 +115,51 @@ let let toplevel = builtins.getAttr system jobs.build; version = toplevel.src.version; + installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; }; in runCommand "nix-binary-tarball-${version}" - { exportReferencesGraph = [ "closure1" toplevel "closure2" cacert ]; - buildInputs = [ perl ] ++ lib.optional (system != "aarch64-linux") shellcheck; + { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; } '' - storePaths=$(perl ${pathsFromGraph} ./closure1 ./closure2) - printRegistration=1 perl ${pathsFromGraph} ./closure1 ./closure2 > $TMPDIR/reginfo + cp ${installerClosureInfo}/registration $TMPDIR/reginfo substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user \ + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} if type -p shellcheck; then - shellcheck -e SC1090 $TMPDIR/install - shellcheck -e SC1091,SC2002 $TMPDIR/install-darwin-multi-user + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user fi chmod +x $TMPDIR/install - chmod +x $TMPDIR/install-darwin-multi-user + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user dir=nix-${version}-${system} fn=$out/$dir.tar.bz2 mkdir -p $out/nix-support @@ -167,22 +171,23 @@ let --transform "s,$TMPDIR/install,$dir/install," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/install-darwin-multi-user $TMPDIR/reginfo $storePaths + $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) ''); coverage = - with import nixpkgs { system = "x86_64-linux"; }; + with pkgs; + + with import ./release-common.nix { inherit pkgs; }; releaseTools.coverageAnalysis { name = "nix-build"; src = tarball; - buildInputs = - [ curl bzip2 openssl pkgconfig sqlite xz libsodium libseccomp - # These are for "make check" only: - graphviz libxml2 libxslt git mercurial - ]; + buildInputs = buildDeps; configureFlags = '' --disable-init-state @@ -201,17 +206,14 @@ let }; - rpm_fedora25i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora25i386) [ "libsodium-devel" ]; - rpm_fedora25x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora25x86_64) [ "libsodium-devel" ]; + rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ]; #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ]; #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ]; - deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; - deb_ubuntu1610i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1610i386) [ "libsodium-dev" ] [ "libsodium18" ]; - deb_ubuntu1610x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1610x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; + deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ]; + deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ]; # System tests. @@ -266,7 +268,8 @@ let export NIX_STATE_DIR=$TMPDIR nix-store --init - nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run + nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \ + --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }' touch $out ''; @@ -284,12 +287,6 @@ let binaryTarball.i686-linux binaryTarball.x86_64-darwin binaryTarball.x86_64-linux - #deb_debian8i386 - #deb_debian8x86_64 - deb_ubuntu1604i386 - deb_ubuntu1604x86_64 - rpm_fedora25i386 - rpm_fedora25x86_64 tests.remoteBuilds tests.nix-copy-closure tests.binaryTarball @@ -314,7 +311,7 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" ] + [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" ] ++ extraPackages; }; # At most 2047MB can be simulated in qemu-system-i386 memSize = 2047; @@ -337,9 +334,9 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" ] + [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ] ++ extraPackages; }; - memSize = 1024; + memSize = 2047; meta.schedulingPriority = 50; postInstall = "make installcheck"; configureFlags = "--sysconfdir=/etc"; diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 716b6e9bc9a3..87c4c2b0582a 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -1,819 +1,144 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu set -o pipefail -# Sourced from: -# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh -# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 -# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh -# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh -# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a -# -# however tracking which bits came from which would be impossible. - -readonly ESC='\033[0m' -readonly BOLD='\033[38;1m' -readonly BLUE='\033[38;34m' -readonly BLUE_UL='\033[38;4;34m' -readonly GREEN='\033[38;32m' -readonly GREEN_UL='\033[38;4;32m' -readonly RED='\033[38;31m' -readonly RED_UL='\033[38;4;31m' -readonly YELLOW='\033[38;33m' -readonly YELLOW_UL='\033[38;4;33m' - -readonly CORES=$(sysctl -n hw.ncpu) -readonly NIX_USER_COUNT="32" -readonly NIX_BUILD_GROUP_ID="30000" -readonly NIX_BUILD_GROUP_NAME="nixbld" -readonly NIX_FIRST_BUILD_UID="30001" -# Please don't change this. We don't support it, because the -# default shell profile that comes with Nix doesn't support it. -readonly NIX_ROOT="/nix" readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/zshrc") -readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" -readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" - -readonly NIX_INSTALLED_NIX="@nix@" -readonly NIX_INSTALLED_CACERT="@cacert@" -readonly EXTRACTED_NIX_PATH="$(dirname "$0")" - -readonly ROOT_HOME="/var/root" - -if [ -t 0 ]; then - readonly IS_HEADLESS='no' -else - readonly IS_HEADLESS='yes' -fi +dsclattr() { + /usr/bin/dscl . -read "$1" \ + | awk "/$2/ { print \$2 }" +} -headless() { - if [ "$IS_HEADLESS" = "yes" ]; then - return 0 - else - return 1 +poly_validate_assumptions() { + if [ "$(uname -s)" != "Darwin" ]; then + failure "This script is for use with macOS!" fi } -contactme() { - echo "We'd love to help if you need it." - echo "" - echo "If you can, open an issue at https://github.com/nixos/nix/issues" - echo "" - echo "Or feel free to contact the team," - echo " - on IRC #nixos on irc.freenode.net" - echo " - on twitter @nixos_org" +poly_service_installed_check() { + [ -e "$PLIST_DEST" ] } -uninstall_directions() { - subheader "Uninstalling nix:" - local step=0 - - if [ -e "$PLIST_DEST" ]; then - step=$((step + 1)) +poly_service_uninstall_directions() { cat <<EOF -$step. Delete $PLIST_DEST +$1. Delete $PLIST_DEST sudo launchctl unload $PLIST_DEST sudo rm $PLIST_DEST EOF - fi - - for profile_target in "${PROFILE_TARGETS[@]}"; do - if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then - step=$((step + 1)) - cat <<EOF -$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target - - sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target - -(after this one, you may need to re-open any terminals that were -opened while it existed.) - -EOF - fi - done +} - step=$((step + 1)) +poly_service_setup_note() { cat <<EOF -$step. Delete the files Nix added to your system: - - sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels - -and that is it. + - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon EOF - -} - -nix_user_for_core() { - printf "nixbld%d" "$1" -} - -nix_uid_for_core() { - echo $((NIX_FIRST_BUILD_UID + $1 - 1)) } -dsclattr() { - /usr/bin/dscl . -read "$1" \ - | awk "/$2/ { print \$2 }" -} - -_textout() { - echo -en "$1" - shift - if [ "$*" = "" ]; then - cat - else - echo "$@" - fi - echo -en "$ESC" -} - -header() { - follow="---------------------------------------------------------" - header=$(echo "---- $* $follow$follow$follow" | head -c 80) - echo "" - _textout "$BLUE" "$header" -} - -warningheader() { - follow="---------------------------------------------------------" - header=$(echo "---- $* $follow$follow$follow" | head -c 80) - echo "" - _textout "$RED" "$header" -} - -subheader() { - echo "" - _textout "$BLUE_UL" "$*" -} - -row() { - printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2" -} - -task() { - echo "" - ok "~~> $1" -} - -bold() { - echo "$BOLD$*$ESC" -} - -ok() { - _textout "$GREEN" "$@" -} - -warning() { - warningheader "warning!" - cat - echo "" -} - -failure() { - header "oh no!" - _textout "$RED" "$@" - echo "" - _textout "$RED" "$(contactme)" - trap finish_cleanup EXIT - exit 1 -} - -ui_confirm() { - _textout "$GREEN$GREEN_UL" "$1" - - if headless; then - echo "No TTY, assuming you would say yes :)" - return 0 - fi - - local prompt="[y/n] " - echo -n "$prompt" - while read -r y; do - if [ "$y" = "y" ]; then - echo "" - return 0 - elif [ "$y" = "n" ]; then - echo "" - return 1 - else - _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" - echo -n "$prompt" - fi - done - echo "" - return 1 -} - -__sudo() { - local expl="$1" - local cmd="$2" - shift - header "sudo execution" - - echo "I am executing:" - echo "" - printf " $ sudo %s\\n" "$cmd" - echo "" - echo "$expl" - echo "" - - return 0 -} +poly_configure_nix_daemon_service() { + _sudo "to set up the nix-daemon as a LaunchDaemon" \ + ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" -_sudo() { - local expl="$1" - shift - if ! headless; then - __sudo "$expl" "$*" - fi - sudo "$@" -} + _sudo "to load the LaunchDaemon plist for nix-daemon" \ + launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist + _sudo "to start the nix-daemon" \ + launchctl start org.nixos.nix-daemon -readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish_cleanup { - rm -rf "$SCRATCH" } -function finish_fail { - finish_cleanup - - failure <<EOF -Jeeze, something went wrong. If you can take all the output and open -an issue, we'd love to fix the problem so nobody else has this issue. - -:( -EOF +poly_group_exists() { + /usr/bin/dscl . -read "/Groups/$1" > /dev/null 2>&1 } -trap finish_fail EXIT - -function finish_success { - finish_cleanup - ok "Alright! We're done!" - cat <<EOF - -Before Nix will work in your existing shells, you'll need to close -them and open them again. Other than that, you should be ready to go. - -Try it! Open a new terminal, and type: - - $ nix-shell -p nix-info --run "nix-info -m" - -Thank you for using this installer. If you have any feedback, don't -hesitate: - -$(contactme) -EOF +poly_group_id_get() { + dsclattr "/Groups/$1" "PrimaryGroupID" } - -validate_starting_assumptions() { - if [ "$(uname -s)" != "Darwin" ]; then - failure "This script is for use with macOS!" - fi - - if [ $EUID -eq 0 ]; then - failure <<EOF -Please do not run this script with root privileges. We will call sudo -when we need to. -EOF - fi - - if type nix-env 2> /dev/null >&2; then - failure <<EOF -Nix already appears to be installed, and this tool assumes it is -_not_ yet installed. - -$(uninstall_directions) -EOF - fi - - if [ "${NIX_REMOTE:-}" != "" ]; then - failure <<EOF -For some reason, \$NIX_REMOTE is set. It really should not be set -before this installer runs, and it hints that Nix is currently -installed. Please delete the old Nix installation and start again. - -Note: You might need to close your shell window and open a new shell -to clear the variable. -EOF - fi - - if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then - failure <<EOF -It looks like \$SSL_CERT_FILE is set to a path that used to be part of -the old Nix installation. Please unset that variable and try again: - - $ unset SSL_CERT_FILE - -EOF - fi - - for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do - if [ -f "$file" ]; then - if grep -l "^[^#].*.nix-profile" "$file"; then - failure <<EOF -I found a reference to a ".nix-profile" in $file. -This has a high chance of breaking a new nix installation. It was most -likely put there by a previous Nix installer. - -Please remove this reference and try running this again. You should -also look for similar references in: - - - ~/.bash_profile - - ~/.bash_login - - ~/.profile - -or other shell init files that you may have. - -$(uninstall_directions) -EOF - fi - fi - done - - if [ -d /nix ]; then - failure <<EOF -There are some relics of a previous installation of Nix at /nix, and -this scripts assumes Nix is _not_ yet installed. Please delete the old -Nix installation and start again. - -$(uninstall_directions) -EOF - fi - - if [ -d /etc/nix ]; then - failure <<EOF -There are some relics of a previous installation of Nix at /etc/nix, and -this scripts assumes Nix is _not_ yet installed. Please delete the old -Nix installation and start again. - -$(uninstall_directions) -EOF - fi - - for profile_target in "${PROFILE_TARGETS[@]}"; do - if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then - failure <<EOF -When this script runs, it backs up the current $profile_target to -$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though. - -Please follow these instructions to clean up the old backup file: - -1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just -in case. - -2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like -it has anything nix-related in it. If it does, something is probably -quite wrong. Please open an issue or get in touch immediately. - -3. Take care to make sure that $profile_target doesn't look like it has -anything nix-related in it. If it does, and $profile_target _did not_, -run: - - $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target - -and try again. -EOF - fi - - if grep -qi "nix" "$profile_target"; then - failure <<EOF -It looks like $profile_target already has some Nix configuration in -there. There should be no reason to run this again. If you're having -trouble, please open an issue. -EOF - fi - done - - danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile") - for danger_path in "${danger_paths[@]}"; do - if _sudo "making sure that $danger_path doesn't exist" \ - test -e "$danger_path"; then - failure <<EOF -I found a file at $danger_path, which is a relic of a previous -installation. You must first delete this file before continuing. - -$(uninstall_directions) -EOF - fi - done +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + /usr/sbin/dseditgroup -o create \ + -r "Nix build group for nix-daemon" \ + -i "$NIX_BUILD_GROUP_ID" \ + "$NIX_BUILD_GROUP_NAME" >&2 } -setup_report() { - header "hardware report" - row " Cores" "$CORES" - - header "Nix config report" - row " Temp Dir" "$SCRATCH" - row " Nix Root" "$NIX_ROOT" - row " Build Users" "$NIX_USER_COUNT" - row " Build Group ID" "$NIX_BUILD_GROUP_ID" - row "Build Group Name" "$NIX_BUILD_GROUP_NAME" - if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then - row "Preexisting Install" "Allowed" - fi - - subheader "build users:" - - row " Username" "UID" - for i in $(seq 1 "$NIX_USER_COUNT"); do - row " $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")" - done - echo "" +poly_user_exists() { + /usr/bin/dscl . -read "/Users/$1" > /dev/null 2>&1 } -create_build_group() { - local primary_group_id - - task "Setting up the build group $NIX_BUILD_GROUP_NAME" - if ! /usr/bin/dscl . -read "/Groups/$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1; then - _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ - /usr/sbin/dseditgroup -o create \ - -r "Nix build group for nix-daemon" \ - -i "$NIX_BUILD_GROUP_ID" \ - "$NIX_BUILD_GROUP_NAME" >&2 - row " Created" "Yes" - else - primary_group_id=$(dsclattr "/Groups/$NIX_BUILD_GROUP_NAME" "PrimaryGroupID") - if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then - failure <<EOF -It seems the build group $NIX_BUILD_GROUP_NAME already exists, but -with the UID $primary_group_id. This script can't really handle -that right now, so I'm going to give up. - -You can fix this by editing this script and changing the -NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID -to $primary_group_id and re-run. -EOF - else - row " Exists" "Yes" - fi - fi +poly_user_id_get() { + dsclattr "/Users/$1" "UniqueID" } -create_build_user_for_core() { - local coreid - local username - local uid - - coreid="$1" - username=$(nix_user_for_core "$coreid") - uid=$(nix_uid_for_core "$coreid") - dsclpath="/Users/$username" - - task "Setting up the build user $username" - - if ! /usr/bin/dscl . -read "$dsclpath" > /dev/null 2>&1; then - _sudo "Creating the Nix build user, $username" \ - /usr/bin/dscl . create "$dsclpath" \ - UniqueID "${uid}" - row " Created" "Yes" - else - actual_uid=$(dsclattr "$dsclpath" "UniqueID") - if [ "$actual_uid" -ne "$uid" ]; then - failure <<EOF -It seems the build user $username already exists, but with the UID -with the UID $actual_uid. This script can't really handle that right -now, so I'm going to give up. - -If you already created the users and you know they start from -$actual_uid and go up from there, you can edit this script and change -NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try -again. -EOF - else - row " Exists" "Yes" - fi - fi - - if [ "$(dsclattr "$dsclpath" "IsHidden")" = "1" ]; then - row " IsHidden" "Yes" - else - _sudo "in order to make $username a hidden user" \ - /usr/bin/dscl . -create "$dsclpath" "IsHidden" "1" - row " IsHidden" "Yes" - fi - - if [ "$(dsclattr "$dsclpath" "NFSHomeDirectory")" = "/var/empty" ]; then - row " NFSHomeDirectory" "/var/empty" - else - _sudo "in order to give $username a safe home directory" \ - /usr/bin/dscl . -create "$dsclpath" "NFSHomeDirectory" "/var/empty" - row " NFSHomeDirectory" "/var/empty" - fi - - if [ "$(dsclattr "$dsclpath" "RealName")" = "Nix build user $coreid" ]; then - row " RealName" "Nix build user $coreid" - else - _sudo "in order to give $username a useful name" \ - /usr/bin/dscl . -create "$dsclpath" "RealName" "Nix build user $coreid" - row " RealName" "Nix build user $coreid" - fi - - if [ "$(dsclattr "$dsclpath" "UserShell")" = "/sbin/nologin" ]; then - row " Logins Disabled" "Yes" - else - _sudo "in order to prevent $username from logging in" \ - /usr/bin/dscl . -create "$dsclpath" "UserShell" "/sbin/nologin" - row " Logins Disabled" "Yes" - fi - - if dseditgroup -o checkmember -m "$username" "$NIX_BUILD_GROUP_NAME" > /dev/null 2>&1 ; then - row " Member of $NIX_BUILD_GROUP_NAME" "Yes" - else - _sudo "Add $username to the $NIX_BUILD_GROUP_NAME group"\ - /usr/sbin/dseditgroup -o edit -t user \ - -a "$username" "$NIX_BUILD_GROUP_NAME" - row " Member of $NIX_BUILD_GROUP_NAME" "Yes" - fi - - if [ "$(dsclattr "$dsclpath" "PrimaryGroupID")" = "$NIX_BUILD_GROUP_ID" ]; then - row " PrimaryGroupID" "$NIX_BUILD_GROUP_ID" - else - _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ - /usr/bin/dscl . -create "$dsclpath" "PrimaryGroupID" "$NIX_BUILD_GROUP_ID" - row " PrimaryGroupID" "$NIX_BUILD_GROUP_ID" - - fi +poly_user_hidden_get() { + dsclattr "/Users/$1" "IsHidden" } -create_build_users() { - for i in $(seq 1 "$NIX_USER_COUNT"); do - create_build_user_for_core "$i" - done +poly_user_hidden_set() { + _sudo "in order to make $1 a hidden user" \ + /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1" } -create_directories() { - _sudo "to make the basic directory structure of Nix (part 1)" \ - mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} - - _sudo "to make the basic directory structure of Nix (part 2)" \ - mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user - - _sudo "to make the basic directory structure of Nix (part 3)" \ - mkdir -pv -m 1775 /nix/store - - _sudo "to make the basic directory structure of Nix (part 4)" \ - chgrp "$NIX_BUILD_GROUP_NAME" /nix/store - - _sudo "to set up the root user's profile (part 1)" \ - mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root - - _sudo "to set up the root user's profile (part 2)" \ - mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr" - - _sudo "to place the default nix daemon configuration (part 1)" \ - mkdir -pv -m 0555 /etc/nix +poly_user_home_get() { + dsclattr "/Users/$1" "NFSHomeDirectory" } -place_channel_configuration() { - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" - _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2" } -welcome_to_nix() { - ok "Welcome to the Multi-User Nix Installation" - - cat <<EOF - -This installation tool will set up your computer with the Nix package -manager. This will happen in a few stages: - -1. Make sure your computer doesn't already have Nix. If it does, I - will show you instructions on how to clean up your old one. - -2. Show you what we are going to install and where. Then we will ask - if you are ready to continue. - -3. Create the system users and groups that the Nix daemon uses to run - builds. - -4. Perform the basic installation of the Nix files daemon. - -5. Configure your shell to import special Nix Profile files, so you - can use Nix. - -6. Start the Nix daemon. - -EOF - - if ui_confirm "Would you like to see a more detailed list of what we will do?"; then - cat <<EOF - -We will: - - - make sure your computer doesn't already have Nix files - (if it does, I will tell you how to clean them up.) - - create local users (see the list above for the users we'll make) - - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT - - create a configuration file in /etc/nix - - set up the "default profile" by creating some Nix-related files in - $ROOT_HOME -EOF - for profile_target in "${PROFILE_TARGETS[@]}"; do - if [ -e "$profile_target" ]; then - cat <<EOF - - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX - - update $profile_target to include some Nix configuration -EOF - fi - done - cat <<EOF - - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon - -EOF - if ! ui_confirm "Ready to continue?"; then - failure <<EOF -Okay, maybe you would like to talk to the team. -EOF - fi - fi +poly_user_note_get() { + dsclattr "/Users/$1" "RealName" } -chat_about_sudo() { - header "let's talk about sudo" - - if headless; then - cat <<EOF -This script is going to call sudo a lot. Normally, it would show you -exactly what commands it is running and why. However, the script is -run in a headless fashion, like this: - - $ curl https://nixos.org/nix/install | sh - -or maybe in a CI pipeline. Because of that, we're going to skip the -verbose output in the interest of brevity. - -If you would like to -see the output, try like this: - - $ curl -o install-nix https://nixos.org/nix/install - $ sh ./install-nix - -EOF - return 0 - fi - - cat <<EOF -This script is going to call sudo a lot. Every time we do, it'll -output exactly what it'll do, and why. - -Just like this: -EOF - - __sudo "to demonstrate how our sudo prompts look" \ - echo "this is a sudo prompt" - - cat <<EOF - -This might look scary, but everything can be undone by running just a -few commands. We used to ask you to confirm each time sudo ran, but it -was too many times. Instead, I'll just ask you this one time: - -EOF - if ui_confirm "Can we use sudo?"; then - ok "Yay! Thanks! Let's get going!" - else - failure <<EOF -That is okay, but we can't install. -EOF - fi +poly_user_note_set() { + _sudo "in order to give $username a useful note" \ + /usr/bin/dscl . -create "/Users/$1" "RealName" "$2" } -install_from_extracted_nix() { - ( - cd "$EXTRACTED_NIX_PATH" - - _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ - rsync -rlpt ./store/* "$NIX_ROOT/store/" - - if [ -d "$NIX_INSTALLED_NIX" ]; then - echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" - else - failure <<EOF -Something went wrong, and I didn't find Nix installed at -$NIX_INSTALLED_NIX. -EOF - fi - - _sudo "to initialize the Nix Database" \ - $NIX_INSTALLED_NIX/bin/nix-store --init - - cat ./.reginfo \ - | _sudo "to load data for the first time in to the Nix Database" \ - "$NIX_INSTALLED_NIX/bin/nix-store" --load-db - - echo " Just finished getting the nix database ready." - ) +poly_user_shell_get() { + dsclattr "/Users/$1" "UserShell" } -shell_source_lines() { - cat <<EOF - -# Nix -if [ -e '$PROFILE_NIX_FILE' ]; then - . '$PROFILE_NIX_FILE' -fi -# End Nix - -EOF +poly_user_shell_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" } -configure_shell_profile() { - for profile_target in "${PROFILE_TARGETS[@]}"; do - if [ -e "$profile_target" ]; then - _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \ - cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX" - - shell_source_lines \ - | _sudo "extend your $profile_target with nix-daemon settings" \ - tee -a "$profile_target" - fi - done +poly_user_in_group_check() { + username=$1 + group=$2 + dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 } -setup_default_profile() { - _sudo "to installing a bootstrapping Nix in to the default Profile" \ - HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX" - - _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ - HOME=$ROOT_HOME "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" +poly_user_in_group_set() { + username=$1 + group=$2 - _sudo "to update the default channel in the default profile" \ - HOME=$ROOT_HOME NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs + _sudo "Add $username to the $group group"\ + /usr/sbin/dseditgroup -o edit -t user \ + -a "$username" "$group" } - -place_nix_configuration() { - cat <<EOF > "$SCRATCH/nix.conf" -build-users-group = $NIX_BUILD_GROUP_NAME - -max-jobs = $NIX_USER_COUNT -cores = 1 -sandbox = false -EOF - _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +poly_user_primary_group_get() { + dsclattr "/Users/$1" "PrimaryGroupID" } -configure_nix_daemon_plist() { - _sudo "to set up the nix-daemon as a LaunchDaemon" \ - ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" - - _sudo "to load the LaunchDaemon plist for nix-daemon" \ - launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist - - _sudo "to start the nix-daemon" \ - launchctl start org.nixos.nix-daemon - +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + /usr/bin/dscl . -create "/Users/$1" "PrimaryGroupID" "$2" } +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 -main() { - welcome_to_nix - chat_about_sudo - - if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then - validate_starting_assumptions - fi - - setup_report - - if ! ui_confirm "Ready to continue?"; then - ok "Alright, no changes have been made :)" - contactme - trap finish_cleanup EXIT - exit 1 - fi - - create_build_group - create_build_users - create_directories - place_channel_configuration - install_from_extracted_nix - - configure_shell_profile - - set +eu - . /etc/profile - set -eu - - setup_default_profile - place_nix_configuration - configure_nix_daemon_plist - - trap finish_success EXIT + _sudo "Creating the Nix build user (#$builder_num), $username" \ + /usr/bin/dscl . create "/Users/$username" \ + UniqueID "${uid}" } - - -main diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh new file mode 100644 index 000000000000..5f6542355e0c --- /dev/null +++ b/scripts/install-multi-user.sh @@ -0,0 +1,797 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +# Sourced from: +# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh +# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 +# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh +# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh +# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a +# +# however tracking which bits came from which would be impossible. + +readonly ESC='\033[0m' +readonly BOLD='\033[38;1m' +readonly BLUE='\033[38;34m' +readonly BLUE_UL='\033[38;4;34m' +readonly GREEN='\033[38;32m' +readonly GREEN_UL='\033[38;4;32m' +readonly RED='\033[38;31m' +readonly RED_UL='\033[38;4;31m' +readonly YELLOW='\033[38;33m' +readonly YELLOW_UL='\033[38;4;33m' + +readonly NIX_USER_COUNT="32" +readonly NIX_BUILD_GROUP_ID="30000" +readonly NIX_BUILD_GROUP_NAME="nixbld" +readonly NIX_FIRST_BUILD_UID="30001" +# Please don't change this. We don't support it, because the +# default shell profile that comes with Nix doesn't support it. +readonly NIX_ROOT="/nix" + +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" +readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + +readonly NIX_INSTALLED_NIX="@nix@" +readonly NIX_INSTALLED_CACERT="@cacert@" +readonly EXTRACTED_NIX_PATH="$(dirname "$0")" + +readonly ROOT_HOME=$(echo ~root) + +if [ -t 0 ]; then + readonly IS_HEADLESS='no' +else + readonly IS_HEADLESS='yes' +fi + +headless() { + if [ "$IS_HEADLESS" = "yes" ]; then + return 0 + else + return 1 + fi +} + +contactme() { + echo "We'd love to help if you need it." + echo "" + echo "If you can, open an issue at https://github.com/nixos/nix/issues" + echo "" + echo "Or feel free to contact the team," + echo " - on IRC #nixos on irc.freenode.net" + echo " - on twitter @nixos_org" +} + +uninstall_directions() { + subheader "Uninstalling nix:" + local step=0 + + if poly_service_installed_check; then + step=$((step + 1)) + poly_service_uninstall_directions "$step" + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + step=$((step + 1)) + cat <<EOF +$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target + + sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target + +(after this one, you may need to re-open any terminals that were +opened while it existed.) + +EOF + fi + done + + step=$((step + 1)) + cat <<EOF +$step. Delete the files Nix added to your system: + + sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels + +and that is it. + +EOF + +} + +nix_user_for_core() { + printf "nixbld%d" "$1" +} + +nix_uid_for_core() { + echo $((NIX_FIRST_BUILD_UID + $1 - 1)) +} + +_textout() { + echo -en "$1" + shift + if [ "$*" = "" ]; then + cat + else + echo "$@" + fi + echo -en "$ESC" +} + +header() { + follow="---------------------------------------------------------" + header=$(echo "---- $* $follow$follow$follow" | head -c 80) + echo "" + _textout "$BLUE" "$header" +} + +warningheader() { + follow="---------------------------------------------------------" + header=$(echo "---- $* $follow$follow$follow" | head -c 80) + echo "" + _textout "$RED" "$header" +} + +subheader() { + echo "" + _textout "$BLUE_UL" "$*" +} + +row() { + printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2" +} + +task() { + echo "" + ok "~~> $1" +} + +bold() { + echo "$BOLD$*$ESC" +} + +ok() { + _textout "$GREEN" "$@" +} + +warning() { + warningheader "warning!" + cat + echo "" +} + +failure() { + header "oh no!" + _textout "$RED" "$@" + echo "" + _textout "$RED" "$(contactme)" + trap finish_cleanup EXIT + exit 1 +} + +ui_confirm() { + _textout "$GREEN$GREEN_UL" "$1" + + if headless; then + echo "No TTY, assuming you would say yes :)" + return 0 + fi + + local prompt="[y/n] " + echo -n "$prompt" + while read -r y; do + if [ "$y" = "y" ]; then + echo "" + return 0 + elif [ "$y" = "n" ]; then + echo "" + return 1 + else + _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" + echo -n "$prompt" + fi + done + echo "" + return 1 +} + +__sudo() { + local expl="$1" + local cmd="$2" + shift + header "sudo execution" + + echo "I am executing:" + echo "" + printf " $ sudo %s\\n" "$cmd" + echo "" + echo "$expl" + echo "" + + return 0 +} + +_sudo() { + local expl="$1" + shift + if ! headless; then + __sudo "$expl" "$*" + fi + sudo "$@" +} + + +readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish_cleanup { + rm -rf "$SCRATCH" +} + +function finish_fail { + finish_cleanup + + failure <<EOF +Jeeze, something went wrong. If you can take all the output and open +an issue, we'd love to fix the problem so nobody else has this issue. + +:( +EOF +} +trap finish_fail EXIT + +function finish_success { + finish_cleanup + + ok "Alright! We're done!" + cat <<EOF + +Before Nix will work in your existing shells, you'll need to close +them and open them again. Other than that, you should be ready to go. + +Try it! Open a new terminal, and type: + + $ nix-shell -p nix-info --run "nix-info -m" + +Thank you for using this installer. If you have any feedback, don't +hesitate: + +$(contactme) +EOF +} + + +validate_starting_assumptions() { + poly_validate_assumptions + + if [ $EUID -eq 0 ]; then + failure <<EOF +Please do not run this script with root privileges. We will call sudo +when we need to. +EOF + fi + + if type nix-env 2> /dev/null >&2; then + failure <<EOF +Nix already appears to be installed, and this tool assumes it is +_not_ yet installed. + +$(uninstall_directions) +EOF + fi + + if [ "${NIX_REMOTE:-}" != "" ]; then + failure <<EOF +For some reason, \$NIX_REMOTE is set. It really should not be set +before this installer runs, and it hints that Nix is currently +installed. Please delete the old Nix installation and start again. + +Note: You might need to close your shell window and open a new shell +to clear the variable. +EOF + fi + + if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then + failure <<EOF +It looks like \$SSL_CERT_FILE is set to a path that used to be part of +the old Nix installation. Please unset that variable and try again: + + $ unset SSL_CERT_FILE + +EOF + fi + + for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do + if [ -f "$file" ]; then + if grep -l "^[^#].*.nix-profile" "$file"; then + failure <<EOF +I found a reference to a ".nix-profile" in $file. +This has a high chance of breaking a new nix installation. It was most +likely put there by a previous Nix installer. + +Please remove this reference and try running this again. You should +also look for similar references in: + + - ~/.bash_profile + - ~/.bash_login + - ~/.profile + +or other shell init files that you may have. + +$(uninstall_directions) +EOF + fi + fi + done + + if [ -d /nix ]; then + failure <<EOF +There are some relics of a previous installation of Nix at /nix, and +this scripts assumes Nix is _not_ yet installed. Please delete the old +Nix installation and start again. + +$(uninstall_directions) +EOF + fi + + if [ -d /etc/nix ]; then + failure <<EOF +There are some relics of a previous installation of Nix at /etc/nix, and +this scripts assumes Nix is _not_ yet installed. Please delete the old +Nix installation and start again. + +$(uninstall_directions) +EOF + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + failure <<EOF +When this script runs, it backs up the current $profile_target to +$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though. + +Please follow these instructions to clean up the old backup file: + +1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just +in case. + +2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like +it has anything nix-related in it. If it does, something is probably +quite wrong. Please open an issue or get in touch immediately. + +3. Take care to make sure that $profile_target doesn't look like it has +anything nix-related in it. If it does, and $profile_target _did not_, +run: + + $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target + +and try again. +EOF + fi + + if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then + failure <<EOF +It looks like $profile_target already has some Nix configuration in +there. There should be no reason to run this again. If you're having +trouble, please open an issue. +EOF + fi + done + + danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile") + for danger_path in "${danger_paths[@]}"; do + if _sudo "making sure that $danger_path doesn't exist" \ + test -e "$danger_path"; then + failure <<EOF +I found a file at $danger_path, which is a relic of a previous +installation. You must first delete this file before continuing. + +$(uninstall_directions) +EOF + fi + done +} + +setup_report() { + header "Nix config report" + row " Temp Dir" "$SCRATCH" + row " Nix Root" "$NIX_ROOT" + row " Build Users" "$NIX_USER_COUNT" + row " Build Group ID" "$NIX_BUILD_GROUP_ID" + row "Build Group Name" "$NIX_BUILD_GROUP_NAME" + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then + row "Preexisting Install" "Allowed" + fi + + subheader "build users:" + + row " Username" "UID" + for i in $(seq 1 "$NIX_USER_COUNT"); do + row " $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")" + done + echo "" +} + +create_build_group() { + local primary_group_id + + task "Setting up the build group $NIX_BUILD_GROUP_NAME" + if ! poly_group_exists "$NIX_BUILD_GROUP_NAME"; then + poly_create_build_group + row " Created" "Yes" + else + primary_group_id=$(poly_group_id_get "$NIX_BUILD_GROUP_NAME") + if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then + failure <<EOF +It seems the build group $NIX_BUILD_GROUP_NAME already exists, but +with the UID $primary_group_id. This script can't really handle +that right now, so I'm going to give up. + +You can fix this by editing this script and changing the +NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID +to $primary_group_id and re-run. +EOF + else + row " Exists" "Yes" + fi + fi +} + +create_build_user_for_core() { + local coreid + local username + local uid + + coreid="$1" + username=$(nix_user_for_core "$coreid") + uid=$(nix_uid_for_core "$coreid") + + task "Setting up the build user $username" + + if ! poly_user_exists "$username"; then + poly_create_build_user "$username" "$uid" "$coreid" + row " Created" "Yes" + else + actual_uid=$(poly_user_id_get "$username") + if [ "$actual_uid" != "$uid" ]; then + failure <<EOF +It seems the build user $username already exists, but with the UID +with the UID '$actual_uid'. This script can't really handle that right +now, so I'm going to give up. + +If you already created the users and you know they start from +$actual_uid and go up from there, you can edit this script and change +NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try +again. +EOF + else + row " Exists" "Yes" + fi + fi + + if [ "$(poly_user_hidden_get "$username")" = "1" ]; then + row " Hidden" "Yes" + else + poly_user_hidden_set "$username" + row " Hidden" "Yes" + fi + + if [ "$(poly_user_home_get "$username")" = "/var/empty" ]; then + row " Home Directory" "/var/empty" + else + poly_user_home_set "$username" "/var/empty" + row " Home Directory" "/var/empty" + fi + + # We use grep instead of an equality check because it is difficult + # to extract _just_ the user's note, instead it is prefixed with + # some plist junk. This was causing the user note to always be set, + # even if there was no reason for it. + if ! poly_user_note_get "$username" | grep -q "Nix build user $coreid"; then + row " Note" "Nix build user $coreid" + else + poly_user_note_set "$username" "Nix build user $coreid" + row " Note" "Nix build user $coreid" + fi + + if [ "$(poly_user_shell_get "$username")" = "/sbin/nologin" ]; then + row " Logins Disabled" "Yes" + else + poly_user_shell_set "$username" "/sbin/nologin" + row " Logins Disabled" "Yes" + fi + + if poly_user_in_group_check "$username" "$NIX_BUILD_GROUP_NAME"; then + row " Member of $NIX_BUILD_GROUP_NAME" "Yes" + else + poly_user_in_group_set "$username" "$NIX_BUILD_GROUP_NAME" + row " Member of $NIX_BUILD_GROUP_NAME" "Yes" + fi + + if [ "$(poly_user_primary_group_get "$username")" = "$NIX_BUILD_GROUP_ID" ]; then + row " PrimaryGroupID" "$NIX_BUILD_GROUP_ID" + else + poly_user_primary_group_set "$username" "$NIX_BUILD_GROUP_ID" + row " PrimaryGroupID" "$NIX_BUILD_GROUP_ID" + fi +} + +create_build_users() { + for i in $(seq 1 "$NIX_USER_COUNT"); do + create_build_user_for_core "$i" + done +} + +create_directories() { + _sudo "to make the basic directory structure of Nix (part 1)" \ + mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} + + _sudo "to make the basic directory structure of Nix (part 2)" \ + mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user + + _sudo "to make the basic directory structure of Nix (part 3)" \ + mkdir -pv -m 1775 /nix/store + + _sudo "to make the basic directory structure of Nix (part 4)" \ + chgrp "$NIX_BUILD_GROUP_NAME" /nix/store + + _sudo "to set up the root user's profile (part 1)" \ + mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root + + _sudo "to set up the root user's profile (part 2)" \ + mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr" + + _sudo "to place the default nix daemon configuration (part 1)" \ + mkdir -pv -m 0555 /etc/nix +} + +place_channel_configuration() { + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" +} + +welcome_to_nix() { + ok "Welcome to the Multi-User Nix Installation" + + cat <<EOF + +This installation tool will set up your computer with the Nix package +manager. This will happen in a few stages: + +1. Make sure your computer doesn't already have Nix. If it does, I + will show you instructions on how to clean up your old one. + +2. Show you what we are going to install and where. Then we will ask + if you are ready to continue. + +3. Create the system users and groups that the Nix daemon uses to run + builds. + +4. Perform the basic installation of the Nix files daemon. + +5. Configure your shell to import special Nix Profile files, so you + can use Nix. + +6. Start the Nix daemon. + +EOF + + if ui_confirm "Would you like to see a more detailed list of what we will do?"; then + cat <<EOF + +We will: + + - make sure your computer doesn't already have Nix files + (if it does, I will tell you how to clean them up.) + - create local users (see the list above for the users we'll make) + - create a local group ($NIX_BUILD_GROUP_NAME) + - install Nix in to $NIX_ROOT + - create a configuration file in /etc/nix + - set up the "default profile" by creating some Nix-related files in + $ROOT_HOME +EOF + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ]; then + cat <<EOF + - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX + - update $profile_target to include some Nix configuration +EOF + fi + done + poly_service_setup_note + if ! ui_confirm "Ready to continue?"; then + failure <<EOF +Okay, maybe you would like to talk to the team. +EOF + fi + fi +} + +chat_about_sudo() { + header "let's talk about sudo" + + if headless; then + cat <<EOF +This script is going to call sudo a lot. Normally, it would show you +exactly what commands it is running and why. However, the script is +run in a headless fashion, like this: + + $ curl https://nixos.org/nix/install | sh + +or maybe in a CI pipeline. Because of that, we're going to skip the +verbose output in the interest of brevity. + +If you would like to +see the output, try like this: + + $ curl -o install-nix https://nixos.org/nix/install + $ sh ./install-nix + +EOF + return 0 + fi + + cat <<EOF +This script is going to call sudo a lot. Every time we do, it'll +output exactly what it'll do, and why. + +Just like this: +EOF + + __sudo "to demonstrate how our sudo prompts look" \ + echo "this is a sudo prompt" + + cat <<EOF + +This might look scary, but everything can be undone by running just a +few commands. We used to ask you to confirm each time sudo ran, but it +was too many times. Instead, I'll just ask you this one time: + +EOF + if ui_confirm "Can we use sudo?"; then + ok "Yay! Thanks! Let's get going!" + else + failure <<EOF +That is okay, but we can't install. +EOF + fi +} + +install_from_extracted_nix() { + ( + cd "$EXTRACTED_NIX_PATH" + + _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ + rsync -rlpt ./store/* "$NIX_ROOT/store/" + + if [ -d "$NIX_INSTALLED_NIX" ]; then + echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" + else + failure <<EOF +Something went wrong, and I didn't find Nix installed at +$NIX_INSTALLED_NIX. +EOF + fi + + _sudo "to initialize the Nix Database" \ + $NIX_INSTALLED_NIX/bin/nix-store --init + + cat ./.reginfo \ + | _sudo "to load data for the first time in to the Nix Database" \ + "$NIX_INSTALLED_NIX/bin/nix-store" --load-db + + echo " Just finished getting the nix database ready." + ) +} + +shell_source_lines() { + cat <<EOF + +# Nix +if [ -e '$PROFILE_NIX_FILE' ]; then + . '$PROFILE_NIX_FILE' +fi +# End Nix + +EOF +} + +configure_shell_profile() { + # If there is an /etc/profile.d directory, we want to ensure there + # is a nix.sh within it, so we can use the following loop to add + # the source lines to it. Note that I'm _not_ adding the source + # lines here, because we want to be using the regular machinery. + # + # If we go around that machinery, it becomes more complicated and + # adds complications to the uninstall instruction generator and + # old instruction sniffer as well. + if [ -d /etc/profile.d ]; then + _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \ + touch /etc/profile.d/nix.sh + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ]; then + _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \ + cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX" + + shell_source_lines \ + | _sudo "extend your $profile_target with nix-daemon settings" \ + tee -a "$profile_target" + fi + done +} + +setup_default_profile() { + _sudo "to installing a bootstrapping Nix in to the default Profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX" + + _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" + + _sudo "to update the default channel in the default profile" \ + HOME="$ROOT_HOME" NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs +} + + +place_nix_configuration() { + cat <<EOF > "$SCRATCH/nix.conf" +build-users-group = $NIX_BUILD_GROUP_NAME + +max-jobs = $NIX_USER_COUNT +cores = 1 +sandbox = false +EOF + _sudo "to place the default nix daemon configuration (part 2)" \ + install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +} + +main() { + if [ "$(uname -s)" = "Darwin" ]; then + # shellcheck source=./install-darwin-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, I don't know what to do on $(uname)" + fi + + welcome_to_nix + chat_about_sudo + + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then + validate_starting_assumptions + fi + + setup_report + + if ! ui_confirm "Ready to continue?"; then + ok "Alright, no changes have been made :)" + contactme + trap finish_cleanup EXIT + exit 1 + fi + + create_build_group + create_build_users + create_directories + place_channel_configuration + install_from_extracted_nix + + configure_shell_profile + + set +eu + . /etc/profile + set -eu + + setup_default_profile + place_nix_configuration + poly_configure_nix_daemon_service + + trap finish_success EXIT +} + + +main diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 3e5676f419ba..cd71d7947d77 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -28,9 +28,41 @@ if [ "$(uname -s)" = "Darwin" ]; then echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher" exit 1 fi +fi + +# Determine if we should punt to the single-user installer or not +if [ "$(uname -s)" = "Darwin" ]; then + INSTALL_MODE=daemon +elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then + INSTALL_MODE=daemon +else + INSTALL_MODE=no-daemon +fi + +# Trivially handle the --daemon / --no-daemon options +if [ "x${1:-}" = "x--no-daemon" ]; then + INSTALL_MODE=no-daemon +elif [ "x${1:-}" = "x--daemon" ]; then + INSTALL_MODE=daemon +elif [ "x${1:-}" != "x" ]; then + ( + echo "Nix Installer [--daemon|--no-daemon]" + echo "" + echo " --daemon: Force the installer to use the Daemon" + echo " based installer, even though it may not" + echo " work." + echo "" + echo " --no-daemon: Force a no-daemon, single-user" + echo " installation even when the preferred" + echo " method is with the daemon." + echo "" + ) >&2 + exit +fi - printf '\e[1;31mSwitching to the Multi-User Darwin Installer\e[0m\n' - exec "$self/install-darwin-multi-user" +if [ "$INSTALL_MODE" = "daemon" ]; then + printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' + exec "$self/install-multi-user" exit 0 fi diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh new file mode 100644 index 000000000000..04bc539a1099 --- /dev/null +++ b/scripts/install-systemd-multi-user.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service +readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service + +readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket +readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + +poly_validate_assumptions() { + if [ "$(uname -s)" != "Linux" ]; then + failure "This script is for use with Linux!" + fi +} + +poly_service_installed_check() { + [ "$(systemctl is-enabled nix-daemon.service)" = "linked" ] \ + || [ "$(systemctl is-enabled nix-daemon.socket)" = "enabled" ] +} + +poly_service_uninstall_directions() { + cat <<EOF +$1. Delete the systemd service and socket units + + sudo systemctl stop nix-daemon.socket + sudo systemctl stop nix-daemon.service + sudo systemctl disable nix-daemon.socket + sudo systemctl disable nix-daemon.service + sudo systemctl daemon-reload +EOF +} + +poly_service_setup_note() { + cat <<EOF + - load and start a service (at $SERVICE_DEST + and $SOCKET_DEST) for nix-daemon + +EOF +} + +poly_configure_nix_daemon_service() { + _sudo "to set up the nix-daemon service" \ + systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC" + + _sudo "to set up the nix-daemon socket service" \ + systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC" + + _sudo "to load the systemd unit for nix-daemon" \ + systemctl daemon-reload + + _sudo "to start the nix-daemon.socket" \ + systemctl start nix-daemon.socket + + _sudo "to start the nix-daemon.service" \ + systemctl start nix-daemon.service + +} + +poly_group_exists() { + getent group "$1" > /dev/null 2>&1 +} + +poly_group_id_get() { + getent group "$1" | cut -d: -f3 +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + groupadd -g "$NIX_BUILD_GROUP_ID" --system \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + getent passwd "$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + getent passwd "$1" | cut -d: -f3 +} + +poly_user_hidden_get() { + echo "1" +} + +poly_user_hidden_set() { + true +} + +poly_user_home_get() { + getent passwd "$1" | cut -d: -f6 +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + usermod --home "$2" "$1" +} + +poly_user_note_get() { + getent passwd "$1" | cut -d: -f5 +} + +poly_user_note_set() { + _sudo "in order to give $1 a useful comment" \ + usermod --comment "$2" "$1" +} + +poly_user_shell_get() { + getent passwd "$1" | cut -d: -f7 +} + +poly_user_shell_set() { + _sudo "in order to prevent $1 from logging in" \ + usermod --shell "$2" "$1" +} + +poly_user_in_group_check() { + groups "$1" | grep -q "$2" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + _sudo "Add $1 to the $2 group"\ + usermod --append --groups "$2" "$1" +} + +poly_user_primary_group_get() { + getent passwd "$1" | cut -d: -f4 +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + usermod --gid "$2" "$1" + +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user, $username" \ + useradd \ + --home-dir /var/empty \ + --comment "Nix build user $builder_num" \ + --gid "$NIX_BUILD_GROUP_ID" \ + --groups "$NIX_BUILD_GROUP_NAME" \ + --no-user-group \ + --system \ + --shell /sbin/nologin \ + --uid "$uid" \ + --password "!" \ + "$username" +} diff --git a/shell.nix b/shell.nix index d6ae5a55226e..c04bcd151309 100644 --- a/shell.nix +++ b/shell.nix @@ -1,33 +1,13 @@ { useClang ? false }: -with import <nixpkgs> {}; +with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.03"; }) {}; with import ./release-common.nix { inherit pkgs; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; - buildInputs = - [ curl bison flex libxml2 libxslt - bzip2 xz brotli - pkgconfig sqlite libsodium boehmgc - docbook5 docbook5_xsl - autoconf-archive - (aws-sdk-cpp.override { - apis = ["s3"]; - customMemoryManagement = false; - }) - autoreconfHook - - # For nix-perl - perl - perlPackages.DBDSQLite - - # Tests - git - mercurial - ] - ++ lib.optional stdenv.isLinux libseccomp; + buildInputs = buildDeps ++ tarballDeps ++ perlDeps; inherit configureFlags; diff --git a/src/boost/assert.hpp b/src/boost/assert.hpp deleted file mode 100644 index 754ebb954bce..000000000000 --- a/src/boost/assert.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// -// boost/assert.hpp - BOOST_ASSERT(expr) -// -// Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd. -// -// Permission to copy, use, modify, sell and distribute this software -// is granted provided this copyright notice appears in all copies. -// This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. -// -// Note: There are no include guards. This is intentional. -// -// See http://www.boost.org/libs/utility/assert.html for documentation. -// - -#undef BOOST_ASSERT - -#if defined(BOOST_DISABLE_ASSERTS) - -# define BOOST_ASSERT(expr) ((void)0) - -#elif defined(BOOST_ENABLE_ASSERT_HANDLER) - -#include <boost/current_function.hpp> - -namespace boost -{ - -void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined - -} // namespace boost - -#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__)) - -#else -# include <assert.h> -# define BOOST_ASSERT(expr) assert(expr) -#endif diff --git a/src/boost/format.hpp b/src/boost/format.hpp deleted file mode 100644 index f965f0f33e9a..000000000000 --- a/src/boost/format.hpp +++ /dev/null @@ -1,64 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// format.hpp : primary header -// ---------------------------------------------------------------------------- - -#ifndef BOOST_FORMAT_HPP -#define BOOST_FORMAT_HPP - -#include <vector> -#include <string> -#include <sstream> -#include <cassert> - -#if HAVE_LOCALE -#include <locale> -#else -#define BOOST_NO_STD_LOCALE -#define BOOST_NO_LOCALE_ISIDIGIT -#include <cctype> -#endif - -#include <boost/format/macros_default.hpp> - - -// **** Forward declarations ---------------------------------- -#include <boost/format/format_fwd.hpp> // basic_format<Ch,Tr>, and other frontends -#include <boost/format/internals_fwd.hpp> // misc forward declarations for internal use - - -// **** Auxiliary structs (stream_format_state<Ch,Tr> , and format_item<Ch,Tr> ) -#include <boost/format/internals.hpp> - -// **** Format class interface -------------------------------- -#include <boost/format/format_class.hpp> - -// **** Exceptions ----------------------------------------------- -#include <boost/format/exceptions.hpp> - -// **** Implementation ------------------------------------------- -//#include <boost/format/format_implementation.hpp> // member functions - -#include <boost/format/group.hpp> // class for grouping arguments - -#include <boost/format/feed_args.hpp> // argument-feeding functions -//#include <boost/format/parsing.hpp> // format-string parsing (member-)functions - -// **** Implementation of the free functions ---------------------- -//#include <boost/format/free_funcs.hpp> - - -#endif // BOOST_FORMAT_HPP diff --git a/src/boost/format/exceptions.hpp b/src/boost/format/exceptions.hpp deleted file mode 100644 index a7641458c95e..000000000000 --- a/src/boost/format/exceptions.hpp +++ /dev/null @@ -1,96 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// exceptions.hpp -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_EXCEPTIONS_HPP -#define BOOST_FORMAT_EXCEPTIONS_HPP - - -#include <stdexcept> - - -namespace boost { - -namespace io { - -// **** exceptions ----------------------------------------------- - -class format_error : public std::exception -{ -public: - format_error() { abort(); } - virtual const char *what() const throw() - { - return "boost::format_error: " - "format generic failure"; - } -}; - -class bad_format_string : public format_error -{ -public: - bad_format_string() { abort(); } - virtual const char *what() const throw() - { - return "boost::bad_format_string: " - "format-string is ill-formed"; - } -}; - -class too_few_args : public format_error -{ -public: - too_few_args() { abort(); } - virtual const char *what() const throw() - { - return "boost::too_few_args: " - "format-string refered to more arguments than were passed"; - } -}; - -class too_many_args : public format_error -{ -public: - too_many_args() { abort(); } - virtual const char *what() const throw() - { - return "boost::too_many_args: " - "format-string refered to less arguments than were passed"; - } -}; - - -class out_of_range : public format_error -{ -public: - out_of_range() { abort(); } - virtual const char *what() const throw() - { - return "boost::out_of_range: " - "tried to refer to an argument (or item) number which is out of range, " - "according to the format string."; - } -}; - - -} // namespace io - -} // namespace boost - - -#endif // BOOST_FORMAT_EXCEPTIONS_HPP diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp deleted file mode 100644 index cdd57fdf2bf1..000000000000 --- a/src/boost/format/feed_args.hpp +++ /dev/null @@ -1,254 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// feed_args.hpp : functions for processing each argument -// (feed, feed_manip, and distribute) -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_FEED_ARGS_HPP -#define BOOST_FORMAT_FEED_ARGS_HPP - -#include "boost/format/format_class.hpp" -#include "boost/format/group.hpp" - -#include "boost/throw_exception.hpp" - -namespace boost { -namespace io { -namespace detail { -namespace { - - inline - void empty_buf(BOOST_IO_STD ostringstream & os) { - static const std::string emptyStr; - os.str(emptyStr); - } - - void do_pad( std::string & s, - std::streamsize w, - const char c, - std::ios::fmtflags f, - bool center) - __attribute__ ((unused)); - - void do_pad( std::string & s, - std::streamsize w, - const char c, - std::ios::fmtflags f, - bool center) - // applies centered / left / right padding to the string s. - // Effects : string s is padded. - { - std::streamsize n=w-s.size(); - if(n<=0) { - return; - } - if(center) - { - s.reserve(w); // allocate once for the 2 inserts - const std::streamsize n1 = n /2, n0 = n - n1; - s.insert(s.begin(), n0, c); - s.append(n1, c); - } - else - { - if(f & std::ios::left) { - s.append(n, c); - } - else { - s.insert(s.begin(), n, c); - } - } - } // -do_pad(..) - - - template<class T> inline - void put_head(BOOST_IO_STD ostream& , const T& ) { - } - - template<class T> inline - void put_head( BOOST_IO_STD ostream& os, const group1<T>& x ) { - os << group_head(x.a1_); // send the first N-1 items, not the last - } - - template<class T> inline - void put_last( BOOST_IO_STD ostream& os, const T& x ) { - os << x ; - } - - template<class T> inline - void put_last( BOOST_IO_STD ostream& os, const group1<T>& x ) { - os << group_last(x.a1_); // this selects the last element - } - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - template<class T> inline - void put_head( BOOST_IO_STD ostream& , T& ) { - } - - template<class T> inline - void put_last( BOOST_IO_STD ostream& os, T& x ) { - os << x ; - } -#endif - - - - -template<class T> -void put( T x, - const format_item& specs, - std::string & res, - BOOST_IO_STD ostringstream& oss_ ) -{ - // does the actual conversion of x, with given params, into a string - // using the *supplied* strinstream. (the stream state is important) - - typedef std::string string_t; - typedef format_item format_item_t; - - stream_format_state prev_state(oss_); - - specs.state_.apply_on(oss_); - - // in case x is a group, apply the manip part of it, - // in order to find width - put_head( oss_, x ); - empty_buf( oss_); - - const std::streamsize w=oss_.width(); - const std::ios::fmtflags fl=oss_.flags(); - const bool internal = (fl & std::ios::internal) != 0; - const bool two_stepped_padding = internal - && ! ( specs.pad_scheme_ & format_item_t::spacepad ) - && specs.truncate_ < 0 ; - - - if(! two_stepped_padding) - { - if(w>0) // handle simple padding via do_pad, not natively in stream - oss_.width(0); - put_last( oss_, x); - res = oss_.str(); - - if (specs.truncate_ >= 0) - res.erase(specs.truncate_); - - // complex pads : - if(specs.pad_scheme_ & format_item_t::spacepad) - { - if( res.size()==0 || ( res[0]!='+' && res[0]!='-' )) - { - res.insert(res.begin(), 1, ' '); // insert 1 space at pos 0 - } - } - if(w > 0) // need do_pad - { - do_pad(res,w,oss_.fill(), fl, (specs.pad_scheme_ & format_item_t::centered) !=0 ); - } - } - else // 2-stepped padding - { - put_last( oss_, x); // oss_.width() may result in padding. - res = oss_.str(); - - if (specs.truncate_ >= 0) - res.erase(specs.truncate_); - - if( res.size() - w > 0) - { // length w exceeded - // either it was multi-output with first output padding up all width.. - // either it was one big arg and we are fine. - empty_buf( oss_); - oss_.width(0); - put_last(oss_, x ); - string_t tmp = oss_.str(); // minimal-length output - std::streamsize d; - if( (d=w - tmp.size()) <=0 ) - { - // minimal length is already >= w, so no padding (cool!) - res.swap(tmp); - } - else - { // hum.. we need to pad (it was necessarily multi-output) - typedef typename string_t::size_type size_type; - size_type i = 0; - while( i<tmp.size() && tmp[i] == res[i] ) // find where we should pad. - ++i; - tmp.insert(i, static_cast<size_type>( d ), oss_.fill()); - res.swap( tmp ); - } - } - else - { // okay, only one thing was printed and padded, so res is fine. - } - } - - prev_state.apply_on(oss_); - empty_buf( oss_); - oss_.clear(); -} // end- put(..) - - -} // local namespace - - - - - -template<class T> -void distribute(basic_format& self, T x) - // call put(x, ..) on every occurence of the current argument : -{ - if(self.cur_arg_ >= self.num_args_) - { - if( self.exceptions() & too_many_args_bit ) - boost::throw_exception(too_many_args()); // too many variables have been supplied ! - else return; - } - for(unsigned long i=0; i < self.items_.size(); ++i) - { - if(self.items_[i].argN_ == self.cur_arg_) - { - put<T> (x, self.items_[i], self.items_[i].res_, self.oss_ ); - } - } -} - -template<class T> -basic_format& feed(basic_format& self, T x) -{ - if(self.dumped_) self.clear(); - distribute<T> (self, x); - ++self.cur_arg_; - if(self.bound_.size() != 0) - { - while( self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_] ) - ++self.cur_arg_; - } - - // this arg is finished, reset the stream's format state - self.state0_.apply_on(self.oss_); - return self; -} - - -} // namespace detail -} // namespace io -} // namespace boost - - -#endif // BOOST_FORMAT_FEED_ARGS_HPP diff --git a/src/boost/format/format_class.hpp b/src/boost/format/format_class.hpp deleted file mode 100644 index 6875623acb47..000000000000 --- a/src/boost/format/format_class.hpp +++ /dev/null @@ -1,135 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// format_class.hpp : class interface -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_CLASS_HPP -#define BOOST_FORMAT_CLASS_HPP - -#include <vector> -#include <string> - -#include <boost/format/format_fwd.hpp> -#include <boost/format/internals_fwd.hpp> - -#include <boost/format/internals.hpp> - -namespace boost { - -class basic_format -{ -public: - typedef std::string string_t; - typedef BOOST_IO_STD ostringstream internal_stream_t; -private: - typedef BOOST_IO_STD ostream stream_t; - typedef io::detail::stream_format_state stream_format_state; - typedef io::detail::format_item format_item_t; - -public: - basic_format(const char* str); - basic_format(const string_t& s); -#ifndef BOOST_NO_STD_LOCALE - basic_format(const char* str, const std::locale & loc); - basic_format(const string_t& s, const std::locale & loc); -#endif // no locale - basic_format(const basic_format& x); - basic_format& operator= (const basic_format& x); - - basic_format& clear(); // empty the string buffers (except bound arguments, see clear_binds() ) - - // pass arguments through those operators : - template<class T> basic_format& operator%(const T& x) - { - return io::detail::feed<const T&>(*this,x); - } - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - template<class T> basic_format& operator%(T& x) - { - return io::detail::feed<T&>(*this,x); - } -#endif - - - // system for binding arguments : - template<class T> - basic_format& bind_arg(int argN, const T& val) - { - return io::detail::bind_arg_body(*this, argN, val); - } - basic_format& clear_bind(int argN); - basic_format& clear_binds(); - - // modify the params of a directive, by applying a manipulator : - template<class T> - basic_format& modify_item(int itemN, const T& manipulator) - { - return io::detail::modify_item_body(*this, itemN, manipulator) ; - } - - // Choosing which errors will throw exceptions : - unsigned char exceptions() const; - unsigned char exceptions(unsigned char newexcept); - - // final output - string_t str() const; - friend BOOST_IO_STD ostream& - operator<< ( BOOST_IO_STD ostream& , const basic_format& ); - - - template<class T> friend basic_format& - io::detail::feed(basic_format&, T); - - template<class T> friend - void io::detail::distribute(basic_format&, T); - - template<class T> friend - basic_format& io::detail::modify_item_body(basic_format&, int, const T&); - - template<class T> friend - basic_format& io::detail::bind_arg_body(basic_format&, int, const T&); - -// make the members private only if the friend templates are supported -private: - - // flag bits, used for style_ - enum style_values { ordered = 1, // set only if all directives are positional directives - special_needs = 4 }; - - // parse the format string : - void parse(const string_t&); - - int style_; // style of format-string : positional or not, etc - int cur_arg_; // keep track of wich argument will come - int num_args_; // number of expected arguments - mutable bool dumped_; // true only after call to str() or << - std::vector<format_item_t> items_; // vector of directives (aka items) - string_t prefix_; // piece of string to insert before first item - - std::vector<bool> bound_; // stores which arguments were bound - // size = num_args OR zero - internal_stream_t oss_; // the internal stream. - stream_format_state state0_; // reference state for oss_ - unsigned char exceptions_; -}; // class basic_format - - -} // namespace boost - - -#endif // BOOST_FORMAT_CLASS_HPP diff --git a/src/boost/format/format_fwd.hpp b/src/boost/format/format_fwd.hpp deleted file mode 100644 index 97c55f6684c3..000000000000 --- a/src/boost/format/format_fwd.hpp +++ /dev/null @@ -1,49 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// format_fwd.hpp : forward declarations, for primary header format.hpp -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_FWD_HPP -#define BOOST_FORMAT_FWD_HPP - -#include <string> -#include <iosfwd> - -namespace boost { - -class basic_format; - -typedef basic_format format; - -namespace io { -enum format_error_bits { bad_format_string_bit = 1, - too_few_args_bit = 2, too_many_args_bit = 4, - out_of_range_bit = 8, - all_error_bits = 255, no_error_bits=0 }; - -// Convertion: format to string -std::string str(const basic_format& ) ; - -} // namespace io - - -BOOST_IO_STD ostream& -operator<<( BOOST_IO_STD ostream&, const basic_format&); - - -} // namespace boost - -#endif // BOOST_FORMAT_FWD_HPP diff --git a/src/boost/format/format_implementation.cc b/src/boost/format/format_implementation.cc deleted file mode 100644 index aa191afe1132..000000000000 --- a/src/boost/format/format_implementation.cc +++ /dev/null @@ -1,256 +0,0 @@ -// -*- C++ -*- -// Boost general library format --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// format_implementation.hpp Implementation of the basic_format class -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_IMPLEMENTATION_HPP -#define BOOST_FORMAT_IMPLEMENTATION_HPP - -#include <boost/throw_exception.hpp> -#include <boost/assert.hpp> -#include <boost/format.hpp> - -namespace boost { - -// -------- format:: ------------------------------------------- -basic_format::basic_format(const char* str) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - state0_.set_by_stream(oss_); - string_t emptyStr; - if( !str) str = emptyStr.c_str(); - parse( str ); -} - -#ifndef BOOST_NO_STD_LOCALE -basic_format::basic_format(const char* str, const std::locale & loc) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - oss_.imbue( loc ); - state0_.set_by_stream(oss_); - string_t emptyStr; - if( !str) str = emptyStr.c_str(); - parse( str ); -} - -basic_format::basic_format(const string_t& s, const std::locale & loc) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - oss_.imbue( loc ); - state0_.set_by_stream(oss_); - parse(s); -} -#endif //BOOST_NO_STD_LOCALE - -basic_format::basic_format(const string_t& s) - : style_(0), cur_arg_(0), num_args_(0), dumped_(false), - items_(), oss_(), exceptions_(io::all_error_bits) -{ - state0_.set_by_stream(oss_); - parse(s); -} - -basic_format:: basic_format(const basic_format& x) - : style_(x.style_), cur_arg_(x.cur_arg_), num_args_(x.num_args_), dumped_(false), - items_(x.items_), prefix_(x.prefix_), bound_(x.bound_), - oss_(), // <- we obviously can't copy x.oss_ - state0_(x.state0_), exceptions_(x.exceptions_) -{ - state0_.apply_on(oss_); -} - -basic_format& basic_format::operator= (const basic_format& x) -{ - if(this == &x) - return *this; - state0_ = x.state0_; - state0_.apply_on(oss_); - - // plus all the other (trivial) assignments : - exceptions_ = x.exceptions_; - items_ = x.items_; - prefix_ = x.prefix_; - bound_=x.bound_; - style_=x.style_; - cur_arg_=x.cur_arg_; - num_args_=x.num_args_; - dumped_=x.dumped_; - return *this; -} - - -unsigned char basic_format::exceptions() const -{ - return exceptions_; -} - -unsigned char basic_format::exceptions(unsigned char newexcept) -{ - unsigned char swp = exceptions_; - exceptions_ = newexcept; - return swp; -} - - -basic_format& basic_format ::clear() - // empty the string buffers (except bound arguments, see clear_binds() ) - // and make the format object ready for formatting a new set of arguments -{ - BOOST_ASSERT( bound_.size()==0 || num_args_ == static_cast<int>(bound_.size()) ); - - for(unsigned long i=0; i<items_.size(); ++i){ - items_[i].state_ = items_[i].ref_state_; - // clear converted strings only if the corresponding argument is not bound : - if( bound_.size()==0 || !bound_[ items_[i].argN_ ] ) items_[i].res_.resize(0); - } - cur_arg_=0; dumped_=false; - // maybe first arg is bound: - if(bound_.size() != 0) - { - while(cur_arg_ < num_args_ && bound_[cur_arg_] ) ++cur_arg_; - } - return *this; -} - -basic_format& basic_format ::clear_binds() - // cancel all bindings, and clear() -{ - bound_.resize(0); - clear(); - return *this; -} - -basic_format& basic_format::clear_bind(int argN) - // cancel the binding of ONE argument, and clear() -{ - if(argN<1 || argN > num_args_ || bound_.size()==0 || !bound_[argN-1] ) - { - if( exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // arg not in range. - else return *this; - } - bound_[argN-1]=false; - clear(); - return *this; -} - - - -std::string basic_format::str() const -{ - dumped_=true; - if(items_.size()==0) - return prefix_; - if( cur_arg_ < num_args_) - if( exceptions() & io::too_few_args_bit ) - boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! - - unsigned long sz = prefix_.size(); - unsigned long i; - for(i=0; i < items_.size(); ++i) - sz += items_[i].res_.size() + items_[i].appendix_.size(); - string_t res; - res.reserve(sz); - - res += prefix_; - for(i=0; i < items_.size(); ++i) - { - const format_item_t& item = items_[i]; - res += item.res_; - if( item.argN_ == format_item_t::argN_tabulation) - { - BOOST_ASSERT( item.pad_scheme_ & format_item_t::tabulation); - std::streamsize n = item.state_.width_ - res.size(); - if( n > 0 ) - res.append( n, item.state_.fill_ ); - } - res += item.appendix_; - } - return res; -} - -namespace io { -namespace detail { - -template<class T> -basic_format& bind_arg_body( basic_format& self, - int argN, - const T& val) - // bind one argument to a fixed value - // this is persistent over clear() calls, thus also over str() and << -{ - if(self.dumped_) self.clear(); // needed, because we will modify cur_arg_.. - if(argN<1 || argN > self.num_args_) - { - if( self.exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // arg not in range. - else return self; - } - if(self.bound_.size()==0) - self.bound_.assign(self.num_args_,false); - else - BOOST_ASSERT( self.num_args_ == static_cast<signed int>(self.bound_.size()) ); - int o_cur_arg = self.cur_arg_; - self.cur_arg_ = argN-1; // arrays begin at 0 - - self.bound_[self.cur_arg_]=false; // if already set, we unset and re-sets.. - self.operator%(val); // put val at the right place, because cur_arg is set - - - // Now re-position cur_arg before leaving : - self.cur_arg_ = o_cur_arg; - self.bound_[argN-1]=true; - if(self.cur_arg_ == argN-1 ) - // hum, now this arg is bound, so move to next free arg - { - while(self.cur_arg_ < self.num_args_ && self.bound_[self.cur_arg_]) ++self.cur_arg_; - } - // In any case, we either have all args, or are on a non-binded arg : - BOOST_ASSERT( self.cur_arg_ >= self.num_args_ || ! self.bound_[self.cur_arg_]); - return self; -} - -template<class T> -basic_format& modify_item_body( basic_format& self, - int itemN, - const T& manipulator) - // applies a manipulator to the format_item describing a given directive. - // this is a permanent change, clear or clear_binds won't cancel that. -{ - if(itemN<1 || itemN >= static_cast<signed int>(self.items_.size() )) - { - if( self.exceptions() & io::out_of_range_bit ) - boost::throw_exception(io::out_of_range()); // item not in range. - else return self; - } - self.items_[itemN-1].ref_state_.apply_manip( manipulator ); - self.items_[itemN-1].state_ = self.items_[itemN-1].ref_state_; - return self; -} - -} // namespace detail - -} // namespace io - -} // namespace boost - - - -#endif // BOOST_FORMAT_IMPLEMENTATION_HPP diff --git a/src/boost/format/free_funcs.cc b/src/boost/format/free_funcs.cc deleted file mode 100644 index 151db37a0ac9..000000000000 --- a/src/boost/format/free_funcs.cc +++ /dev/null @@ -1,71 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// free_funcs.hpp : implementation of the free functions declared in namespace format -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_FUNCS_HPP -#define BOOST_FORMAT_FUNCS_HPP - -#include "boost/format.hpp" -#include "boost/throw_exception.hpp" - -namespace boost { - -namespace io { - inline - std::string str(const basic_format& f) - // adds up all pieces of strings and converted items, and return the formatted string - { - return f.str(); - } -} // - namespace io - -BOOST_IO_STD ostream& -operator<<( BOOST_IO_STD ostream& os, - const boost::basic_format& f) - // effect: "return os << str(f);" but we can try to do it faster -{ - typedef boost::basic_format format_t; - if(f.items_.size()==0) - os << f.prefix_; - else { - if(f.cur_arg_ < f.num_args_) - if( f.exceptions() & io::too_few_args_bit ) - boost::throw_exception(io::too_few_args()); // not enough variables have been supplied ! - if(f.style_ & format_t::special_needs) - os << f.str(); - else { - // else we dont have to count chars output, so we dump directly to os : - os << f.prefix_; - for(unsigned long i=0; i<f.items_.size(); ++i) - { - const format_t::format_item_t& item = f.items_[i]; - os << item.res_; - os << item.appendix_; - - } - } - } - f.dumped_=true; - return os; -} - - - -} // namespace boost - - -#endif // BOOST_FORMAT_FUNCS_HPP diff --git a/src/boost/format/group.hpp b/src/boost/format/group.hpp deleted file mode 100644 index ac63f3f0bab0..000000000000 --- a/src/boost/format/group.hpp +++ /dev/null @@ -1,680 +0,0 @@ - -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- - -// group.hpp : encapsulates a group of manipulators along with an argument -// -// group_head : cut the last element of a group out. -// (is overloaded below on each type of group) - -// group_last : returns the last element of a group -// (is overloaded below on each type of group) - -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_GROUP_HPP -#define BOOST_FORMAT_GROUP_HPP - - -namespace boost { -namespace io { - - -namespace detail { - - -// empty group, but useful even though. -struct group0 -{ - group0() {} -}; - -template <class Ch, class Tr> -inline -BOOST_IO_STD ostream& -operator << ( BOOST_IO_STD ostream& os, - const group0& ) -{ - return os; -} - -template <class T1> -struct group1 -{ - T1 a1_; - group1(T1 a1) - : a1_(a1) - {} -}; - -template <class Ch, class Tr, class T1> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group1<T1>& x) -{ - os << x.a1_; - return os; -} - - - - -template <class T1,class T2> -struct group2 -{ - T1 a1_; - T2 a2_; - group2(T1 a1,T2 a2) - : a1_(a1),a2_(a2) - {} -}; - -template <class Ch, class Tr, class T1,class T2> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group2<T1,T2>& x) -{ - os << x.a1_<< x.a2_; - return os; -} - -template <class T1,class T2,class T3> -struct group3 -{ - T1 a1_; - T2 a2_; - T3 a3_; - group3(T1 a1,T2 a2,T3 a3) - : a1_(a1),a2_(a2),a3_(a3) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group3<T1,T2,T3>& x) -{ - os << x.a1_<< x.a2_<< x.a3_; - return os; -} - -template <class T1,class T2,class T3,class T4> -struct group4 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - group4(T1 a1,T2 a2,T3 a3,T4 a4) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group4<T1,T2,T3,T4>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5> -struct group5 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - group5(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group5<T1,T2,T3,T4,T5>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6> -struct group6 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - group6(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group6<T1,T2,T3,T4,T5,T6>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> -struct group7 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - group7(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group7<T1,T2,T3,T4,T5,T6,T7>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -struct group8 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - group8(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group8<T1,T2,T3,T4,T5,T6,T7,T8>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -struct group9 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - T9 a9_; - group9(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group9<T1,T2,T3,T4,T5,T6,T7,T8,T9>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_; - return os; -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -struct group10 -{ - T1 a1_; - T2 a2_; - T3 a3_; - T4 a4_; - T5 a5_; - T6 a6_; - T7 a7_; - T8 a8_; - T9 a9_; - T10 a10_; - group10(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9,T10 a10) - : a1_(a1),a2_(a2),a3_(a3),a4_(a4),a5_(a5),a6_(a6),a7_(a7),a8_(a8),a9_(a9),a10_(a10) - {} -}; - -template <class Ch, class Tr, class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -inline -BOOST_IO_STD ostream& -operator << (BOOST_IO_STD ostream& os, - const group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10>& x) -{ - os << x.a1_<< x.a2_<< x.a3_<< x.a4_<< x.a5_<< x.a6_<< x.a7_<< x.a8_<< x.a9_<< x.a10_; - return os; -} - - - - -template <class T1,class T2> -inline -group1<T1> -group_head( group2<T1,T2> const& x) -{ - return group1<T1> (x.a1_); -} - -template <class T1,class T2> -inline -group1<T2> -group_last( group2<T1,T2> const& x) -{ - return group1<T2> (x.a2_); -} - - - -template <class T1,class T2,class T3> -inline -group2<T1,T2> -group_head( group3<T1,T2,T3> const& x) -{ - return group2<T1,T2> (x.a1_,x.a2_); -} - -template <class T1,class T2,class T3> -inline -group1<T3> -group_last( group3<T1,T2,T3> const& x) -{ - return group1<T3> (x.a3_); -} - - - -template <class T1,class T2,class T3,class T4> -inline -group3<T1,T2,T3> -group_head( group4<T1,T2,T3,T4> const& x) -{ - return group3<T1,T2,T3> (x.a1_,x.a2_,x.a3_); -} - -template <class T1,class T2,class T3,class T4> -inline -group1<T4> -group_last( group4<T1,T2,T3,T4> const& x) -{ - return group1<T4> (x.a4_); -} - - - -template <class T1,class T2,class T3,class T4,class T5> -inline -group4<T1,T2,T3,T4> -group_head( group5<T1,T2,T3,T4,T5> const& x) -{ - return group4<T1,T2,T3,T4> (x.a1_,x.a2_,x.a3_,x.a4_); -} - -template <class T1,class T2,class T3,class T4,class T5> -inline -group1<T5> -group_last( group5<T1,T2,T3,T4,T5> const& x) -{ - return group1<T5> (x.a5_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6> -inline -group5<T1,T2,T3,T4,T5> -group_head( group6<T1,T2,T3,T4,T5,T6> const& x) -{ - return group5<T1,T2,T3,T4,T5> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6> -inline -group1<T6> -group_last( group6<T1,T2,T3,T4,T5,T6> const& x) -{ - return group1<T6> (x.a6_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> -inline -group6<T1,T2,T3,T4,T5,T6> -group_head( group7<T1,T2,T3,T4,T5,T6,T7> const& x) -{ - return group6<T1,T2,T3,T4,T5,T6> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7> -inline -group1<T7> -group_last( group7<T1,T2,T3,T4,T5,T6,T7> const& x) -{ - return group1<T7> (x.a7_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -inline -group7<T1,T2,T3,T4,T5,T6,T7> -group_head( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x) -{ - return group7<T1,T2,T3,T4,T5,T6,T7> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8> -inline -group1<T8> -group_last( group8<T1,T2,T3,T4,T5,T6,T7,T8> const& x) -{ - return group1<T8> (x.a8_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -inline -group8<T1,T2,T3,T4,T5,T6,T7,T8> -group_head( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x) -{ - return group8<T1,T2,T3,T4,T5,T6,T7,T8> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9> -inline -group1<T9> -group_last( group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> const& x) -{ - return group1<T9> (x.a9_); -} - - - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -inline -group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> -group_head( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x) -{ - return group9<T1,T2,T3,T4,T5,T6,T7,T8,T9> (x.a1_,x.a2_,x.a3_,x.a4_,x.a5_,x.a6_,x.a7_,x.a8_,x.a9_); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9,class T10> -inline -group1<T10> -group_last( group10<T1,T2,T3,T4,T5,T6,T7,T8,T9,T10> const& x) -{ - return group1<T10> (x.a10_); -} - - - - - -} // namespace detail - - - -// helper functions - - -inline detail::group1< detail::group0 > -group() { return detail::group1< detail::group0 > ( detail::group0() ); } - -template <class T1, class Var> -inline -detail::group1< detail::group2<T1, Var const&> > - group(T1 a1, Var const& var) -{ - return detail::group1< detail::group2<T1, Var const&> > - ( detail::group2<T1, Var const&> - (a1, var) - ); -} - -template <class T1,class T2, class Var> -inline -detail::group1< detail::group3<T1,T2, Var const&> > - group(T1 a1,T2 a2, Var const& var) -{ - return detail::group1< detail::group3<T1,T2, Var const&> > - ( detail::group3<T1,T2, Var const&> - (a1,a2, var) - ); -} - -template <class T1,class T2,class T3, class Var> -inline -detail::group1< detail::group4<T1,T2,T3, Var const&> > - group(T1 a1,T2 a2,T3 a3, Var const& var) -{ - return detail::group1< detail::group4<T1,T2,T3, Var const&> > - ( detail::group4<T1,T2,T3, Var const&> - (a1,a2,a3, var) - ); -} - -template <class T1,class T2,class T3,class T4, class Var> -inline -detail::group1< detail::group5<T1,T2,T3,T4, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4, Var const& var) -{ - return detail::group1< detail::group5<T1,T2,T3,T4, Var const&> > - ( detail::group5<T1,T2,T3,T4, Var const&> - (a1,a2,a3,a4, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5, class Var> -inline -detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var const& var) -{ - return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var const&> > - ( detail::group6<T1,T2,T3,T4,T5, Var const&> - (a1,a2,a3,a4,a5, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6, class Var> -inline -detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var const& var) -{ - return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var const&> > - ( detail::group7<T1,T2,T3,T4,T5,T6, Var const&> - (a1,a2,a3,a4,a5,a6, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> -inline -detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var const& var) -{ - return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> > - ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var const&> - (a1,a2,a3,a4,a5,a6,a7, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> -inline -detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var const& var) -{ - return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> > - ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var const&> - (a1,a2,a3,a4,a5,a6,a7,a8, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> -inline -detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var const& var) -{ - return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> > - ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var const&> - (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) - ); -} - - -#ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - -template <class T1, class Var> -inline -detail::group1< detail::group2<T1, Var&> > - group(T1 a1, Var& var) -{ - return detail::group1< detail::group2<T1, Var&> > - ( detail::group2<T1, Var&> - (a1, var) - ); -} - -template <class T1,class T2, class Var> -inline -detail::group1< detail::group3<T1,T2, Var&> > - group(T1 a1,T2 a2, Var& var) -{ - return detail::group1< detail::group3<T1,T2, Var&> > - ( detail::group3<T1,T2, Var&> - (a1,a2, var) - ); -} - -template <class T1,class T2,class T3, class Var> -inline -detail::group1< detail::group4<T1,T2,T3, Var&> > - group(T1 a1,T2 a2,T3 a3, Var& var) -{ - return detail::group1< detail::group4<T1,T2,T3, Var&> > - ( detail::group4<T1,T2,T3, Var&> - (a1,a2,a3, var) - ); -} - -template <class T1,class T2,class T3,class T4, class Var> -inline -detail::group1< detail::group5<T1,T2,T3,T4, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4, Var& var) -{ - return detail::group1< detail::group5<T1,T2,T3,T4, Var&> > - ( detail::group5<T1,T2,T3,T4, Var&> - (a1,a2,a3,a4, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5, class Var> -inline -detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5, Var& var) -{ - return detail::group1< detail::group6<T1,T2,T3,T4,T5, Var&> > - ( detail::group6<T1,T2,T3,T4,T5, Var&> - (a1,a2,a3,a4,a5, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6, class Var> -inline -detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6, Var& var) -{ - return detail::group1< detail::group7<T1,T2,T3,T4,T5,T6, Var&> > - ( detail::group7<T1,T2,T3,T4,T5,T6, Var&> - (a1,a2,a3,a4,a5,a6, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7, class Var> -inline -detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7, Var& var) -{ - return detail::group1< detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> > - ( detail::group8<T1,T2,T3,T4,T5,T6,T7, Var&> - (a1,a2,a3,a4,a5,a6,a7, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8, class Var> -inline -detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8, Var& var) -{ - return detail::group1< detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> > - ( detail::group9<T1,T2,T3,T4,T5,T6,T7,T8, Var&> - (a1,a2,a3,a4,a5,a6,a7,a8, var) - ); -} - -template <class T1,class T2,class T3,class T4,class T5,class T6,class T7,class T8,class T9, class Var> -inline -detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> > - group(T1 a1,T2 a2,T3 a3,T4 a4,T5 a5,T6 a6,T7 a7,T8 a8,T9 a9, Var& var) -{ - return detail::group1< detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> > - ( detail::group10<T1,T2,T3,T4,T5,T6,T7,T8,T9, Var&> - (a1,a2,a3,a4,a5,a6,a7,a8,a9, var) - ); -} - - -#endif //end- #ifndef BOOST_NO_OVERLOAD_FOR_NON_CONST - - -} // namespace io - -} // namespace boost - - -#endif // BOOST_FORMAT_GROUP_HPP diff --git a/src/boost/format/internals.hpp b/src/boost/format/internals.hpp deleted file mode 100644 index d25eb4c864c4..000000000000 --- a/src/boost/format/internals.hpp +++ /dev/null @@ -1,167 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream - -// ---------------------------------------------------------------------------- -// internals.hpp : internal structs. included by format.hpp -// stream_format_state, and format_item -// ---------------------------------------------------------------------------- - - -#ifndef BOOST_FORMAT_INTERNALS_HPP -#define BOOST_FORMAT_INTERNALS_HPP - - -#include <string> -#include <sstream> - -namespace boost { -namespace io { -namespace detail { - - -// -------------- -// set of params that define the format state of a stream - -struct stream_format_state -{ - typedef std::ios basic_ios; - - std::streamsize width_; - std::streamsize precision_; - char fill_; - std::ios::fmtflags flags_; - - stream_format_state() : width_(-1), precision_(-1), fill_(0), flags_(std::ios::dec) {} - stream_format_state(basic_ios& os) {set_by_stream(os); } - - void apply_on(basic_ios & os) const; //- applies format_state to the stream - template<class T> void apply_manip(T manipulator) //- modifies state by applying manipulator. - { apply_manip_body<T>( *this, manipulator) ; } - void reset(); //- sets to default state. - void set_by_stream(const basic_ios& os); //- sets to os's state. -}; - - - -// -------------- -// format_item : stores all parameters that can be defined by directives in the format-string - -struct format_item -{ - enum pad_values { zeropad = 1, spacepad =2, centered=4, tabulation = 8 }; - - enum arg_values { argN_no_posit = -1, // non-positional directive. argN will be set later. - argN_tabulation = -2, // tabulation directive. (no argument read) - argN_ignored = -3 // ignored directive. (no argument read) - }; - typedef BOOST_IO_STD ios basic_ios; - typedef detail::stream_format_state stream_format_state; - typedef std::string string_t; - typedef BOOST_IO_STD ostringstream internal_stream_t; - - - int argN_; //- argument number (starts at 0, eg : %1 => argN=0) - // negative values are used for items that don't process - // an argument - string_t res_; //- result of the formatting of this item - string_t appendix_; //- piece of string between this item and the next - - stream_format_state ref_state_;// set by parsing the format_string, is only affected by modify_item - stream_format_state state_; // always same as ref_state, _unless_ modified by manipulators 'group(..)' - - // non-stream format-state parameters - signed int truncate_; //- is >=0 for directives like %.5s (take 5 chars from the string) - unsigned int pad_scheme_; //- several possible padding schemes can mix. see pad_values - - format_item() : argN_(argN_no_posit), truncate_(-1), pad_scheme_(0) {} - - void compute_states(); // sets states according to truncate and pad_scheme. -}; - - - -// ----------------------------------------------------------- -// Definitions -// ----------------------------------------------------------- - -// --- stream_format_state:: ------------------------------------------- -inline -void stream_format_state::apply_on(basic_ios & os) const - // set the state of this stream according to our params -{ - if(width_ != -1) - os.width(width_); - if(precision_ != -1) - os.precision(precision_); - if(fill_ != 0) - os.fill(fill_); - os.flags(flags_); -} - -inline -void stream_format_state::set_by_stream(const basic_ios& os) - // set our params according to the state of this stream -{ - flags_ = os.flags(); - width_ = os.width(); - precision_ = os.precision(); - fill_ = os.fill(); -} - -template<class T> inline -void apply_manip_body( stream_format_state& self, - T manipulator) - // modify our params according to the manipulator -{ - BOOST_IO_STD stringstream ss; - self.apply_on( ss ); - ss << manipulator; - self.set_by_stream( ss ); -} - -inline -void stream_format_state::reset() - // set our params to standard's default state -{ - width_=-1; precision_=-1; fill_=0; - flags_ = std::ios::dec; -} - - -// --- format_items:: ------------------------------------------- -inline -void format_item::compute_states() - // reflect pad_scheme_ on state_ and ref_state_ - // because some pad_schemes has complex consequences on several state params. -{ - if(pad_scheme_ & zeropad) - { - if(ref_state_.flags_ & std::ios::left) - { - pad_scheme_ = pad_scheme_ & (~zeropad); // ignore zeropad in left alignment - } - else - { - ref_state_.fill_='0'; - ref_state_.flags_ |= std::ios::internal; - } - } - state_ = ref_state_; -} - - -} } } // namespaces boost :: io :: detail - - -#endif // BOOST_FORMAT_INTERNALS_HPP diff --git a/src/boost/format/internals_fwd.hpp b/src/boost/format/internals_fwd.hpp deleted file mode 100644 index a8ebf7c3abc1..000000000000 --- a/src/boost/format/internals_fwd.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// internals_fwd.hpp : forward declarations, for internal headers -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_INTERNAL_FWD_HPP -#define BOOST_FORMAT_INTERNAL_FWD_HPP - -#include "boost/format/format_fwd.hpp" - - -namespace boost { -namespace io { - -namespace detail { - struct stream_format_state; - struct format_item; -} - - -namespace detail { - - // these functions were intended as methods, - // but MSVC have problems with template member functions : - - // defined in format_implementation.hpp : - template<class T> - basic_format& modify_item_body( basic_format& self, - int itemN, const T& manipulator); - - template<class T> - basic_format& bind_arg_body( basic_format& self, - int argN, const T& val); - - template<class T> - void apply_manip_body( stream_format_state& self, - T manipulator); - - // argument feeding (defined in feed_args.hpp ) : - template<class T> - void distribute(basic_format& self, T x); - - template<class T> - basic_format& feed(basic_format& self, T x); - -} // namespace detail - -} // namespace io -} // namespace boost - - -#endif // BOOST_FORMAT_INTERNAL_FWD_HPP diff --git a/src/boost/format/local.mk b/src/boost/format/local.mk deleted file mode 100644 index 3776eff382fe..000000000000 --- a/src/boost/format/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -libraries += libformat - -libformat_NAME = libnixformat - -libformat_DIR := $(d) - -libformat_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/boost/format/macros_default.hpp b/src/boost/format/macros_default.hpp deleted file mode 100644 index 4fd84a163fb3..000000000000 --- a/src/boost/format/macros_default.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rdiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// macros_default.hpp : configuration for the format library -// provides default values for the stl workaround macros -// ------------------------------------------------------------------------------ - -#ifndef BOOST_FORMAT_MACROS_DEFAULT_HPP -#define BOOST_FORMAT_MACROS_DEFAULT_HPP - -// *** This should go to "boost/config/suffix.hpp". - -#ifndef BOOST_IO_STD -# define BOOST_IO_STD std:: -#endif - -// **** Workaround for io streams, stlport and msvc. -#ifdef BOOST_IO_NEEDS_USING_DECLARATION -namespace boost { - using std::char_traits; - using std::basic_ostream; - using std::basic_ostringstream; - namespace io { - using std::basic_ostream; - namespace detail { - using std::basic_ios; - using std::basic_ostream; - using std::basic_ostringstream; - } - } -} -#endif - -// ------------------------------------------------------------------------------ - -#endif // BOOST_FORMAT_MACROS_DEFAULT_HPP diff --git a/src/boost/format/parsing.cc b/src/boost/format/parsing.cc deleted file mode 100644 index 34c36adeb734..000000000000 --- a/src/boost/format/parsing.cc +++ /dev/null @@ -1,454 +0,0 @@ -// -*- C++ -*- -// Boost general library 'format' --------------------------- -// See http://www.boost.org for updates, documentation, and revision history. - -// (C) Samuel Krempp 2001 -// krempp@crans.ens-cachan.fr -// Permission to copy, use, modify, sell and -// distribute this software is granted provided this copyright notice appears -// in all copies. This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. - -// ideas taken from Rudiger Loos's format class -// and Karl Nelson's ofstream (also took its parsing code as basis for printf parsing) - -// ------------------------------------------------------------------------------ -// parsing.hpp : implementation of the parsing member functions -// ( parse, parse_printf_directive) -// ------------------------------------------------------------------------------ - - -#ifndef BOOST_FORMAT_PARSING_HPP -#define BOOST_FORMAT_PARSING_HPP - - -#include <boost/format.hpp> -#include <boost/throw_exception.hpp> -#include <boost/assert.hpp> - - -namespace boost { -namespace io { -namespace detail { - - template<class Stream> inline - bool wrap_isdigit(char c, Stream &os) - { -#ifndef BOOST_NO_LOCALE_ISIDIGIT - return std::isdigit(c, os.rdbuf()->getloc() ); -# else - using namespace std; - return isdigit(c); -#endif - } //end- wrap_isdigit(..) - - template<class Res> inline - Res str2int(const std::string& s, - std::string::size_type start, - BOOST_IO_STD ios &os, - const Res = Res(0) ) - // Input : char string, with starting index - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // Effects : reads s[start:] and converts digits into an integral n, of type Res - // Returns : n - { - Res n = 0; - while(start<s.size() && wrap_isdigit(s[start], os) ) { - char cur_ch = s[start]; - BOOST_ASSERT(cur_ch != 0 ); // since we called isdigit, this should not happen. - n *= 10; - n += cur_ch - '0'; // 22.2.1.1.2 of the C++ standard - ++start; - } - return n; - } - - void skip_asterisk(const std::string & buf, - std::string::size_type * pos_p, - BOOST_IO_STD ios &os) - // skip printf's "asterisk-fields" directives in the format-string buf - // Input : char string, with starting index *pos_p - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // Effects : advance *pos_p by skipping printf's asterisk fields. - // Returns : nothing - { - using namespace std; - BOOST_ASSERT( pos_p != 0); - if(*pos_p >= buf.size() ) return; - if(buf[ *pos_p]=='*') { - ++ (*pos_p); - while (*pos_p < buf.size() && wrap_isdigit(buf[*pos_p],os)) ++(*pos_p); - if(buf[*pos_p]=='$') ++(*pos_p); - } - } - - - inline void maybe_throw_exception( unsigned char exceptions) - // auxiliary func called by parse_printf_directive - // for centralising error handling - // it either throws if user sets the corresponding flag, or does nothing. - { - if(exceptions & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); - } - - - - bool parse_printf_directive(const std::string & buf, - std::string::size_type * pos_p, - detail::format_item * fpar, - BOOST_IO_STD ios &os, - unsigned char exceptions) - // Input : a 'printf-directive' in the format-string, starting at buf[ *pos_p ] - // a basic_ios& merely to call its widen/narrow member function in the desired locale. - // a bitset'excpetions' telling whether to throw exceptions on errors. - // Returns : true if parse somehow succeeded (possibly ignoring errors if exceptions disabled) - // false if it failed so bad that the directive should be printed verbatim - // Effects : - *pos_p is incremented so that buf[*pos_p] is the first char after the directive - // - *fpar is set with the parameters read in the directive - { - typedef format_item format_item_t; - BOOST_ASSERT( pos_p != 0); - std::string::size_type &i1 = *pos_p, - i0; - fpar->argN_ = format_item_t::argN_no_posit; // if no positional-directive - - bool in_brackets=false; - if(buf[i1]=='|') - { - in_brackets=true; - if( ++i1 >= buf.size() ) { - maybe_throw_exception(exceptions); - return false; - } - } - - // the flag '0' would be picked as a digit for argument order, but here it's a flag : - if(buf[i1]=='0') - goto parse_flags; - - // handle argument order (%2$d) or possibly width specification: %2d - i0 = i1; // save position before digits - while (i1 < buf.size() && wrap_isdigit(buf[i1], os)) - ++i1; - if (i1!=i0) - { - if( i1 >= buf.size() ) { - maybe_throw_exception(exceptions); - return false; - } - int n=str2int(buf,i0, os, int(0) ); - - // %N% case : this is already the end of the directive - if( buf[i1] == '%' ) - { - fpar->argN_ = n-1; - ++i1; - if( in_brackets) - maybe_throw_exception(exceptions); - // but don't return. maybe "%" was used in lieu of '$', so we go on. - else return true; - } - - if ( buf[i1]=='$' ) - { - fpar->argN_ = n-1; - ++i1; - } - else - { - // non-positionnal directive - fpar->ref_state_.width_ = n; - fpar->argN_ = format_item_t::argN_no_posit; - goto parse_precision; - } - } - - parse_flags: - // handle flags - while ( i1 <buf.size()) // as long as char is one of + - = # 0 l h or ' ' - { - // misc switches - switch (buf[i1]) - { - case '\'' : break; // no effect yet. (painful to implement) - case 'l': - case 'h': // short/long modifier : for printf-comaptibility (no action needed) - break; - case '-': - fpar->ref_state_.flags_ |= std::ios::left; - break; - case '=': - fpar->pad_scheme_ |= format_item_t::centered; - break; - case ' ': - fpar->pad_scheme_ |= format_item_t::spacepad; - break; - case '+': - fpar->ref_state_.flags_ |= std::ios::showpos; - break; - case '0': - fpar->pad_scheme_ |= format_item_t::zeropad; - // need to know alignment before really setting flags, - // so just add 'zeropad' flag for now, it will be processed later. - break; - case '#': - fpar->ref_state_.flags_ |= std::ios::showpoint | std::ios::showbase; - break; - default: - goto parse_width; - } - ++i1; - } // loop on flag. - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - - parse_width: - // handle width spec - skip_asterisk(buf, &i1, os); // skips 'asterisk fields' : *, or *N$ - i0 = i1; // save position before digits - while (i1<buf.size() && wrap_isdigit(buf[i1], os)) - i1++; - - if (i1!=i0) - { fpar->ref_state_.width_ = str2int( buf,i0, os, std::streamsize(0) ); } - - parse_precision: - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - // handle precision spec - if (buf[i1]=='.') - { - ++i1; - skip_asterisk(buf, &i1, os); - i0 = i1; // save position before digits - while (i1<buf.size() && wrap_isdigit(buf[i1], os)) - ++i1; - - if(i1==i0) - fpar->ref_state_.precision_ = 0; - else - fpar->ref_state_.precision_ = str2int(buf,i0, os, std::streamsize(0) ); - } - - // handle formatting-type flags : - while( i1<buf.size() && - ( buf[i1]=='l' || buf[i1]=='L' || buf[i1]=='h') ) - ++i1; - if( i1>=buf.size()) { - maybe_throw_exception(exceptions); - return true; - } - - if( in_brackets && buf[i1]=='|' ) - { - ++i1; - return true; - } - switch (buf[i1]) - { - case 'X': - fpar->ref_state_.flags_ |= std::ios::uppercase; - case 'p': // pointer => set hex. - case 'x': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::hex; - break; - - case 'o': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::oct; - break; - - case 'E': - fpar->ref_state_.flags_ |= std::ios::uppercase; - case 'e': - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - fpar->ref_state_.flags_ |= std::ios::scientific; - - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - break; - - case 'f': - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - fpar->ref_state_.flags_ |= std::ios::fixed; - case 'u': - case 'd': - case 'i': - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - break; - - case 'T': - ++i1; - if( i1 >= buf.size()) - maybe_throw_exception(exceptions); - else - fpar->ref_state_.fill_ = buf[i1]; - fpar->pad_scheme_ |= format_item_t::tabulation; - fpar->argN_ = format_item_t::argN_tabulation; - break; - case 't': - fpar->ref_state_.fill_ = ' '; - fpar->pad_scheme_ |= format_item_t::tabulation; - fpar->argN_ = format_item_t::argN_tabulation; - break; - - case 'G': - fpar->ref_state_.flags_ |= std::ios::uppercase; - break; - case 'g': // 'g' conversion is default for floats. - fpar->ref_state_.flags_ &= ~std::ios::basefield; - fpar->ref_state_.flags_ |= std::ios::dec; - - // CLEAR all floatield flags, so stream will CHOOSE - fpar->ref_state_.flags_ &= ~std::ios::floatfield; - break; - - case 'C': - case 'c': - fpar->truncate_ = 1; - break; - case 'S': - case 's': - fpar->truncate_ = fpar->ref_state_.precision_; - fpar->ref_state_.precision_ = -1; - break; - case 'n' : - fpar->argN_ = format_item_t::argN_ignored; - break; - default: - maybe_throw_exception(exceptions); - } - ++i1; - - if( in_brackets ) - { - if( i1<buf.size() && buf[i1]=='|' ) - { - ++i1; - return true; - } - else maybe_throw_exception(exceptions); - } - return true; - } - -} // detail namespace -} // io namespace - - -// ----------------------------------------------- -// format :: parse(..) - -void basic_format::parse(const string_t & buf) - // parse the format-string -{ - using namespace std; - const char arg_mark = '%'; - bool ordered_args=true; - int max_argN=-1; - string_t::size_type i1=0; - int num_items=0; - - // A: find upper_bound on num_items and allocates arrays - i1=0; - while( (i1=buf.find(arg_mark,i1)) != string::npos ) - { - if( i1+1 >= buf.size() ) { - if(exceptions() & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); // must not end in "bla bla %" - else break; // stop there, ignore last '%' - } - if(buf[i1+1] == buf[i1] ) { i1+=2; continue; } // escaped "%%" / "##" - ++i1; - - // in case of %N% directives, dont count it double (wastes allocations..) : - while(i1 < buf.size() && io::detail::wrap_isdigit(buf[i1],oss_)) ++i1; - if( i1 < buf.size() && buf[i1] == arg_mark ) ++ i1; - - ++num_items; - } - items_.assign( num_items, format_item_t() ); - - // B: Now the real parsing of the format string : - num_items=0; - i1 = 0; - string_t::size_type i0 = i1; - bool special_things=false; - int cur_it=0; - while( (i1=buf.find(arg_mark,i1)) != string::npos ) - { - string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; - - if( buf[i1+1] == buf[i1] ) // escaped mark, '%%' - { - piece += buf.substr(i0, i1-i0) + buf[i1]; - i1+=2; i0=i1; - continue; - } - BOOST_ASSERT( static_cast<unsigned int>(cur_it) < items_.size() || cur_it==0); - - if(i1!=i0) piece += buf.substr(i0, i1-i0); - ++i1; - - bool parse_ok; - parse_ok = io::detail::parse_printf_directive(buf, &i1, &items_[cur_it], oss_, exceptions()); - if( ! parse_ok ) continue; // the directive will be printed verbatim - - i0=i1; - items_[cur_it].compute_states(); // process complex options, like zeropad, into stream params. - - int argN=items_[cur_it].argN_; - if(argN == format_item_t::argN_ignored) - continue; - if(argN ==format_item_t::argN_no_posit) - ordered_args=false; - else if(argN == format_item_t::argN_tabulation) special_things=true; - else if(argN > max_argN) max_argN = argN; - ++num_items; - ++cur_it; - } // loop on %'s - BOOST_ASSERT(cur_it == num_items); - - // store the final piece of string - string_t & piece = (cur_it==0) ? prefix_ : items_[cur_it-1].appendix_; - piece += buf.substr(i0); - - if( !ordered_args) - { - if(max_argN >= 0 ) // dont mix positional with non-positionnal directives - { - if(exceptions() & io::bad_format_string_bit) - boost::throw_exception(io::bad_format_string()); - // else do nothing. => positionnal arguments are processed as non-positionnal - } - // set things like it would have been with positional directives : - int non_ordered_items = 0; - for(int i=0; i< num_items; ++i) - if(items_[i].argN_ == format_item_t::argN_no_posit) - { - items_[i].argN_ = non_ordered_items; - ++non_ordered_items; - } - max_argN = non_ordered_items-1; - } - - // C: set some member data : - items_.resize(num_items); - - if(special_things) style_ |= special_needs; - num_args_ = max_argN + 1; - if(ordered_args) style_ |= ordered; - else style_ &= ~ordered; -} - -} // namespace boost - - -#endif // BOOST_FORMAT_PARSING_HPP diff --git a/src/boost/throw_exception.hpp b/src/boost/throw_exception.hpp deleted file mode 100644 index 07b4ae5ceae7..000000000000 --- a/src/boost/throw_exception.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED -#define BOOST_THROW_EXCEPTION_HPP_INCLUDED - -// MS compatible compilers support #pragma once - -#if defined(_MSC_VER) && (_MSC_VER >= 1020) -# pragma once -#endif - -// -// boost/throw_exception.hpp -// -// Copyright (c) 2002 Peter Dimov and Multi Media Ltd. -// -// Permission to copy, use, modify, sell and distribute this software -// is granted provided this copyright notice appears in all copies. -// This software is provided "as is" without express or implied -// warranty, and with no claim as to its suitability for any purpose. -// -// http://www.boost.org/libs/utility/throw_exception.html -// - -//#include <boost/config.hpp> - -#ifdef BOOST_NO_EXCEPTIONS -# include <exception> -#endif - -namespace boost -{ - -#ifdef BOOST_NO_EXCEPTIONS - -void throw_exception(std::exception const & e); // user defined - -#else - -template<class E> void throw_exception(E const & e) -{ - throw e; -} - -#endif - -} // namespace boost - -#endif // #ifndef BOOST_THROW_EXCEPTION_HPP_INCLUDED diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 5ec49b5252d2..38dbe3e58b26 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -193,8 +193,10 @@ int main (int argc, char * * argv) storeUri = bestMachine->storeUri; } catch (std::exception & e) { - printError("unable to open SSH connection to '%s': %s; trying other available machines...", - bestMachine->storeUri, e.what()); + auto msg = chomp(drainFD(5, false)); + printError("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + (msg.empty() ? "" : ": " + msg)); bestMachine->enabled = false; continue; } @@ -204,6 +206,8 @@ int main (int argc, char * * argv) } connected: + close(5); + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings<PathSet>(source); @@ -246,7 +250,7 @@ connected: if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute); + copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } return; diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 64368a43ff73..50b0409d1886 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -4,6 +4,6 @@ build-remote_DIR := $(d) build-remote_INSTALL_DIR := $(libexecdir)/nix -build-remote_LIBS = libmain libutil libformat libstore +build-remote_LIBS = libmain libformat libstore libutil build-remote_SOURCES := $(d)/build-remote.cc diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc deleted file mode 100644 index eddb9fdaa8d2..000000000000 --- a/src/buildenv/buildenv.cc +++ /dev/null @@ -1,187 +0,0 @@ -#include "shared.hh" -#include <sys/stat.h> -#include <sys/types.h> -#include <fcntl.h> -#include <algorithm> - -using namespace nix; - -typedef std::map<Path,int> Priorities; - -static bool isDirectory (const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting status of '%1%'") % path); - return S_ISDIR(st.st_mode); -} - -static auto priorities = Priorities{}; - -static auto symlinks = 0; - -/* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) -{ - auto srcFiles = readDirectory(srcDir); - for (const auto & ent : srcFiles) { - if (ent.name[0] == '.') - /* not matched by glob */ - continue; - const auto & srcFile = srcDir + "/" + ent.name; - auto dstFile = dstDir + "/" + ent.name; - - /* The files below are special-cased to that they don't show up - * in user profiles, either because they are useless, or - * because they would cauase pointless collisions (e.g., each - * Python package brings its own - * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) - */ - if (hasSuffix(srcFile, "/propagated-build-inputs") || - hasSuffix(srcFile, "/nix-support") || - hasSuffix(srcFile, "/perllocal.pod") || - hasSuffix(srcFile, "/info/dir") || - hasSuffix(srcFile, "/log")) { - continue; - } else if (isDirectory(srcFile)) { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { - if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); - continue; - } else if (S_ISLNK(dstSt.st_mode)) { - auto target = readLink(dstFile); - if (!isDirectory(target)) - throw Error(format("collision between '%1%' and non-directory '%2%'") - % srcFile % target); - if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); - if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); - continue; - } - } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); - } else { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { - if (S_ISLNK(dstSt.st_mode)) { - auto target = readLink(dstFile); - auto prevPriority = priorities[dstFile]; - if (prevPriority == priority) - throw Error(format( - "Packages '%1%' and '%2%' have the same priority '%3%'" - "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "to change the priority of one of the conflicting packages" - " ('0' being the highest priority)" - ) % srcFile % target % priority); - if (prevPriority < priority) - continue; - if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); - } - } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); - } - createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; - } -} - -typedef std::set<Path> FileProp; - -static auto done = FileProp{}; -static auto postponed = FileProp{}; - -static auto out = string{}; - -static void addPkg(const Path & pkgDir, int priority) -{ - if (done.find(pkgDir) != done.end()) - return; - done.insert(pkgDir); - createLinks(pkgDir, out, priority); - auto propagatedFN = pkgDir + "/nix-support/propagated-user-env-packages"; - auto propagated = string{}; - { - AutoCloseFD fd = open(propagatedFN.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) { - if (errno == ENOENT) - return; - throw SysError(format("opening '%1%'") % propagatedFN); - } - propagated = readFile(fd.get()); - } - for (const auto & p : tokenizeString<std::vector<string>>(propagated, " \n")) - if (done.find(p) == done.end()) - postponed.insert(p); -} - -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{std::move(path)}, active{active}, priority{priority} {} -}; - -typedef std::vector<Package> Packages; - -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - out = getEnv("out"); - if (mkdir(out.c_str(), 0755) == -1) - throw SysError(format("creating %1%") % out); - - /* Convert the stuff we get from the environment back into a coherent - * data type. - */ - auto pkgs = Packages{}; - auto derivations = tokenizeString<Strings>(getEnv("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); - } - } - - /* Symlink to the packages that have been installed explicitly by the - * user. Process in priority order to reduce unnecessary - * symlink/unlink steps. - */ - std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { - return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); - }); - for (const auto & pkg : pkgs) - if (pkg.active) - addPkg(pkg.path, pkg.priority); - - /* Symlink to the packages that have been "propagated" by packages - * installed by the user (i.e., package X declares that it wants Y - * installed as well). We do these later because they have a lower - * priority in case of collisions. - */ - auto priorityCounter = 1000; - while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; - for (const auto & pkgDir : pkgDirs) - addPkg(pkgDir, priorityCounter++); - } - - std::cerr << "created " << symlinks << " symlinks in user environment\n"; - - createSymlink(getEnv("manifest"), out + "/manifest.nix"); - }); -} - diff --git a/src/buildenv/local.mk b/src/buildenv/local.mk deleted file mode 100644 index 17ec13b235f4..000000000000 --- a/src/buildenv/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += buildenv - -buildenv_DIR := $(d) - -buildenv_INSTALL_DIR := $(libexecdir)/nix - -buildenv_LIBS = libmain libstore libutil libformat - -buildenv_SOURCES := $(d)/buildenv.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 63afccbec188..8548990795ab 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -313,7 +313,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); } - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs")); + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); if (settings.restrictEval || settings.pureEval) { allowedPaths = PathSet(); @@ -628,7 +628,7 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, Pos * pos) { - if (pos) { + if (pos && pos->file.set()) { mkAttrs(v, 3); mkString(*allocAttr(v, sFile), pos->file); mkInt(*allocAttr(v, sLine), pos->line); @@ -1316,7 +1316,8 @@ void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists, co auto out = v.listElems(); for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { unsigned int l = lists[n]->listSize(); - memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); + if (l) + memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); pos += l; } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index e5e01fb5831a..1e9c29afa133 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -85,6 +85,7 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) %} +ANY .|\n ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* INT [0-9]+ FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? @@ -146,8 +147,8 @@ or { return OR_KW; } <INITIAL,INSIDE_DOLLAR_CURLY>\" { PUSH_STATE(STRING); return '"'; } -<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | -<STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ @@ -178,7 +179,7 @@ or { return OR_KW; } yylval->e = new ExprIndStr("''"); return IND_STR; } -<IND_STRING>\'\'\\. { +<IND_STRING>\'\'\\{ANY} { yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } @@ -208,7 +209,7 @@ or { return OR_KW; } \#[^\r\n]* /* single-line comments */ \/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ -. return yytext[0]; +{ANY} return yytext[0]; } diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 21b6c38dd133..79f3e2f4506e 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix +Cflags: -I${includedir}/nix -std=c++14 diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 7b0a127cd41c..63cbef1ddf84 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -10,7 +10,7 @@ namespace nix { /* Displaying abstract syntax trees. */ -std::ostream & operator << (std::ostream & str, Expr & e) +std::ostream & operator << (std::ostream & str, const Expr & e) { e.show(str); return str; @@ -58,48 +58,48 @@ std::ostream & operator << (std::ostream & str, const Symbol & sym) return str; } -void Expr::show(std::ostream & str) +void Expr::show(std::ostream & str) const { abort(); } -void ExprInt::show(std::ostream & str) +void ExprInt::show(std::ostream & str) const { str << n; } -void ExprFloat::show(std::ostream & str) +void ExprFloat::show(std::ostream & str) const { str << nf; } -void ExprString::show(std::ostream & str) +void ExprString::show(std::ostream & str) const { showString(str, s); } -void ExprPath::show(std::ostream & str) +void ExprPath::show(std::ostream & str) const { str << s; } -void ExprVar::show(std::ostream & str) +void ExprVar::show(std::ostream & str) const { str << name; } -void ExprSelect::show(std::ostream & str) +void ExprSelect::show(std::ostream & str) const { str << "(" << *e << ")." << showAttrPath(attrPath); if (def) str << " or (" << *def << ")"; } -void ExprOpHasAttr::show(std::ostream & str) +void ExprOpHasAttr::show(std::ostream & str) const { str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; } -void ExprAttrs::show(std::ostream & str) +void ExprAttrs::show(std::ostream & str) const { if (recursive) str << "rec "; str << "{ "; @@ -113,7 +113,7 @@ void ExprAttrs::show(std::ostream & str) str << "}"; } -void ExprList::show(std::ostream & str) +void ExprList::show(std::ostream & str) const { str << "[ "; for (auto & i : elems) @@ -121,7 +121,7 @@ void ExprList::show(std::ostream & str) str << "]"; } -void ExprLambda::show(std::ostream & str) +void ExprLambda::show(std::ostream & str) const { str << "("; if (matchAttrs) { @@ -143,7 +143,7 @@ void ExprLambda::show(std::ostream & str) str << ": " << *body << ")"; } -void ExprLet::show(std::ostream & str) +void ExprLet::show(std::ostream & str) const { str << "(let "; for (auto & i : attrs->attrs) @@ -155,27 +155,27 @@ void ExprLet::show(std::ostream & str) str << "in " << *body << ")"; } -void ExprWith::show(std::ostream & str) +void ExprWith::show(std::ostream & str) const { str << "(with " << *attrs << "; " << *body << ")"; } -void ExprIf::show(std::ostream & str) +void ExprIf::show(std::ostream & str) const { str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; } -void ExprAssert::show(std::ostream & str) +void ExprAssert::show(std::ostream & str) const { str << "assert " << *cond << "; " << *body; } -void ExprOpNot::show(std::ostream & str) +void ExprOpNot::show(std::ostream & str) const { str << "(! " << *e << ")"; } -void ExprConcatStrings::show(std::ostream & str) +void ExprConcatStrings::show(std::ostream & str) const { bool first = true; str << "("; @@ -186,7 +186,7 @@ void ExprConcatStrings::show(std::ostream & str) str << ")"; } -void ExprPos::show(std::ostream & str) +void ExprPos::show(std::ostream & str) const { str << "__curPos"; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 30be79bb57a6..8c8f39640681 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -76,17 +76,17 @@ string showAttrPath(const AttrPath & attrPath); struct Expr { virtual ~Expr() { }; - virtual void show(std::ostream & str); + virtual void show(std::ostream & str) const; virtual void bindVars(const StaticEnv & env); virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol & name); }; -std::ostream & operator << (std::ostream & str, Expr & e); +std::ostream & operator << (std::ostream & str, const Expr & e); #define COMMON_METHODS \ - void show(std::ostream & str); \ + void show(std::ostream & str) const; \ void eval(EvalState & state, Env & env, Value & v); \ void bindVars(const StaticEnv & env); @@ -283,13 +283,13 @@ struct ExprOpNot : Expr }; #define MakeBinOp(name, s) \ - struct Expr##name : Expr \ + struct name : Expr \ { \ Pos pos; \ Expr * e1, * e2; \ - Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ - Expr##name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ - void show(std::ostream & str) \ + name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ + name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ + void show(std::ostream & str) const \ { \ str << "(" << *e1 << " " s " " << *e2 << ")"; \ } \ @@ -300,14 +300,14 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(App, "") -MakeBinOp(OpEq, "==") -MakeBinOp(OpNEq, "!=") -MakeBinOp(OpAnd, "&&") -MakeBinOp(OpOr, "||") -MakeBinOp(OpImpl, "->") -MakeBinOp(OpUpdate, "//") -MakeBinOp(OpConcatLists, "++") +MakeBinOp(ExprApp, "") +MakeBinOp(ExprOpEq, "==") +MakeBinOp(ExprOpNEq, "!=") +MakeBinOp(ExprOpAnd, "&&") +MakeBinOp(ExprOpOr, "||") +MakeBinOp(ExprOpImpl, "->") +MakeBinOp(ExprOpUpdate, "//") +MakeBinOp(ExprOpConcatLists, "++") struct ExprConcatStrings : Expr { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a800d24290ae..57dc7bd1279d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -155,7 +155,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ -static void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); @@ -193,7 +193,7 @@ static void prim_importNative(EvalState & state, const Pos & pos, Value * * args /* Execute a program and parse its output */ -static void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); auto elems = args[0]->listElems(); @@ -271,7 +271,18 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0]); - mkBool(v, args[0]->type == tLambda); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); } @@ -1601,12 +1612,16 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V state.mkAttrs(v, 2); Value * vRight = state.allocAttr(v, state.sRight); - state.mkList(*vRight, right.size()); - memcpy(vRight->listElems(), right.data(), sizeof(Value *) * right.size()); + auto rsize = right.size(); + state.mkList(*vRight, rsize); + if (rsize) + memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize); Value * vWrong = state.allocAttr(v, state.sWrong); - state.mkList(*vWrong, wrong.size()); - memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wrong.size()); + auto wsize = wrong.size(); + state.mkList(*vWrong, wsize); + if (wsize) + memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize); v.attrs->sort(); } @@ -1913,21 +1928,32 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar auto s = state.forceString(*args[2], context, pos); string res; - for (size_t p = 0; p < s.size(); ) { + // Loops one past last character to handle the case where 'from' contains an empty string. + for (size_t p = 0; p <= s.size(); ) { bool found = false; auto i = from.begin(); auto j = to.begin(); for (; i != from.end(); ++i, ++j) if (s.compare(p, i->size(), *i) == 0) { found = true; - p += i->size(); res += j->first; + if (i->empty()) { + if (p < s.size()) + res += s[p]; + p++; + } else { + p += i->size(); + } for (auto& path : j->second) context.insert(path); j->second.clear(); break; } - if (!found) res += s[p++]; + if (!found) { + if (p < s.size()) + res += s[p]; + p++; + } } mkString(v, res, context); @@ -2225,7 +2251,7 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix"; + string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); sDerivationNix = symbols.create(path); evalFile(path, v); addConstant("derivation", v); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 31bf3f84f6c7..c790b30f6d0b 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -15,4 +15,12 @@ struct RegisterPrimOp RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); }; +/* These primops are disabled without enableNativeCode, but plugins + may wish to use them in limited contexts without globally enabling + them. */ +/* Load a ValueInitializer from a DSO and return whatever it initializes */ +void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); +/* Execute a program and parse its output */ +void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); + } diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 2e3e2634db8f..8bb74dad639e 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -79,7 +79,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, ref = "HEAD"s; } - if (!ref) ref = "master"s; + if (!ref) ref = "HEAD"s; if (rev != "" && !std::regex_match(rev, revRegex)) throw Error("invalid Git revision '%s'", rev); @@ -114,7 +114,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime <= now - settings.tarballTtl; + st.st_mtime + settings.tarballTtl <= now; } if (doFetch) { @@ -138,7 +138,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - printTalkative("using revision %s of repo '%s'", uri, gitInfo.rev); + printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 5517d83df824..a75c5fc2ddff 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime <= now - settings.tarballTtl) + st.st_mtime + settings.tarballTtl <= now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index de1bdf706f72..38bc85c484eb 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix +Cflags: -I${includedir}/nix -std=c++14 diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7d888202bbf1..91a4eaf922a6 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -263,7 +263,7 @@ void showManPage(const string & name) { restoreSignals(); setenv("MANPATH", settings.nixManDir.c_str(), 1); - execlp("man", "man", name.c_str(), NULL); + execlp("man", "man", name.c_str(), nullptr); throw SysError(format("command 'man %1%' failed") % name.c_str()); } @@ -325,10 +325,10 @@ RunPager::RunPager() setenv("LESS", "FRSXMK", 1); restoreSignals(); if (pager) - execl("/bin/sh", "sh", "-c", pager, NULL); - execlp("pager", "pager", NULL); - execlp("less", "less", NULL); - execlp("more", "more", NULL); + execl("/bin/sh", "sh", "-c", pager, nullptr); + execlp("pager", "pager", nullptr); + execlp("less", "less", nullptr); + execlp("more", "more", nullptr); throw SysError(format("executing '%1%'") % pager); }); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index d1b278b8efbe..2e9a13e564ca 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -203,22 +203,18 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) stats.narRead++; stats.narReadCompressedBytes += nar->size(); - /* Decompress the NAR. FIXME: would be nice to have the remote - side do this. */ - try { - nar = decompress(info->compression, *nar); - } catch (UnknownCompressionMethod &) { - throw Error(format("binary cache path '%s' uses unknown compression method '%s'") - % storePath % info->compression); - } + uint64_t narSize = 0; - stats.narReadBytes += nar->size(); + StringSource source(*nar); - printMsg(lvlTalkative, format("exporting path '%1%' (%2% bytes)") % storePath % nar->size()); + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + narSize += len; + }); - assert(nar->size() % 8 == 0); + decompress(info->compression, source, wrapperSink); - sink((unsigned char *) nar->c_str(), nar->size()); + stats.narReadBytes += narSize; } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2466f9bd6b06..6108785447a7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -156,7 +156,7 @@ public: abort(); } - void trace(const format & f); + void trace(const FormatOrString & fs); string getName() { @@ -417,9 +417,9 @@ void Goal::amDone(ExitCode result) } -void Goal::trace(const format & f) +void Goal::trace(const FormatOrString & fs) { - debug(format("%1%: %2%") % name % f); + debug("%1%: %2%", name, fs.s); } @@ -652,6 +652,11 @@ HookInstance::HookInstance() if (dup2(builderOut.writeSide.get(), 4) == -1) throw SysError("dupping builder's stdout/stderr"); + /* Hack: pass the read side of that fd to allow build-remote + to read SSH error messages. */ + if (dup2(builderOut.readSide.get(), 5) == -1) + throw SysError("dupping builder's stdout/stderr"); + Strings args = { baseNameOf(settings.buildHook), std::to_string(verbosity), @@ -959,6 +964,8 @@ private: } void done(BuildResult::Status status, const string & msg = ""); + + PathSet exportReferences(PathSet storePaths); }; @@ -1189,7 +1196,7 @@ void DerivationGoal::outputsSubstituted() for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) - throw Error(format("dependency of '%1%' of '%2%' does not exist, and substitution is disabled") + throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled") % i % drvPath); addWaitee(worker.makeSubstitutionGoal(i)); } @@ -1458,7 +1465,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % rand()).str(); + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); if (pathExists(storePath)) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) @@ -1725,22 +1732,23 @@ int childEntry(void * arg) } -PathSet exportReferences(Store & store, PathSet storePaths) +PathSet DerivationGoal::exportReferences(PathSet storePaths) { PathSet paths; for (auto storePath : storePaths) { /* Check that the store path is valid. */ - if (!store.isInStore(storePath)) + if (!worker.store.isInStore(storePath)) throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") % storePath); - storePath = store.toStorePath(storePath); - if (!store.isValidPath(storePath)) - throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") - % storePath); - store.computeFSClosure(storePath, paths); + storePath = worker.store.toStorePath(storePath); + + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath); + + worker.store.computeFSClosure(storePath, paths); } /* If there are derivations in the graph, then include their @@ -1751,9 +1759,9 @@ PathSet exportReferences(Store & store, PathSet storePaths) for (auto & j : paths2) { if (isDerivation(j)) { - Derivation drv = store.derivationFromPath(j); + Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) - store.computeFSClosure(k.second.path, paths); + worker.store.computeFSClosure(k.second.path, paths); } } @@ -1877,7 +1885,7 @@ void DerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences(worker.store, {storePath}), false, false)); + exportReferences({storePath}), false, false)); } } @@ -2379,7 +2387,7 @@ void DerivationGoal::writeStructuredAttrs() for (auto & p : *i) storePaths.insert(p.get<std::string>()); worker.store.pathInfoToJSON(jsonRoot, - exportReferences(worker.store, storePaths), false, true); + exportReferences(storePaths), false, true); } json[i.key()] = nlohmann::json::parse(str.str()); // urgh } @@ -2695,8 +2703,8 @@ void DerivationGoal::runChild() } else { if (errno != EINVAL) throw SysError("mounting /dev/pts"); - doBind("/dev/pts", "/dev/pts"); - doBind("/dev/ptmx", "/dev/ptmx"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); } } @@ -2948,6 +2956,8 @@ void DerivationGoal::runChild() if (drv->builder == "builtin:fetchurl") builtinFetchurl(drv2, netrcData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(drv2); else throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); _exit(0); @@ -3684,7 +3694,7 @@ void SubstitutionGoal::tryNext() only after we've downloaded the path. */ if (worker.store.requireSigs && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.publicKeys)) + && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { printError("warning: substituter '%s' does not have a valid signature for path '%s'", sub->getUri(), storePath); @@ -4158,10 +4168,10 @@ void Worker::waitForInput() assert(goal); set<int> fds2(j->fds); + std::vector<unsigned char> buffer(4096); for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { - unsigned char buffer[4096]; - ssize_t rd = read(k, buffer, sizeof(buffer)); + ssize_t rd = read(k, buffer.data(), buffer.size()); if (rd == -1) { if (errno != EINTR) throw SysError(format("reading from %1%") @@ -4173,7 +4183,7 @@ void Worker::waitForInput() } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); - string data((char *) buffer, rd); + string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); } diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0cc6ba31f658..0d2da873ece4 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -4,6 +4,8 @@ namespace nix { +// TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); +void builtinBuildenv(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc new file mode 100644 index 000000000000..74e706664694 --- /dev/null +++ b/src/libstore/builtins/buildenv.cc @@ -0,0 +1,204 @@ +#include "builtins.hh" + +#include <sys/stat.h> +#include <sys/types.h> +#include <fcntl.h> +#include <algorithm> + +namespace nix { + +typedef std::map<Path,int> Priorities; + +// FIXME: change into local variables. + +static Priorities priorities; + +static unsigned long symlinks; + +/* For each activated package, create symlinks */ +static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +{ + DirEntries srcFiles; + + try { + srcFiles = readDirectory(srcDir); + } catch (SysError & e) { + if (e.errNo == ENOTDIR) { + printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); + return; + } + throw; + } + + for (const auto & ent : srcFiles) { + if (ent.name[0] == '.') + /* not matched by glob */ + continue; + auto srcFile = srcDir + "/" + ent.name; + auto dstFile = dstDir + "/" + ent.name; + + struct stat srcSt; + try { + if (stat(srcFile.c_str(), &srcSt) == -1) + throw SysError("getting status of '%1%'", srcFile); + } catch (SysError & e) { + if (e.errNo == ENOENT || e.errNo == ENOTDIR) { + printError("warning: skipping dangling symlink '%s'", dstFile); + continue; + } + throw; + } + + /* The files below are special-cased to that they don't show up + * in user profiles, either because they are useless, or + * because they would cauase pointless collisions (e.g., each + * Python package brings its own + * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) + */ + if (hasSuffix(srcFile, "/propagated-build-inputs") || + hasSuffix(srcFile, "/nix-support") || + hasSuffix(srcFile, "/perllocal.pod") || + hasSuffix(srcFile, "/info/dir") || + hasSuffix(srcFile, "/log")) + continue; + + else if (S_ISDIR(srcSt.st_mode)) { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISDIR(dstSt.st_mode)) { + createLinks(srcFile, dstFile, priority); + continue; + } else if (S_ISLNK(dstSt.st_mode)) { + auto target = canonPath(dstFile, true); + if (!S_ISDIR(lstat(target).st_mode)) + throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); + if (unlink(dstFile.c_str()) == -1) + throw SysError(format("unlinking '%1%'") % dstFile); + if (mkdir(dstFile.c_str(), 0755) == -1) + throw SysError(format("creating directory '%1%'")); + createLinks(target, dstFile, priorities[dstFile]); + createLinks(srcFile, dstFile, priority); + continue; + } + } else if (errno != ENOENT) + throw SysError(format("getting status of '%1%'") % dstFile); + } + + else { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISLNK(dstSt.st_mode)) { + auto prevPriority = priorities[dstFile]; + if (prevPriority == priority) + throw Error( + "packages '%1%' and '%2%' have the same priority %3%; " + "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "to change the priority of one of the conflicting packages" + " (0 being the highest priority)", + srcFile, readLink(dstFile), priority); + if (prevPriority < priority) + continue; + if (unlink(dstFile.c_str()) == -1) + throw SysError(format("unlinking '%1%'") % dstFile); + } else if (S_ISDIR(dstSt.st_mode)) + throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); + } else if (errno != ENOENT) + throw SysError(format("getting status of '%1%'") % dstFile); + } + + createSymlink(srcFile, dstFile); + priorities[dstFile] = priority; + symlinks++; + } +} + +typedef std::set<Path> FileProp; + +static FileProp done; +static FileProp postponed = FileProp{}; + +static Path out; + +static void addPkg(const Path & pkgDir, int priority) +{ + if (done.count(pkgDir)) return; + done.insert(pkgDir); + createLinks(pkgDir, out, priority); + + try { + for (const auto & p : tokenizeString<std::vector<string>>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; + } +} + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector<Package> Packages; + +void builtinBuildenv(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString<Strings>(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + auto active = derivations.front(); derivations.pop_front(); + auto priority = stoi(derivations.front()); derivations.pop_front(); + auto outputs = stoi(derivations.front()); derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { + return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); + }); + for (const auto & pkg : pkgs) + if (pkg.active) + addPkg(pkg.path, pkg.priority); + + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + auto pkgDirs = postponed; + postponed = FileProp{}; + for (const auto & pkgDir : pkgDirs) + addPkg(pkgDir, priorityCounter++); + } + + printError("created %d symlinks in user environment", symlinks); + + createSymlink(getAttr("manifest"), out + "/manifest.nix"); +} + +} diff --git a/src/libstore/builtins.cc b/src/libstore/builtins/fetchurl.cc index 4ca4a838e3c4..4ca4a838e3c4 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins/fetchurl.cc diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 5ab625f42288..18f9094f82e0 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -173,7 +173,11 @@ struct CurlDownloader : public Downloader int progressCallback(double dltotal, double dlnow) { - act.progress(dlnow, dltotal); + try { + act.progress(dlnow, dltotal); + } catch (nix::Interrupted &) { + assert(_isInterrupted); + } return _isInterrupted; } @@ -195,6 +199,7 @@ struct CurlDownloader : public Downloader if (readOffset == request.data->length()) return 0; auto count = std::min(size * nitems, request.data->length() - readOffset); + assert(count); memcpy(buffer, request.data->data() + readOffset, count); readOffset += count; return count; @@ -339,6 +344,7 @@ struct CurlDownloader : public Downloader case CURLE_BAD_FUNCTION_ARGUMENT: case CURLE_INTERFACE_FAILED: case CURLE_UNKNOWN_OPTION: + case CURLE_SSL_CACERT_BADFILE: err = Misc; break; default: // Shut up warnings @@ -630,7 +636,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (expectedHash) { expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); if (store->isValidPath(expectedStorePath)) - return expectedStorePath; + return store->toRealPath(expectedStorePath); } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -724,8 +730,13 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa storePath = unpackedStorePath; } - if (expectedStorePath != "" && storePath != expectedStorePath) - throw nix::Error("store path mismatch in file downloaded from '%s'", url); + if (expectedStorePath != "" && storePath != expectedStorePath) { + Hash gotHash = unpack + ? hashPath(expectedHash.type, store->toRealPath(storePath)).first + : hashFile(expectedHash.type, store->toRealPath(storePath)); + throw nix::Error("hash mismatch in file downloaded from '%s': got hash '%s' instead of the expected hash '%s'", + url, gotHash.to_string(), expectedHash.to_string()); + } return store->toRealPath(storePath); } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index d9d525d4e65f..0b8d29b21dfe 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -22,7 +22,7 @@ struct DownloadRequest std::string mimeType; DownloadRequest(const std::string & uri) - : uri(uri), parentAct(curActivity) { } + : uri(uri), parentAct(getCurActivity()) { } }; struct DownloadResult diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 943b16c28fa3..ba49749d830a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -59,7 +59,7 @@ static void makeSymlink(const Path & link, const Path & target) /* Create the new symlink. */ Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % rand()).str(); + % link % getpid() % random()).str(); createSymlink(target, tempLink); /* Atomically replace the old one. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f46e8326235f..544566e0b573 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -159,7 +159,7 @@ void initPlugins() void *handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror()); + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); } } /* We handle settings registrations here, since plugins can add settings */ diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 897d53b7b94c..0ae69242a59c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -29,7 +29,7 @@ struct CaseHackSetting : public BaseSetting<bool> void set(const std::string & str) override { BaseSetting<bool>::set(str); - nix::useCaseHack = true; + nix::useCaseHack = value; } }; @@ -320,6 +320,14 @@ public: Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", "Which users or groups are trusted to ask the daemon to do unsafe things."}; + Setting<unsigned int> ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", + "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " + "return an invalid path result"}; + + Setting<unsigned int> ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " + "return a valid path result."}; + /* ?Who we trust to use the daemon in safe ways */ Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", "Which users or groups are allowed to connect to the daemon."}; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dfefdb9bc874..5dee25308f7f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -16,6 +16,7 @@ struct LegacySSHStore : public Store const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; // Hack for getting remote build log output. const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; @@ -55,7 +56,7 @@ struct LegacySSHStore : public Store ref<Connection> openConnection() { auto conn = make_ref<Connection>(); - conn->sshConn = master.startCommand("nix-store --serve --write"); + conn->sshConn = master.startCommand(fmt("%s --serve --write", remoteProgram)); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); @@ -119,7 +120,7 @@ struct LegacySSHStore : public Store }); } - void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override { @@ -130,7 +131,7 @@ struct LegacySSHStore : public Store conn->to << cmdImportPaths << 1; - conn->to(*nar); + copyNAR(source, conn->to); conn->to << exportMagic << info.path @@ -150,12 +151,7 @@ struct LegacySSHStore : public Store conn->to << cmdDumpStorePath << path; conn->to.flush(); - - /* FIXME: inefficient. */ - ParseSink parseSink; /* null sink; just parse the NAR */ - TeeSource savedNAR(conn->from); - parseDump(parseSink, savedNAR); - sink(*savedNAR.data); + copyNAR(conn->from, sink); } PathSet queryAllValidPaths() override { unsupported(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4afe51ea91ec..b63584f28a30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -53,7 +53,6 @@ LocalStore::LocalStore(const Params & params) , trashDir(realStoreDir + "/trash") , tempRootsDir(stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) - , publicKeys(getDefaultPublicKeys()) { auto state(_state.lock()); @@ -964,21 +963,21 @@ void LocalStore::invalidatePath(State & state, const Path & path) } -void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) +const PublicKeys & LocalStore::getPublicKeys() { - assert(info.narHash); + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys()); + return *state->publicKeys; +} - Hash h = hashString(htSHA256, *nar); - if (h != info.narHash) - throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", - info.path, info.narHash.to_string(), h.to_string()); - if (nar->size() != info.narSize) - throw Error("size mismatch importing path '%s'; expected %s, got %s", - info.path, info.narSize, nar->size()); +void LocalStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) +{ + assert(info.narHash); - if (requireSigs && checkSigs && !info.checkSignatures(*this, publicKeys)) + if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) throw Error("cannot add path '%s' because it lacks a valid signature", info.path); addTempRoot(info.path); @@ -999,8 +998,27 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & deletePath(realPath); - StringSource source(*nar); - restorePath(realPath, source); + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(htSHA256); + + LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { + size_t n = source.read(data, len); + hashSink(data, n); + return n; + }); + + restorePath(realPath, wrapperSource); + + auto hashResult = hashSink.finish(); + + if (hashResult.first != info.narHash) + throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", + info.path, info.narHash.to_string(), hashResult.first.to_string()); + + if (hashResult.second != info.narSize) + throw Error("size mismatch importing path '%s'; expected %s, got %s", + info.path, info.narSize, hashResult.second); autoGC(); @@ -1215,7 +1233,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of '%1%'") % i); - HashResult current = hashPath(info->narHash.type, i); + HashResult current = hashPath(info->narHash.type, toRealPath(i)); if (info->narHash != nullHash && info->narHash != current.first) { printError(format("path '%1%' was modified! " diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bbd50e1c1451..1209a06356f7 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -77,6 +77,8 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); + + std::unique_ptr<PublicKeys> publicKeys; }; Sync<State, std::recursive_mutex> _state; @@ -100,7 +102,7 @@ private: settings.requireSigs, "require-sigs", "whether store paths should have a trusted signature on import"}; - PublicKeys publicKeys; + const PublicKeys & getPublicKeys(); public: @@ -143,7 +145,7 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index e11efa5c2b54..a7279aa3939f 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,7 +4,7 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) libstore_LIBS = libutil libformat diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 6e155e877803..35403e5df56f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,6 +1,7 @@ #include "nar-info-disk-cache.hh" #include "sync.hh" #include "sqlite.hh" +#include "globals.hh" #include <sqlite3.h> @@ -47,10 +48,6 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { public: - /* How long negative and positive lookups are valid. */ - const int ttlNegative = 3600; - const int ttlPositive = 30 * 24 * 3600; - /* How often to purge expired entries from the cache. */ const int purgeInterval = 24 * 3600; @@ -116,8 +113,8 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - ttlNegative) - (now - ttlPositive) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); @@ -186,8 +183,8 @@ public: auto queryNAR(state->queryNAR.use() (cache.id) (hashPart) - (now - ttlNegative) - (now - ttlPositive)); + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); if (!queryNAR.next()) return {oUnknown, 0}; @@ -260,11 +257,8 @@ public: ref<NarInfoDiskCache> getNarInfoDiskCache() { - static Sync<std::shared_ptr<NarInfoDiskCache>> cache; - - auto cache_(cache.lock()); - if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>(); - return ref<NarInfoDiskCache>(*cache_); + static ref<NarInfoDiskCache> cache = make_ref<NarInfoDiskCacheImpl>(); + return cache; } } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 3f1a2d83d2f2..5cf22faadcbe 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -5,5 +5,5 @@ includedir=@includedir@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lnixstore -lnixutil -lnixformat -Cflags: -I${includedir}/nix +Libs: -L${libdir} -lnixstore -lnixutil +Cflags: -I${includedir}/nix -std=c++14 diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 891540ae4c1d..7840167d7772 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -213,7 +213,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % rand()).str(); + % realStoreDir % getpid() % random()).str(); if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (errno == EMLINK) { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8f0b65557ac4..080cef93d214 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -377,7 +377,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) } -void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, +void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { auto conn(connections->get()); @@ -385,22 +385,21 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { conn->to << wopImportPaths; - StringSink sink; - sink << 1 // == path follows - ; - assert(nar->size() % 8 == 0); - sink((unsigned char *) nar->data(), nar->size()); - sink - << exportMagic - << info.path - << info.references - << info.deriver - << 0 // == no legacy signature - << 0 // == no path follows - ; - - StringSource source(*sink.s); - conn->processStderr(0, &source); + auto source2 = sinkToSource([&](Sink & sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink + << exportMagic + << info.path + << info.references + << info.deriver + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn->processStderr(0, source2.get()); auto importedPaths = readStorePaths<PathSet>(*this, conn->from); assert(importedPaths.size() <= 1); @@ -412,7 +411,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> << info.references << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca << repair << !checkSigs; - conn->to(*nar); + copyNAR(source, conn->to); conn->processStderr(); } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 7f36e206416b..95fa59a2069d 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -58,7 +58,7 @@ public: void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; - void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; @@ -122,11 +122,12 @@ protected: ref<Pool<Connection>> connections; + virtual void setOptions(Connection & conn); + private: std::atomic_bool failed{false}; - void setOptions(Connection & conn); }; class UDSRemoteStore : public LocalFSStore, public RemoteStore diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index b13001b06d57..42d40e71d8be 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -7,7 +7,7 @@ namespace nix { -[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) { int err = sqlite3_errcode(db); @@ -21,7 +21,7 @@ namespace nix { : fmt("SQLite database '%s' is busy", path)); } else - throw SQLiteError("%s: %s (in '%s')", f.str(), sqlite3_errstr(err), path); + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(err), path); } SQLite::SQLite(const Path & path) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 14a7a0dd8996..115679b84159 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -93,7 +93,7 @@ struct SQLiteTxn MakeError(SQLiteError, Error); MakeError(SQLiteBusy, SQLiteError); -[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); +[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); void handleSQLiteBusy(const SQLiteBusy & e); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 107c6e1ecb4d..39205ae2ce12 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -51,21 +51,16 @@ private: std::string host; SSHMaster master; -}; - -class ForwardSource : public Source -{ - Source & readSource; - Sink & writeSink; -public: - ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {} - size_t read(unsigned char * data, size_t len) override + void setOptions(RemoteStore::Connection & conn) override { - auto n = readSource.read(data, len); - writeSink(data, n); - return n; - } + /* TODO Add a way to explicitly ask for some options to be + forwarded. One option: A way to query the daemon for its + settings, and then a series of params to SSHStore like + forward-cores or forward-overridden-cores that only + override the requested settings. + */ + }; }; void SSHStore::narFromPath(const Path & path, Sink & sink) @@ -73,9 +68,7 @@ void SSHStore::narFromPath(const Path & path, Sink & sink) auto conn(connections->get()); conn->to << wopNarFromPath << path; conn->processStderr(); - ParseSink ps; - auto fwd = ForwardSource(conn->from, sink); - parseDump(ps, fwd); + copyNAR(conn->from, sink); } ref<FSAccessor> SSHStore::getFSAccessor() diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 7ff7a9bffc49..033c580936ad 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -49,6 +49,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); + if (verbosity >= lvlChatty) + args.push_back("-v"); args.push_back(command); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); @@ -93,6 +95,8 @@ Path SSHMaster::startMaster() , "-o", "LocalCommand=echo started" , "-o", "PermitLocalCommand=yes" }; + if (verbosity >= lvlChatty) + args.push_back("-v"); addCommonSSHOpts(args); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8830edcc3449..1a0d12ca78c2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -590,32 +590,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - auto progress = [&](size_t len) { - total += len; - act.progress(total, info->narSize); - }; - - struct MyStringSink : StringSink - { - typedef std::function<void(size_t)> Callback; - Callback callback; - MyStringSink(Callback callback) : callback(callback) { } - void operator () (const unsigned char * data, size_t len) override - { - StringSink::operator ()(data, len); - callback(len); - }; - }; - - MyStringSink sink(progress); - srcStore->narFromPath({storePath}, sink); - + // FIXME +#if 0 if (!info->narHash) { auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); info = info2; } +#endif if (info->ultimate) { auto info2 = make_ref<ValidPathInfo>(*info); @@ -623,7 +606,16 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, info = info2; } - dstStore->addToStore(*info, sink.s, repair, checkSigs); + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }); + + dstStore->addToStore(*info, *source, repair, checkSigs); } @@ -765,7 +757,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; Hash hash(std::string(ca, recursive ? 8 : 6)); - if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) + if (references.empty() && + store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) return true; else warn(); @@ -808,6 +801,21 @@ std::string makeFixedOutputCA(bool recursive, const Hash & hash) } +void Store::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) +{ + addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor); +} + +void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) +{ + StringSource source(*nar); + addToStore(info, source, repair, checkSigs, accessor); +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 563aa566bd37..ea259f07e8ab 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -399,9 +399,14 @@ public: virtual bool wantMassQuery() { return false; } /* Import a path into the store. */ + virtual void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr<FSAccessor> accessor = 0); + + // FIXME: remove virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0) = 0; + std::shared_ptr<FSAccessor> accessor = 0); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index f71229d8fdd6..154e2d20430c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -40,14 +40,14 @@ static void dumpContents(const Path & path, size_t size, AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) throw SysError(format("opening file '%1%'") % path); - unsigned char buf[65536]; + std::vector<unsigned char> buf(65536); size_t left = size; while (left > 0) { - size_t n = left > sizeof(buf) ? sizeof(buf) : left; - readFull(fd.get(), buf, n); + auto n = std::min(left, buf.size()); + readFull(fd.get(), buf.data(), n); left -= n; - sink(buf, n); + sink(buf.data(), n); } writePadding(size, sink); @@ -146,14 +146,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) sink.preallocateContents(size); unsigned long long left = size; - unsigned char buf[65536]; + std::vector<unsigned char> buf(65536); while (left) { checkInterrupt(); - unsigned int n = sizeof(buf); - if ((unsigned long long) n > left) n = left; - source(buf, n); - sink.receiveContents(buf, n); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); left -= n; } @@ -350,4 +350,21 @@ void restorePath(const Path & path, Source & source) } +void copyNAR(Source & source, Sink & sink) +{ + // FIXME: if 'source' is the output of dumpPath() followed by EOF, + // we should just forward all data directly without parsing. + + ParseSink parseSink; /* null sink; just parse the NAR */ + + LambdaSource wrapper([&](unsigned char * data, size_t len) { + auto n = source.read(data, len); + sink(data, n); + return n; + }); + + parseDump(parseSink, wrapper); +} + + } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 8a15e849c7b8..7a0e688e4201 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -74,6 +74,9 @@ void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); +/* Read a NAR from 'source' and write it to 'sink'. */ +void copyNAR(Source & source, Sink & sink); + // FIXME: global variables are bad m'kay. extern bool useCaseHack; diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 470c925ed7a6..81cb5e98c763 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -17,7 +17,23 @@ namespace nix { -static ref<std::string> decompressXZ(const std::string & in) +static const size_t bufSize = 32 * 1024; + +static void decompressNone(Source & source, Sink & sink) +{ + std::vector<unsigned char> buf(bufSize); + while (true) { + size_t n; + try { + n = source.read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + sink(buf.data(), n); + } +} + +static void decompressXZ(Source & source, Sink & sink) { lzma_stream strm(LZMA_STREAM_INIT); @@ -29,36 +45,44 @@ static ref<std::string> decompressXZ(const std::string & in) Finally free([&]() { lzma_end(&strm); }); lzma_action action = LZMA_RUN; - uint8_t outbuf[BUFSIZ]; - ref<std::string> res = make_ref<std::string>(); - strm.next_in = (uint8_t *) in.c_str(); - strm.avail_in = in.size(); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize); + strm.next_in = nullptr; + strm.avail_in = 0; + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); + bool eof = false; while (true) { checkInterrupt(); + if (strm.avail_in == 0 && !eof) { + strm.next_in = inbuf.data(); + try { + strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size()); + } catch (EndOfFile &) { + eof = true; + } + } + if (strm.avail_in == 0) action = LZMA_FINISH; lzma_ret ret = lzma_code(&strm, action); - if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + if (strm.avail_out < outbuf.size()) { + sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out); + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); } - if (ret == LZMA_STREAM_END) - return res; + if (ret == LZMA_STREAM_END) return; if (ret != LZMA_OK) throw CompressionError("error %d while decompressing xz file", ret); } } -static ref<std::string> decompressBzip2(const std::string & in) +static void decompressBzip2(Source & source, Sink & sink) { bz_stream strm; memset(&strm, 0, sizeof(strm)); @@ -69,39 +93,50 @@ static ref<std::string> decompressBzip2(const std::string & in) Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); - char outbuf[BUFSIZ]; - ref<std::string> res = make_ref<std::string>(); - strm.next_in = (char *) in.c_str(); - strm.avail_in = in.size(); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + std::vector<char> inbuf(bufSize), outbuf(bufSize); + strm.next_in = nullptr; + strm.avail_in = 0; + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); + bool eof = false; while (true) { checkInterrupt(); + if (strm.avail_in == 0 && !eof) { + strm.next_in = inbuf.data(); + try { + strm.avail_in = source.read((unsigned char *) strm.next_in, inbuf.size()); + } catch (EndOfFile &) { + eof = true; + } + } + int ret = BZ2_bzDecompress(&strm); - if (strm.avail_out == 0 || ret == BZ_STREAM_END) { - res->append(outbuf, sizeof(outbuf) - strm.avail_out); - strm.next_out = outbuf; - strm.avail_out = sizeof(outbuf); + if (strm.avail_in == 0 && strm.avail_out == outbuf.size() && eof) + throw CompressionError("bzip2 data ends prematurely"); + + if (strm.avail_out < outbuf.size()) { + sink((unsigned char *) outbuf.data(), outbuf.size() - strm.avail_out); + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); } - if (ret == BZ_STREAM_END) - return res; + if (ret == BZ_STREAM_END) return; if (ret != BZ_OK) throw CompressionError("error while decompressing bzip2 file"); - - if (strm.avail_in == 0) - throw CompressionError("bzip2 data ends prematurely"); } } -static ref<std::string> decompressBrotli(const std::string & in) +static void decompressBrotli(Source & source, Sink & sink) { #if !HAVE_BROTLI - return make_ref<std::string>(runProgram(BROTLI, true, {"-d"}, {in})); + RunOptions options(BROTLI, {"-d"}); + options.standardIn = &source; + options.standardOut = &sink; + runProgram2(options); #else auto *s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); if (!s) @@ -109,16 +144,26 @@ static ref<std::string> decompressBrotli(const std::string & in) Finally free([s]() { BrotliDecoderDestroyInstance(s); }); - uint8_t outbuf[BUFSIZ]; - ref<std::string> res = make_ref<std::string>(); - const uint8_t *next_in = (uint8_t *)in.c_str(); - size_t avail_in = in.size(); - uint8_t *next_out = outbuf; - size_t avail_out = sizeof(outbuf); + std::vector<uint8_t> inbuf(bufSize), outbuf(bufSize); + const uint8_t * next_in = nullptr; + size_t avail_in = 0; + bool eof = false; while (true) { checkInterrupt(); + if (avail_in == 0 && !eof) { + next_in = inbuf.data(); + try { + avail_in = source.read((unsigned char *) next_in, inbuf.size()); + } catch (EndOfFile &) { + eof = true; + } + } + + uint8_t * next_out = outbuf.data(); + size_t avail_out = outbuf.size(); + auto ret = BrotliDecoderDecompressStream(s, &avail_in, &next_in, &avail_out, &next_out, @@ -128,51 +173,49 @@ static ref<std::string> decompressBrotli(const std::string & in) case BROTLI_DECODER_RESULT_ERROR: throw CompressionError("error while decompressing brotli file"); case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: - throw CompressionError("incomplete or corrupt brotli file"); + if (eof) + throw CompressionError("incomplete or corrupt brotli file"); + break; case BROTLI_DECODER_RESULT_SUCCESS: if (avail_in != 0) throw CompressionError("unexpected input after brotli decompression"); break; case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: // I'm not sure if this can happen, but abort if this happens with empty buffer - if (avail_out == sizeof(outbuf)) + if (avail_out == outbuf.size()) throw CompressionError("brotli decompression requires larger buffer"); break; } // Always ensure we have full buffer for next invocation - if (avail_out < sizeof(outbuf)) { - res->append((char*)outbuf, sizeof(outbuf) - avail_out); - next_out = outbuf; - avail_out = sizeof(outbuf); - } + if (avail_out < outbuf.size()) + sink((unsigned char *) outbuf.data(), outbuf.size() - avail_out); - if (ret == BROTLI_DECODER_RESULT_SUCCESS) return res; + if (ret == BROTLI_DECODER_RESULT_SUCCESS) return; } #endif // HAVE_BROTLI } -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) +ref<std::string> decompress(const std::string & method, const std::string & in) { - StringSink ssink; - auto sink = makeCompressionSink(method, ssink, parallel); - (*sink)(in); - sink->finish(); - return ssink.s; + StringSource source(in); + StringSink sink; + decompress(method, source, sink); + return sink.s; } -ref<std::string> decompress(const std::string & method, const std::string & in) +void decompress(const std::string & method, Source & source, Sink & sink) { if (method == "none") - return make_ref<std::string>(in); + return decompressNone(source, sink); else if (method == "xz") - return decompressXZ(in); + return decompressXZ(source, sink); else if (method == "bzip2") - return decompressBzip2(in); + return decompressBzip2(source, sink); else if (method == "br") - return decompressBrotli(in); + return decompressBrotli(source, sink); else - throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); + throw UnknownCompressionMethod("unknown compression method '%s'", method); } struct NoneSink : CompressionSink @@ -499,4 +542,13 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); } +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) +{ + StringSink ssink; + auto sink = makeCompressionSink(method, ssink, parallel); + (*sink)(in); + sink->finish(); + return ssink.s; +} + } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index a0d7530d74fc..f7a3e3fbd32e 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -8,10 +8,12 @@ namespace nix { -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); - ref<std::string> decompress(const std::string & method, const std::string & in); +void decompress(const std::string & method, Source & source, Sink & sink); + +ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); + struct CompressionSink : BufferedSink { virtual void finish() = 0; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 75e4767550f7..a01d651e1ef5 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -191,6 +191,7 @@ Hash::Hash(const std::string & s, HashType type) auto d = base64Decode(std::string(s, pos)); if (d.size() != hashSize) throw BadHash("invalid base-64 hash '%s'", s); + assert(hashSize); memcpy(hash, d.data(), hashSize); } @@ -256,12 +257,12 @@ Hash hashFile(HashType ht, const Path & path) AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) throw SysError(format("opening file '%1%'") % path); - unsigned char buf[8192]; + std::vector<unsigned char> buf(8192); ssize_t n; - while ((n = read(fd.get(), buf, sizeof(buf)))) { + while ((n = read(fd.get(), buf.data(), buf.size()))) { checkInterrupt(); if (n == -1) throw SysError(format("reading file '%1%'") % path); - update(ht, ctx, buf, n); + update(ht, ctx, buf.data(), n); } finish(ht, ctx, hash.hash); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 5fc2aab569da..824f48fbfc9f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,7 +6,7 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context libutil_LIBS = libformat diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 27a631a37d10..799c6e1ae441 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -6,7 +6,16 @@ namespace nix { -thread_local ActivityId curActivity = 0; +static thread_local ActivityId curActivity = 0; + +ActivityId getCurActivity() +{ + return curActivity; +} +void setCurActivity(const ActivityId activityId) +{ + curActivity = activityId; +} Logger * logger = makeDefaultLogger(); @@ -44,7 +53,7 @@ public: prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n"); + writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -221,4 +230,12 @@ bool handleJSONLogMessage(const std::string & msg, return true; } +Activity::~Activity() { + try { + logger.stopActivity(id); + } catch (...) { + ignoreException(); + } +} + } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 677aa4daec4d..678703102e9b 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -77,7 +77,8 @@ public: virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; }; -extern thread_local ActivityId curActivity; +ActivityId getCurActivity(); +void setCurActivity(const ActivityId activityId); struct Activity { @@ -86,16 +87,15 @@ struct Activity const ActivityId id; Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "", - const Logger::Fields & fields = {}, ActivityId parent = curActivity); + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()); Activity(Logger & logger, ActivityType type, - const Logger::Fields & fields = {}, ActivityId parent = curActivity) + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()) : Activity(logger, lvlError, type, "", fields, parent) { }; Activity(const Activity & act) = delete; - ~Activity() - { logger.stopActivity(id); } + ~Activity(); void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const { result(resProgress, done, expected, running, failed); } @@ -122,8 +122,8 @@ struct Activity struct PushActivity { const ActivityId prevAct; - PushActivity(ActivityId act) : prevAct(curActivity) { curActivity = act; } - ~PushActivity() { curActivity = prevAct; } + PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); } + ~PushActivity() { setCurActivity(prevAct); } }; extern Logger * logger; diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 3cb5d50889d9..9b8290e634c9 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,6 +2,7 @@ #include <map> #include <list> +#include <experimental/optional> namespace nix { @@ -63,18 +64,17 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - // FIXME: use boost::optional? - Value * get(const Key & key) + std::experimental::optional<Value> get(const Key & key) { auto i = data.find(key); - if (i == data.end()) return 0; + if (i == data.end()) return {}; /* Move this item to the back of the LRU list. */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; - return &i->second.second; + return i->second.second; } size_t size() diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 950e6362a245..33ae1ea389d7 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -5,6 +5,8 @@ #include <cerrno> #include <memory> +#include <boost/coroutine2/coroutine.hpp> + namespace nix { @@ -67,7 +69,8 @@ void FdSink::write(const unsigned char * data, size_t len) try { writeFull(fd, data, len); } catch (SysError & e) { - _good = true; + _good = false; + throw; } } @@ -87,6 +90,23 @@ void Source::operator () (unsigned char * data, size_t len) } +std::string Source::drain() +{ + std::string s; + std::vector<unsigned char> buf(8192); + while (true) { + size_t n; + try { + n = read(buf.data(), buf.size()); + s.append((char *) buf.data(), n); + } catch (EndOfFile &) { + break; + } + } + return s; +} + + size_t BufferedSource::read(unsigned char * data, size_t len) { if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); @@ -137,6 +157,50 @@ size_t StringSource::read(unsigned char * data, size_t len) } +std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) +{ + struct SinkToSource : Source + { + typedef boost::coroutines2::coroutine<std::string> coro_t; + + coro_t::pull_type coro; + + SinkToSource(std::function<void(Sink &)> fun) + : coro([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }) + { + } + + std::string cur; + size_t pos = 0; + + size_t read(unsigned char * data, size_t len) override + { + if (!coro) + throw EndOfFile("coroutine has finished"); + + if (pos == cur.size()) { + if (!cur.empty()) coro(); + cur = coro.get(); + pos = 0; + } + + auto n = std::min(cur.size() - pos, len); + memcpy(data, (unsigned char *) cur.data() + pos, n); + pos += n; + + return n; + } + }; + + return std::make_unique<SinkToSource>(fun); +} + + void writePadding(size_t len, Sink & sink) { if (len % 8) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 2ea5b6354ee9..d0b4552e3399 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -56,11 +56,13 @@ struct Source void operator () (unsigned char * data, size_t len); /* Store up to โlenโ in the buffer pointed to by โdataโ, and - return the number of bytes stored. If blocks until at least + return the number of bytes stored. It blocks until at least one byte is available. */ virtual size_t read(unsigned char * data, size_t len) = 0; virtual bool good() { return true; } + + std::string drain(); }; @@ -175,6 +177,43 @@ struct TeeSource : Source }; +/* Convert a function into a sink. */ +struct LambdaSink : Sink +{ + typedef std::function<void(const unsigned char *, size_t)> lambda_t; + + lambda_t lambda; + + LambdaSink(const lambda_t & lambda) : lambda(lambda) { } + + virtual void operator () (const unsigned char * data, size_t len) + { + lambda(data, len); + } +}; + + +/* Convert a function into a source. */ +struct LambdaSource : Source +{ + typedef std::function<size_t(unsigned char *, size_t)> lambda_t; + + lambda_t lambda; + + LambdaSource(const lambda_t & lambda) : lambda(lambda) { } + + size_t read(unsigned char * data, size_t len) override + { + return lambda(data, len); + } +}; + + +/* Convert a function that feeds data into a Sink into a Source. The + Source executes the function as a coroutine. */ +std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun); + + void writePadding(size_t len, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2391e14a94bd..15962236ec65 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -3,6 +3,7 @@ #include "affinity.hh" #include "sync.hh" #include "finally.hh" +#include "serialise.hh" #include <cctype> #include <cerrno> @@ -229,16 +230,17 @@ bool pathExists(const Path & path) Path readLink(const Path & path) { checkInterrupt(); + std::vector<char> buf; for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - char buf[bufSize]; - ssize_t rlSize = readlink(path.c_str(), buf, bufSize); + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); if (rlSize == -1) if (errno == EINVAL) throw Error("'%1%' is not a symlink", path); else throw SysError("reading symbolic link '%1%'", path); else if (rlSize < bufSize) - return string(buf, rlSize); + return string(buf.data(), rlSize); } } @@ -293,10 +295,10 @@ string readFile(int fd) if (fstat(fd, &st) == -1) throw SysError("statting file"); - auto buf = std::make_unique<unsigned char[]>(st.st_size); - readFull(fd, buf.get(), st.st_size); + std::vector<unsigned char> buf(st.st_size); + readFull(fd, buf.data(), st.st_size); - return string((char *) buf.get(), st.st_size); + return string((char *) buf.data(), st.st_size); } @@ -438,10 +440,10 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, static Lazy<Path> getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { - char buf[16384]; + std::vector<char> buf(16384); struct passwd pwbuf; struct passwd * pw; - if (getpwuid_r(getuid(), &pwbuf, buf, sizeof(buf), &pw) != 0 + if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; @@ -566,21 +568,44 @@ void writeFull(int fd, const string & s, bool allowInterrupts) } -string drainFD(int fd) +string drainFD(int fd, bool block) { - string result; - unsigned char buffer[4096]; + StringSink sink; + drainFD(fd, sink, block); + return std::move(*sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + int saved; + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + std::vector<unsigned char> buf(4096); while (1) { checkInterrupt(); - ssize_t rd = read(fd, buffer, sizeof buffer); + ssize_t rd = read(fd, buf.data(), buf.size()); if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; if (errno != EINTR) throw SysError("reading from file"); } else if (rd == 0) break; - else result.append((char *) buffer, rd); + else sink(buf.data(), rd); } - return result; } @@ -920,20 +945,47 @@ string runProgram(Path program, bool searchPath, const Strings & args, return res.second; } -std::pair<int, std::string> runProgram(const RunOptions & options) +std::pair<int, std::string> runProgram(const RunOptions & options_) +{ + RunOptions options(options_); + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(*sink.s)}; +} + +void runProgram2(const RunOptions & options) { checkInterrupt(); + assert(!(options.standardIn && options.input)); + + std::unique_ptr<Source> source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique<StringSource>(*options.input); + source = source_.get(); + } + /* Create a pipe. */ Pipe out, in; - out.create(); - if (options.input) in.create(); + if (options.standardOut) out.create(); + if (source) in.create(); /* Fork. */ Pid pid = startProcess([&]() { - if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); - if (options.input && dup2(in.readSide.get(), STDIN_FILENO) == -1) + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); Strings args_(options.args); @@ -961,11 +1013,20 @@ std::pair<int, std::string> runProgram(const RunOptions & options) }); - if (options.input) { + if (source) { in.readSide = -1; writerThread = std::thread([&]() { try { - writeFull(in.writeSide.get(), *options.input); + std::vector<unsigned char> buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), buf.data(), n); + } promise.set_value(); } catch (...) { promise.set_exception(std::current_exception()); @@ -974,15 +1035,17 @@ std::pair<int, std::string> runProgram(const RunOptions & options) }); } - string result = drainFD(out.readSide.get()); + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); /* Wait for the child to finish. */ int status = pid.wait(); /* Wait for the writer thread to finish. */ - if (options.input) promise.get_future().get(); + if (source) promise.get_future().get(); - return {status, result}; + if (status) + throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); } @@ -1185,7 +1248,7 @@ void ignoreException() } -std::string filterANSIEscapes(const std::string & s, unsigned int width) +std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) { std::string t, e; size_t w = 0; @@ -1210,7 +1273,7 @@ std::string filterANSIEscapes(const std::string & s, unsigned int width) if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } - if (last == 'm') + if (!filterAll && last == 'm') t += e; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c5c537ee63d8..743d238611fc 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -25,6 +25,9 @@ namespace nix { +struct Sink; +struct Source; + /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); @@ -148,8 +151,9 @@ MakeError(EndOfFile, Error) /* Read a file descriptor until EOF occurs. */ -string drainFD(int fd); +string drainFD(int fd, bool block = true); +void drainFD(int fd, Sink & sink, bool block = true); /* Automatic cleanup of resources. */ @@ -256,6 +260,8 @@ struct RunOptions bool searchPath = true; Strings args; std::experimental::optional<std::string> input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) @@ -266,6 +272,8 @@ struct RunOptions std::pair<int, std::string> runProgram(const RunOptions & options); +void runProgram2(const RunOptions & options); + class ExecError : public Error { @@ -391,11 +399,13 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" -/* Truncate a string to 'width' printable characters. Certain ANSI - escape sequences (such as colour setting) are copied but not - included in the character count. Other ANSI escape sequences are - filtered. Also, tabs are expanded to spaces. */ +/* Truncate a string to 'width' printable characters. If 'filterAll' + is true, all ANSI escape sequences are filtered out. Otherwise, + some escape sequences (such as colour setting) are copied but not + included in the character count. Also, tabs are expanded to + spaces. */ std::string filterANSIEscapes(const std::string & s, + bool filterAll = false, unsigned int width = std::numeric_limits<unsigned int>::max()); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 99f773451ffe..a63b3e07ae77 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -212,7 +212,7 @@ void mainWrapped(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); } else { envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); } @@ -271,10 +271,14 @@ void mainWrapped(int argc, char * * argv) exprs = {state.parseStdin()}; else for (auto i : left) { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error e) {}; if (fromArgs) exprs.push_back(state.parseExprFromString(i, absPath("."))); - else if (store->isStorePath(i) && std::regex_match(i, std::regex(".*\\.drv(!.*)?"))) - drvs.push_back(DrvInfo(state, store, i)); + else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + drvs.push_back(DrvInfo(state, store, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk index 49fc105c6f79..c14e8c359ca0 100644 --- a/src/nix-channel/local.mk +++ b/src/nix-channel/local.mk @@ -2,6 +2,6 @@ programs += nix-channel nix-channel_DIR := $(d) -nix-channel_LIBS = libmain libutil libformat libstore +nix-channel_LIBS = libmain libformat libstore libutil nix-channel_SOURCES := $(d)/nix-channel.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index ec9a7174ecb9..d1b47ede803f 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -10,8 +10,8 @@ using namespace nix; typedef std::map<string,string> Channels; -static auto channels = Channels{}; -static auto channelsList = Path{}; +static Channels channels; +static Path channelsList; // Reads the list of channels. static void readChannels() @@ -52,7 +52,7 @@ static void addChannel(const string & url, const string & name) writeChannels(); } -static auto profile = Path{}; +static Path profile; // Remove a channel. static void removeChannel(const string & name) @@ -64,7 +64,7 @@ static void removeChannel(const string & name) runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); } -static auto nixDefExpr = Path{}; +static Path nixDefExpr; // Fetch Nix expressions and binary cache URLs from the subscribed channels. static void update(const StringSet & channelNames) @@ -74,7 +74,7 @@ static void update(const StringSet & channelNames) auto store = openStore(); // Download each channel. - auto exprs = Strings{}; + Strings exprs; for (const auto & channel : channels) { auto name = channel.first; auto url = channel.second; @@ -84,7 +84,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto effectiveUrl = string{}; + std::string effectiveUrl; auto dl = getDownloader(); auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl); url = chomp(std::move(effectiveUrl)); @@ -99,9 +99,9 @@ static void update(const StringSet & channelNames) cname = cname + (string) match[1]; } - auto extraAttrs = string{}; + std::string extraAttrs; - auto unpacked = false; + bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); @@ -136,7 +136,7 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. std::cerr << "unpacking channels...\n"; - auto envArgs = Strings{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" }; + Strings envArgs{ "--profile", profile, "--file", "<nix/unpack-channel.nix>", "--install", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); @@ -162,23 +162,15 @@ int main(int argc, char ** argv) return handleExceptions(argv[0], [&]() { initNix(); - // Turn on caching in nix-prefetch-url. - auto channelCache = settings.nixStateDir + "/channel-cache"; - createDirs(channelCache); - setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1); - // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = home + "/.nix-channels"; nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - auto name = string{}; + ; auto pw = getpwuid(getuid()); - if (!pw) - name = getEnv("USER", ""); - else - name = pw->pw_name; + std::string name = pw ? pw->pw_name : getEnv("USER", ""); if (name.empty()) throw Error("cannot figure out user name"); profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels"; @@ -192,7 +184,7 @@ int main(int argc, char ** argv) cUpdate, cRollback } cmd = cNone; - auto args = std::vector<string>{}; + std::vector<string> args; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") { showManPage("nix-channel"); @@ -224,7 +216,7 @@ int main(int argc, char ** argv) throw UsageError("'--add' requires one or two arguments"); { auto url = args[0]; - auto name = string{}; + std::string name; if (args.size() == 2) { name = args[1]; } else { @@ -253,7 +245,7 @@ int main(int argc, char ** argv) case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); - auto envArgs = Strings{"--profile", profile}; + Strings envArgs{"--profile", profile}; if (args.size() == 1) { envArgs.push_back("--switch-generation"); envArgs.push_back(args[0]); diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk index 42bb34dd8201..5018ab975b44 100644 --- a/src/nix-copy-closure/local.mk +++ b/src/nix-copy-closure/local.mk @@ -2,6 +2,6 @@ programs += nix-copy-closure nix-copy-closure_DIR := $(d) -nix-copy-closure_LIBS = libmain libutil libformat libstore +nix-copy-closure_LIBS = libmain libformat libstore libutil nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 890bffa19aa5..35603af7082a 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -37,13 +37,13 @@ using namespace nix; static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) { /* We ignore most parameters, we just have them for conformance with the linux syscall */ - char buf[8192]; - auto read_count = read(fd_in, buf, sizeof(buf)); + std::vector<char> buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); if (read_count == -1) return read_count; auto write_count = decltype(read_count)(0); while (write_count < read_count) { - auto res = write(fd_out, buf + write_count, read_count - write_count); + auto res = write(fd_out, buf.data() + write_count, read_count - write_count); if (res == -1) return res; write_count += res; @@ -695,7 +695,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, parseDump(tee, tee.source); logger->startWork(); - store->addToStore(info, tee.source.data, (RepairFlag) repair, + store.cast<Store>()->addToStore(info, tee.source.data, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); logger->stopWork(); break; @@ -816,8 +816,11 @@ static void processConnection(bool trusted) static void sigChldHandler(int sigNo) { + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; /* Reap all dead children. */ while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; } @@ -1032,7 +1035,7 @@ static void daemonLoop(char * * argv) }, options); } catch (Interrupted & e) { - throw; + return; } catch (Error & e) { printError(format("error processing connection: %1%") % e.msg()); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index dd262bea0918..5049460c7544 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -70,7 +70,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (gcRoot == "") printGCWarning(); else { - Path rootName = gcRoot; + Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); if (store2) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index efef7f15c094..e1e27ceef94d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -631,6 +631,7 @@ static void opDump(Strings opFlags, Strings opArgs) FdSink sink(STDOUT_FILENO); string path = *opArgs.begin(); dumpPath(path, sink); + sink.flush(); } @@ -656,6 +657,7 @@ static void opExport(Strings opFlags, Strings opArgs) FdSink sink(STDOUT_FILENO); store->exportPaths(opArgs, sink); + sink.flush(); } diff --git a/src/nix/command.cc b/src/nix/command.cc index 1e6f0d2bb75d..3d7d582d6f5e 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -57,8 +57,10 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out) } printTable(out, table); +#if 0 out << "\n"; out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n"; +#endif } bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) diff --git a/src/nix/copy.cc b/src/nix/copy.cc index f29429c1ac49..e4e6c3e303ed 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -67,6 +67,12 @@ struct CmdCopy : StorePathsCommand "To copy a closure from another machine via SSH:", "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" }, +#ifdef ENABLE_S3 + Example{ + "To populate the current folder build output to a S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1" + }, +#endif }; } diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 1a1866437b07..f411c0cb7c89 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -29,6 +29,7 @@ struct CmdDumpPath : StorePathCommand { FdSink sink(STDOUT_FILENO); store->narFromPath(storePath, sink); + sink.flush(); } }; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 7eaa86e2f914..c9671f76d0fa 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -61,7 +61,7 @@ struct CmdEdit : InstallableCommand auto editor = getEnv("EDITOR", "cat"); - Strings args{editor}; + auto args = tokenizeString<Strings>(editor); if (editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos || @@ -72,7 +72,7 @@ struct CmdEdit : InstallableCommand stopProgressBar(); - execvp(editor.c_str(), stringsToCharPtrs(args).data()); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); throw SysError("cannot run editor '%s'", editor); } diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index e6553c06f4ae..40b905ba3243 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -308,7 +308,7 @@ public: auto width = getWindowSize().second; if (width <= 0) std::numeric_limits<decltype(width)>::max(); - writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K"); + writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); } std::string getStatus(State & state) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9216209173d9..f84774a53367 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -189,6 +189,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt) if (!s) { switch (auto type = linenoiseKeyType()) { case 1: // ctrl-C + input = ""; return true; case 2: // ctrl-D return false; @@ -197,6 +198,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt) } } input += s; + input += '\n'; return true; } diff --git a/src/nix/search.cc b/src/nix/search.cc index 87cdb2d7ed8a..539676698086 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -25,14 +25,14 @@ std::string hilite(const std::string & s, const std::smatch & m) struct CmdSearch : SourceExprCommand, MixJSON { - std::string re; + std::vector<std::string> res; bool writeCache = true; bool useCache = true; CmdSearch() { - expectArg("regex", &re, true); + expectArgs("regex", &res); mkFlag() .longName("update-cache") @@ -68,9 +68,13 @@ struct CmdSearch : SourceExprCommand, MixJSON "nix search blender" }, Example{ - "To search for Firefox and Chromium:", + "To search for Firefox or Chromium:", "nix search 'firefox|chromium'" }, + Example{ + "To search for git and frontend or gui:", + "nix search git 'frontend|gui'" + }, }; } @@ -78,11 +82,21 @@ struct CmdSearch : SourceExprCommand, MixJSON { settings.readOnlyMode = true; - std::regex regex(re, std::regex::extended | std::regex::icase); + // Empty search string should match all packages + // Use "^" here instead of ".*" due to differences in resulting highlighting + // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + if (res.empty()) { + res.push_back("^"); + } - auto state = getEvalState(); + std::vector<std::regex> regexes; + regexes.reserve(res.size()); + + for (auto &re : res) { + regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + } - bool first = true; + auto state = getEvalState(); auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; @@ -91,12 +105,15 @@ struct CmdSearch : SourceExprCommand, MixJSON bool fromCache = false; + std::map<std::string, std::string> results; + std::function<void(Value *, std::string, bool, JSONObject *)> doExpr; doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { debug("at attribute '%s'", attrPath); try { + uint found = 0; state->forceValue(*v); @@ -110,25 +127,33 @@ struct CmdSearch : SourceExprCommand, MixJSON if (state->isDerivation(*v)) { DrvInfo drv(*state, attrPath, v->attrs); + std::string description; + std::smatch attrPathMatch; + std::smatch descriptionMatch; + std::smatch nameMatch; + std::string name; DrvName parsed(drv.queryName()); - std::smatch attrPathMatch; - std::regex_search(attrPath, attrPathMatch, regex); + for (auto ®ex : regexes) { + std::regex_search(attrPath, attrPathMatch, regex); - auto name = parsed.name; - std::smatch nameMatch; - std::regex_search(name, nameMatch, regex); + name = parsed.name; + std::regex_search(name, nameMatch, regex); - std::string description = drv.queryMetaString("description"); - std::replace(description.begin(), description.end(), '\n', ' '); - std::smatch descriptionMatch; - std::regex_search(description, descriptionMatch, regex); + description = drv.queryMetaString("description"); + std::replace(description.begin(), description.end(), '\n', ' '); + std::regex_search(description, descriptionMatch, regex); + + if (!attrPathMatch.empty() + || !nameMatch.empty() + || !descriptionMatch.empty()) + { + found++; + } + } - if (!attrPathMatch.empty() - || !nameMatch.empty() - || !descriptionMatch.empty()) - { + if (found == res.size()) { if (json) { auto jsonElem = jsonOut->object(attrPath); @@ -138,10 +163,7 @@ struct CmdSearch : SourceExprCommand, MixJSON jsonElem.attr("description", description); } else { - if (!first) std::cout << "\n"; - first = false; - - std::cout << fmt( + results[attrPath] = fmt( "Attribute name: %s\n" "Package name: %s\n" "Version: %s\n" @@ -237,9 +259,12 @@ struct CmdSearch : SourceExprCommand, MixJSON throw Error("error writing to %s", tmpFile); } - if (rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) + if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); } + + for (auto el : results) std::cout << el.second << "\n"; + } }; diff --git a/tests/export.sh b/tests/export.sh index ec7560f19728..2238539bcca9 100644 --- a/tests/export.sh +++ b/tests/export.sh @@ -8,6 +8,11 @@ nix-store --export $outPath > $TEST_ROOT/exp nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all +if nix-store --export $outPath >/dev/full ; then + echo "exporting to a bad file descriptor should fail" + exit 1 +fi + clearStore diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/lang/eval-okay-backslash-newline-1.exp new file mode 100644 index 000000000000..3e754364cc9c --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-1.exp @@ -0,0 +1 @@ +"a\nb" diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/lang/eval-okay-backslash-newline-1.nix new file mode 100644 index 000000000000..7fef3dddd4dd --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-1.nix @@ -0,0 +1,2 @@ +"a\ +b" diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/lang/eval-okay-backslash-newline-2.exp new file mode 100644 index 000000000000..3e754364cc9c --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-2.exp @@ -0,0 +1 @@ +"a\nb" diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/lang/eval-okay-backslash-newline-2.nix new file mode 100644 index 000000000000..35ddf495c63b --- /dev/null +++ b/tests/lang/eval-okay-backslash-newline-2.nix @@ -0,0 +1,2 @@ +''a''\ +b'' diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/lang/eval-okay-getattrpos-undefined.exp new file mode 100644 index 000000000000..19765bd501b6 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-undefined.exp @@ -0,0 +1 @@ +null diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/lang/eval-okay-getattrpos-undefined.nix new file mode 100644 index 000000000000..14dd38f7734c --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-undefined.nix @@ -0,0 +1 @@ +builtins.unsafeGetAttrPos "abort" builtins diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/lang/eval-okay-nested-with.exp new file mode 100644 index 000000000000..0cfbf08886fc --- /dev/null +++ b/tests/lang/eval-okay-nested-with.exp @@ -0,0 +1 @@ +2 diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/lang/eval-okay-nested-with.nix new file mode 100644 index 000000000000..ba9d79aa79b1 --- /dev/null +++ b/tests/lang/eval-okay-nested-with.nix @@ -0,0 +1,3 @@ +with { x = 1; }; +with { x = 2; }; +x diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/lang/eval-okay-regex-split.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/tests/lang/eval-okay-regex-split.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp index a2add1b7b140..72e8274d8c58 100644 --- a/tests/lang/eval-okay-replacestrings.exp +++ b/tests/lang/eval-okay-replacestrings.exp @@ -1 +1 @@ -[ "faabar" "fbar" "fubar" "faboor" "fubar" ] +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix index 6284a0e660ae..bd8031fc004e 100644 --- a/tests/lang/eval-okay-replacestrings.nix +++ b/tests/lang/eval-okay-replacestrings.nix @@ -5,4 +5,7 @@ with builtins; (replaceStrings ["oo"] ["u"] "foobar") (replaceStrings ["oo" "a"] ["a" "oo"] "foobar") (replaceStrings ["oo" "oo"] ["u" "i"] "foobar") + (replaceStrings [""] ["X"] "abc") + (replaceStrings [""] ["X"] "") + (replaceStrings ["-"] ["_"] "a-b") ] diff --git a/tests/local.mk b/tests/local.mk index ec7ebfb0dedc..9df0adf1bfd8 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -24,7 +24,8 @@ nix_tests = \ brotli.sh \ pure-eval.sh \ check.sh \ - plugins.sh + plugins.sh \ + search.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/nar-access.sh b/tests/nar-access.sh index bd849cbfab1d..553d6ca89d7d 100644 --- a/tests/nar-access.sh +++ b/tests/nar-access.sh @@ -36,3 +36,9 @@ diff -u baz.cat-nar $storePath/foo/baz # Test missing files. nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist' + +# Test failure to dump. +if nix-store --dump $storePath >/dev/full ; then + echo "dumping to /dev/full should fail" + exit -1 +fi diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index f7cac6ecf8ef..d25c456cedfb 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -17,6 +17,18 @@ output=$(nix-shell --pure shell.nix -A shellDrv --run \ [[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] +# Test nix-shell on a .drv symlink + +# Legacy: absolute path and .drv extension required +nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv +[[ $(nix-shell --pure $PWD/shell.drv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# New behaviour: just needs to resolve to a derivation in the store +nix-instantiate shell.nix -A shellDrv --indirect --add-root shell +[[ $(nix-shell --pure shell --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + # Test nix-shell -p output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') [ "$output" = "foo bar" ] @@ -27,3 +39,12 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] + +# Test nix-shell shebang mode for ruby +# This uses a fake interpreter that returns the arguments passed +# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. +sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb +chmod a+rx $TEST_ROOT/shell.shebang.rb + +output=$($TEST_ROOT/shell.shebang.rb abc ruby) +[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] diff --git a/tests/restricted.sh b/tests/restricted.sh index 0605383cc86a..a87d8ec2c940 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -11,8 +11,8 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.n (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/boost') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/boost' -I src=../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. diff --git a/tests/search.nix b/tests/search.nix new file mode 100644 index 000000000000..fea6e7a7a647 --- /dev/null +++ b/tests/search.nix @@ -0,0 +1,25 @@ +with import ./config.nix; + +{ + hello = mkDerivation rec { + name = "hello-${version}"; + version = "0.1"; + buildCommand = "touch $out"; + meta.description = "Empty file"; + }; + foo = mkDerivation rec { + name = "foo-5"; + buildCommand = '' + mkdir -p $out + echo ${name} > $out/${name} + ''; + }; + bar = mkDerivation rec { + name = "bar-3"; + buildCommand = '' + echo "Does not build successfully" + exit 1 + ''; + meta.description = "broken bar"; + }; +} diff --git a/tests/search.sh b/tests/search.sh new file mode 100644 index 000000000000..0b26a125120f --- /dev/null +++ b/tests/search.sh @@ -0,0 +1,43 @@ +source common.sh + +clearStore +clearCache + +# No packages +(( $(NIX_PATH= nix search -u|wc -l) == 0 )) + +# Haven't updated cache, still nothing +(( $(nix search -f search.nix hello|wc -l) == 0 )) +(( $(nix search -f search.nix |wc -l) == 0 )) + +# Update cache, search should work +(( $(nix search -f search.nix -u hello|wc -l) > 0 )) + +# Use cache +(( $(nix search -f search.nix foo|wc -l) > 0 )) +(( $(nix search foo|wc -l) > 0 )) + +# Test --no-cache works +# No results from cache +(( $(nix search --no-cache foo |wc -l) == 0 )) +# Does find results from file pointed at +(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 )) + +# Check descriptions are searched +(( $(nix search broken | wc -l) > 0 )) + +# Check search that matches nothing +(( $(nix search nosuchpackageexists | wc -l) == 0 )) + +# Search for multiple arguments +(( $(nix search hello empty | wc -l) == 5 )) + +# Multiple arguments will not exist +(( $(nix search hello broken | wc -l) == 0 )) + +## Search expressions + +# Check that empty search string matches all +nix search|grep -q foo +nix search|grep -q bar +nix search|grep -q hello diff --git a/tests/shell.nix b/tests/shell.nix index 5845d36fc161..eb39f9039a88 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -45,5 +45,12 @@ let pkgs = rec { bash = shell; + # ruby "interpreter" that outputs "$@" + ruby = runCommand "ruby" {} '' + mkdir -p $out/bin + echo 'printf -- "$*"' > $out/bin/ruby + chmod a+rx $out/bin/ruby + ''; + inherit pkgs; }; in pkgs diff --git a/tests/shell.shebang.rb b/tests/shell.shebang.rb new file mode 100644 index 000000000000..ea67eb09c1c6 --- /dev/null +++ b/tests/shell.shebang.rb @@ -0,0 +1,7 @@ +#! @SHELL_PROG@ +#! ruby +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --pure -p ruby -i ruby + +# Contents doesn't matter. +abort("This shouldn't be executed.") diff --git a/version b/version index 415b19fc3623..42f7d2336ea8 100644 --- a/version +++ b/version @@ -1 +1 @@ -2.0 \ No newline at end of file +2.1 \ No newline at end of file |