diff options
67 files changed, 2231 insertions, 542 deletions
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000000..bbaabf93c7a9 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +custom: https://nixos.org/nixos/foundation.html diff --git a/Makefile b/Makefile index 45a3338ed21c..9ac82fda657d 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include config.h -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/Makefile.config.in b/Makefile.config.in index 59730b646387..7e3b35b98196 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -5,10 +5,11 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ +LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ -HAVE_READLINE = @HAVE_READLINE@ HAVE_SECCOMP = @HAVE_SECCOMP@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ diff --git a/configure.ac b/configure.ac index f5b1614f19f1..a52830b3835f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,5 @@ AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -42,27 +43,21 @@ esac AC_MSG_RESULT($system) AC_SUBST(system) -AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')]) +AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Solaris-specific stuff. -AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then - # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" -fi - - -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_17 +AX_CXX_COMPILE_STDCXX_17([noext], [mandatory]) AC_CHECK_TOOL([AR], [ar]) @@ -70,6 +65,14 @@ AC_CHECK_TOOL([AR], [ar]) AC_SYS_LARGEFILE +# Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE +if test "$sys_name" = sunos; then + # Solaris requires -lsocket -lnsl for network functions + LIBS="-lsocket -lnsl $LIBS" +fi + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -145,6 +148,16 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_SUBST(storedir) +# Look for boost, a required dependency. +# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, +# and CPPFLAGS are not passed to the C++ compiler automatically. +# Thus we append the returned CPPFLAGS to the CXXFLAGS here. +AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) +# For unknown reasons, setting this directly in the ACTION-IF-FOUND above +# ends up with LDFLAGS being empty, so we set it afterwards. +LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" + + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -164,7 +177,16 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) # Look for editline, a required dependency. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) +# The the libeditline.pc file was added only in libeditline >= 1.15.2, +# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, +# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for +# editline.h when the pkg-config approach fails. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ + AC_CHECK_HEADERS([editline.h], [true], + [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) + AC_SEARCH_LIBS([readline read_history], [editline], [], + [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) +]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index a654db40e62a..d39a20637818 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -18,21 +18,17 @@ let if [ * != $channelName ]; then mv * $out/$channelName fi - if [ -n "$binaryCacheURL" ]; then - mkdir $out/binary-caches - echo -n "$binaryCacheURL" > $out/binary-caches/$channelName - fi ''; in -{ name, channelName, src, binaryCacheURL ? "" }: +{ name, channelName, src }: derivation { system = builtins.currentSystem; builder = shell; args = [ "-e" builder ]; - inherit name channelName src binaryCacheURL; + inherit name channelName src; PATH = "${nixBinDir}:${coreutils}"; diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index ff4021a765e0..5a2866e6bc4b 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -31,12 +31,11 @@ <refsection><title>Description</title> -<para>A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing both a set of Nix -expressions and a pointer to a binary cache. <phrase -condition="manual">See also <xref linkend="sec-channels" -/>.</phrase></para> +<para>A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. <phrase condition="manual">See also <xref +linkend="sec-channels" />.</phrase></para> <para>This command has the following operations: @@ -172,18 +171,6 @@ following files:</para> </varlistentry> - <varlistentry><term><filename>binary-cache-url</filename></term> - - <listitem><para>A file containing the URL to a binary cache (such - as <uri>https://cache.nixos.org</uri>). Nix will automatically - check this cache for pre-built binaries, if the user has - sufficient rights to add binary caches. For instance, in a - multi-user Nix setup, the binary caches provided by the channels - of the root user are used automatically, but caches corresponding - to the channels of non-root users are ignored.</para></listitem> - - </varlistentry> - </variablelist> </refsection> diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index d73cb92ee223..113a3c2e41ed 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -215,6 +215,48 @@ printed.)</para> </variablelist> +<para>Special exit codes:</para> + +<variablelist> + + <varlistentry><term><literal>100</literal></term> + <listitem><para>Generic build failure, the builder process + returned with a non-zero exit code.</para></listitem> + </varlistentry> + + <varlistentry><term><literal>101</literal></term> + <listitem><para>Build timeout, the build was aborted because it + did not complete within the specified <link + linkend='conf-timeout'><literal>timeout</literal></link>. + </para></listitem> + </varlistentry> + + <varlistentry><term><literal>102</literal></term> + <listitem><para>Hash mismatch, the build output was rejected + because it does not match the specified <link + linkend="fixed-output-drvs"><varname>outputHash</varname></link>. + </para></listitem> + </varlistentry> + + <varlistentry><term><literal>104</literal></term> + <listitem><para>Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable.</para> + </listitem> + </varlistentry> + +</variablelist> + +<para>With the <option>--keep-going</option> flag it's possible for +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. <screen> +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +</screen></para> + </refsection> @@ -883,6 +925,60 @@ $ nix-store --add ./foo.c </refsection> +<!--######################################################################--> + +<refsection><title>Operation <option>--add-fixed</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg><option>--recursive</option></arg> + <arg choice='plain'><option>--add-fixed</option></arg> + <arg choice='plain'><replaceable>algorithm</replaceable></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--add-fixed</option> adds the specified paths to +the Nix store. Unlike <option>--add</option> paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. +</para> + +<para>This operation has the following options: + +<variablelist> + + <varlistentry><term><option>--recursive</option></term> + + <listitem><para> + Use recursive instead of flat hashing mode, used when adding directories + to the store. + </para></listitem> + + </varlistentry> + +</variablelist> + +</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz</screen> + +</refsection> + +</refsection> + <!--######################################################################--> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index deffcb594428..4c1d618e951c 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -425,6 +425,13 @@ stdenv.mkDerivation { … } This is often a branch or tag name. Defaults to <literal>HEAD</literal>. </para> + + <para> + By default, the <varname>ref</varname> value is prefixed + with <literal>refs/heads/</literal>. As of Nix 2.3.0 + Nix will not prefix <literal>refs/heads/</literal> if + <varname>ref</varname> starts with <literal>refs/</literal>. + </para> </listitem> </varlistentry> </variablelist> @@ -439,6 +446,14 @@ stdenv.mkDerivation { … } </example> <example> + <title>Fetching an arbitrary ref</title> + <programlisting>builtins.fetchGit { + url = "https://gitub.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +}</programlisting> + </example> + + <example> <title>Fetching a repository's specific commit on an arbitrary branch</title> <para> If the revision you're looking for is in the default branch diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index f961ed921bca..0d0cbbe1553e 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -43,7 +43,7 @@ encountered</quote>).</para></footnote>.</para> <simplesect xml:id="sect-let-expressions"><title>Let-expressions</title> -<para>A let-expression allows you define local variables for an +<para>A let-expression allows you to define local variables for an expression. For instance, <programlisting> @@ -217,7 +217,25 @@ but can also be written as: ellipsis(<literal>...</literal>) as you can access attribute names as <literal>a</literal>, using <literal>args.a</literal>, which was given as an additional attribute to the function. - </para></listitem> + </para> + + <warning> + <para> + The <literal>args@</literal> expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in <literal>args</literal>. + </para> + <para> + For instance +<programlisting> +let + function = args@{ a ? 23, ... }: args; +in + function {} +</programlisting> + will evaluate to an empty attribute set. + </para> + </warning></listitem> </itemizedlist> diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index e87d0de21ef6..e7bdcf966cf6 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -13,7 +13,7 @@ <listitem><para>Bash Shell. The <literal>./configure</literal> script relies on bashisms, so Bash is required.</para></listitem> - <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem> + <listitem><para>A version of GCC or Clang that supports C++17.</para></listitem> <listitem><para><command>pkg-config</command> to locate dependencies. If your distribution does not provide it, you can get @@ -62,6 +62,10 @@ 1.66.0 or higher. It can be obtained from the official web site <link xlink:href="https://www.boost.org/" />.</para></listitem> + <listitem><para>The <literal>editline</literal> library of version + 1.14.0 or higher. It can be obtained from the its repository + <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem> + <listitem><para>The <command>xmllint</command> and <command>xsltproc</command> programs to build this manual and the man-pages. These are part of the <literal>libxml2</literal> and diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index e7589ffdb034..1722090efecc 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -113,7 +113,7 @@ the S3 URL:</para> exactly <uri>s3://example-nix-cache</uri>.</para> <para>Nix will use the <link - xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain</link> for authenticating requests to Amazon S3.</para> @@ -138,7 +138,7 @@ the S3 URL:</para> be <uri>s3://example-nix-cache</uri>.</para> <para>Nix will use the <link - xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain</link> for authenticating requests to Amazon S3.</para> diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index e8ff586fa43f..2655d68e354b 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,7 @@ </partintro> --> +<xi:include href="rl-2.3.xml" /> <xi:include href="rl-2.2.xml" /> <xi:include href="rl-2.1.xml" /> <xi:include href="rl-2.0.xml" /> diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 000000000000..428213b360ba --- /dev/null +++ b/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,22 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.3"> + +<title>Release 2.3 (????-??-??)</title> + +<para>This release contains the following changes:</para> + +<itemizedlist> + + <listitem> + <para><function>builtins.fetchGit</function>'s <varname>ref</varname> + argument now allows specifying an absolute remote ref. + Nix will automatically prefix <varname>ref</varname> with + <literal>refs/heads</literal> only if <varname>ref</varname> doesn't + already begin with <literal>refs/</literal>. + </para> + </listitem> +</itemizedlist> +</section> diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 000000000000..43087b2e6889 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,951 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com> +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com> +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu> +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> +# Copyright (c) 2015 Paul Norman <penorman@mac.com> +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template <typename T> + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check<void> single_type; + typedef check<check<void>> double_type; + typedef check<check<check<void>>> triple_type; + typedef check<check<check<check<void>>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same<int, decltype(0)>::value == true, ""); + static_assert(is_same<int, decltype(c)>::value == false, ""); + static_assert(is_same<int, decltype(v)>::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same<int, decltype(ac)>::value == true, ""); + static_assert(is_same<int, decltype(av)>::value == true, ""); + static_assert(is_same<int, decltype(sumi)>::value == true, ""); + static_assert(is_same<int, decltype(sumf)>::value == false, ""); + static_assert(is_same<int, decltype(add(c, v))>::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template <int...> + struct sum; + + template <int N0, int... N1toN> + struct sum<N0, N1toN...> + { + static constexpr auto value = N0 + sum<N1toN...>::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template<typename T> + using member = typename T::member_type; + + template<typename T> + void func(...) {} + + template<typename T> + void func(member<T>*) {} + + void test(); + + void test() { func<foo>(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same<int, decltype(f(x))>::value, ""); + static_assert(is_same<int&, decltype(g(x))>::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/m4/ax_cxx_compile_stdcxx_17.m4 b/m4/ax_cxx_compile_stdcxx_17.m4 new file mode 100644 index 000000000000..a6834171739b --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx_17.m4 @@ -0,0 +1,35 @@ +# ============================================================================= +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html +# ============================================================================= +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++17 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++17. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016 Krzesimir Nowak <qdlacz@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])]) diff --git a/mk/libraries.mk b/mk/libraries.mk index 3953446cba32..307e29b9d05b 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -91,7 +91,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifneq ($(OS), Darwin) $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) @@ -105,7 +105,7 @@ define build-library $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) ifneq ($(OS), Darwin) diff --git a/mk/programs.mk b/mk/programs.mk index 2fbda12bd153..d93df4468086 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) @@ -46,7 +46,7 @@ define build-program _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) else diff --git a/perl/Makefile b/perl/Makefile index 284c75022493..f36f5d0e9d88 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -4,11 +4,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/perl/configure.ac b/perl/configure.ac index 966700695ff5..e8e3610a860d 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -2,8 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11 diff --git a/release-common.nix b/release-common.nix index 4c5565985267..2e8a951b9cd2 100644 --- a/release-common.nix +++ b/release-common.nix @@ -42,7 +42,7 @@ rec { libxml2 libxslt docbook5 - docbook5_xsl + docbook_xsl_ns autoconf-archive autoreconfHook ]; diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index c89e18b5116d..9b757e7da3f2 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -749,9 +749,6 @@ setup_default_profile() { place_nix_configuration() { cat <<EOF > "$SCRATCH/nix.conf" build-users-group = $NIX_BUILD_GROUP_NAME - -max-jobs = $NIX_USER_COUNT -cores = 1 EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf diff --git a/scripts/install.in b/scripts/install.in index 7bff7b216d9e..4857638c0265 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -18,7 +18,7 @@ cleanup() { trap cleanup EXIT INT QUIT TERM require_util() { - type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || + command -v "$1" > /dev/null 2>&1 || oops "you do not have '$1' installed, which I need to $2" } @@ -41,11 +41,11 @@ require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" -if type sha256sum > /dev/null 2>&1; then +if command -v sha256sum > /dev/null 2>&1; then hash2="$(sha256sum -b "$tarball" | cut -c1-64)" -elif type shasum > /dev/null 2>&1; then +elif command -v shasum > /dev/null 2>&1; then hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" -elif type openssl > /dev/null 2>&1; then +elif command -v openssl > /dev/null 2>&1; then hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" else oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h index 07de010b1520..5a00da3b4cda 100644 --- a/src/cpptoml/cpptoml.h +++ b/src/cpptoml/cpptoml.h @@ -4,8 +4,8 @@ * @date May 2013 */ -#ifndef _CPPTOML_H_ -#define _CPPTOML_H_ +#ifndef CPPTOML_H +#define CPPTOML_H #include <algorithm> #include <cassert> @@ -84,11 +84,12 @@ class option return &value_; } - const T& value_or(const T& alternative) const + template <class U> + T value_or(U&& alternative) const { if (!empty_) return value_; - return alternative; + return static_cast<T>(std::forward<U>(alternative)); } private: @@ -295,13 +296,12 @@ struct valid_value_or_string_convertible }; template <class T> -struct value_traits<T, typename std:: - enable_if<valid_value_or_string_convertible<T>:: - value>::type> +struct value_traits<T, typename std::enable_if< + valid_value_or_string_convertible<T>::value>::type> { - using value_type = typename std:: - conditional<valid_value<typename std::decay<T>::type>::value, - typename std::decay<T>::type, std::string>::type; + using value_type = typename std::conditional< + valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; using type = value<value_type>; @@ -312,12 +312,11 @@ struct value_traits<T, typename std:: }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_floating_point< - typename std::decay<T>::type>::value>:: - type> +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_floating_point<typename std::decay<T>::type>::value>::type> { using value_type = typename std::decay<T>::type; @@ -330,11 +329,11 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_signed<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && !std::is_floating_point<typename std::decay<T>::type>::value + && std::is_signed<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -356,11 +355,10 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_unsigned<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -395,10 +393,15 @@ struct array_of_trait<array> template <class T> inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); inline std::shared_ptr<array> make_array(); + +namespace detail +{ template <class T> inline std::shared_ptr<T> make_element(); +} + inline std::shared_ptr<table> make_table(); -inline std::shared_ptr<table_array> make_table_array(); +inline std::shared_ptr<table_array> make_table_array(bool is_inline = false); #if defined(CPPTOML_NO_RTTI) /// Base type used to store underlying data type explicitly if RTTI is disabled @@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base> #if defined(CPPTOML_NO_RTTI) base_type type() const { - return type_; + return type_; } protected: @@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as() if (type() == base_type::INT) { auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); - return make_value<double>(static_cast<double>(v->get()));; + return make_value<double>(static_cast<double>(v->get())); } #else if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) @@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const { #if defined(CPPTOML_NO_RTTI) if (type() == base_type::FLOAT) - return std::static_pointer_cast<const value<double>>(shared_from_this()); + return std::static_pointer_cast<const value<double>>( + shared_from_this()); if (type() == base_type::INT) { @@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<array> make_element<array>() { return make_array(); } +} // namespace detail /** * Obtains a option<vector<T>>. The option will be empty if the array @@ -1060,7 +1067,7 @@ class table; class table_array : public base { friend class table; - friend std::shared_ptr<table_array> make_table_array(); + friend std::shared_ptr<table_array> make_table_array(bool); public: std::shared_ptr<base> clone() const override; @@ -1152,14 +1159,25 @@ class table_array : public base array_.reserve(n); } + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + private: #if defined(CPPTOML_NO_RTTI) - table_array() : base(base_type::TABLE_ARRAY) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) { // nothing } #else - table_array() + table_array(bool is_inline = false) : is_inline_(is_inline) { // nothing } @@ -1169,26 +1187,30 @@ class table_array : public base table_array& operator=(const table_array& rhs) = delete; std::vector<std::shared_ptr<table>> array_; + const bool is_inline_ = false; }; -inline std::shared_ptr<table_array> make_table_array() +inline std::shared_ptr<table_array> make_table_array(bool is_inline) { struct make_shared_enabler : public table_array { - make_shared_enabler() + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) { // nothing } }; - return std::make_shared<make_shared_enabler>(); + return std::make_shared<make_shared_enabler>(is_inline); } +namespace detail +{ template <> inline std::shared_ptr<table_array> make_element<table_array>() { - return make_table_array(); + return make_table_array(true); } +} // namespace detail // The below are overloads for fetching specific value types out of a value // where special casting behavior (like bounds checking) is desired @@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<table> make_element<table>() { return make_table(); } +} // namespace detail template <class T> std::shared_ptr<base> value<T>::clone() const @@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const inline std::shared_ptr<base> table_array::clone() const { - auto result = make_table_array(); + auto result = make_table_array(is_inline()); result->reserve(array_.size()); for (const auto& ptr : array_) result->array_.push_back(ptr->clone()->as_table()); @@ -1738,6 +1763,11 @@ inline bool is_number(char c) return c >= '0' && c <= '9'; } +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + /** * Helper object for consuming expected characters. */ @@ -1766,6 +1796,13 @@ class consumer [&](char c) { (*this)(c); }); } + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + int eat_digits(int len) { int val = 0; @@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line) line.push_back(static_cast<char>(c)); } } -} +} // namespace detail /** * The parser class. @@ -1914,21 +1951,25 @@ class parser std::string full_table_name; bool inserted = false; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table name"); if (!full_table_name.empty()) - full_table_name += "."; + full_table_name += '.'; full_table_name += part; if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif if (b->is_table()) curr_table = static_cast<table*>(b.get()); else if (b->is_table_array()) @@ -1946,16 +1987,23 @@ class parser curr_table->insert(part, make_table()); curr_table = static_cast<table*>(curr_table->get(part).get()); } - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); if (it == end) throw_parse_exception( "Unterminated table declaration; did you forget a ']'?"); + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + // table already existed if (!inserted) { @@ -1969,8 +2017,9 @@ class parser // since it has already been defined. If there aren't any // values, then it was implicitly created by something like // [a.b] - if (curr_table->empty() || std::any_of(curr_table->begin(), - curr_table->end(), is_value)) + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) { throw_parse_exception("Redefinition of table " + full_table_name); @@ -1989,36 +2038,45 @@ class parser if (it == end || *it == ']') throw_parse_exception("Table array name cannot be empty"); - std::string full_ta_name; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table array name"); if (!full_ta_name.empty()) - full_ta_name += "."; + full_ta_name += '.'; full_ta_name += part; - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif // if this is the end of the table array name, add an - // element to the table array that we just looked up + // element to the table array that we just looked up, + // provided it was not declared inline if (it != end && *it == ']') { if (!b->is_table_array()) + { throw_parse_exception("Key " + full_ta_name + " is not a table array"); + } + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + v->get().push_back(make_table()); curr_table = v->get().back().get(); } @@ -2059,15 +2117,16 @@ class parser = static_cast<table*>(curr_table->get(part).get()); } } - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); // consume the last "]]" - if (it == end) - throw_parse_exception("Unterminated table array name"); - ++it; - if (it == end) + auto eat = make_consumer(it, end, [this]() { throw_parse_exception("Unterminated table array name"); - ++it; + }); + eat(']'); + eat(']'); consume_whitespace(it, end); eol_or_comment(it, end); @@ -2076,7 +2135,35 @@ class parser void parse_key_value(std::string::iterator& it, std::string::iterator& end, table* curr_table) { - auto key = parse_key(it, end, [](char c) { return c == '='; }); + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast<table*>(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + if (curr_table->contains(key)) throw_parse_exception("Key " + key + " already present"); if (it == end || *it != '=') @@ -2087,18 +2174,57 @@ class parser consume_whitespace(it, end); } - template <class Function> - std::string parse_key(std::string::iterator& it, - const std::string::iterator& end, Function&& fun) + template <class KeyEndFinder, class KeyPartHandler> + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) { consume_whitespace(it, end); - if (*it == '"') + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') { - return parse_quoted_key(it, end); + return string_literal(it, end, *it); } else { - auto bke = std::find_if(it, end, std::forward<Function>(fun)); + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); return parse_bare_key(it, bke); } } @@ -2142,12 +2268,6 @@ class parser return key; } - std::string parse_quoted_key(std::string::iterator& it, - const std::string::iterator& end) - { - return string_literal(it, end, '"'); - } - enum class parse_type { STRING = 1, @@ -2193,7 +2313,7 @@ class parser parse_type determine_value_type(const std::string::iterator& it, const std::string::iterator& end) { - if(it == end) + if (it == end) { throw_parse_exception("Failed to parse value type"); } @@ -2209,7 +2329,11 @@ class parser { return *dtype; } - else if (is_number(*it) || *it == '-' || *it == '+') + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) { return determine_number_type(it, end); } @@ -2235,6 +2359,13 @@ class parser auto check_it = it; if (*check_it == '-' || *check_it == '+') ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + while (check_it != end && is_number(*check_it)) ++check_it; if (check_it != end && *check_it == '.') @@ -2283,57 +2414,56 @@ class parser bool consuming = false; std::shared_ptr<value<std::string>> ret; - auto handle_line - = [&](std::string::iterator& local_it, - std::string::iterator& local_end) { - if (consuming) - { - local_it = std::find_if_not(local_it, local_end, is_ws); - - // whole line is whitespace - if (local_it == local_end) - return; - } - - consuming = false; - - while (local_it != local_end) - { - // handle escaped characters - if (delim == '"' && *local_it == '\\') - { - auto check = local_it; - // check if this is an actual escape sequence or a - // whitespace escaping backslash - ++check; - consume_whitespace(check, local_end); - if (check == local_end) - { - consuming = true; - break; - } - - ss << parse_escape_code(local_it, local_end); - continue; - } - - // if we can end the string - if (std::distance(local_it, local_end) >= 3) - { - auto check = local_it; - // check for """ - if (*check++ == delim && *check++ == delim - && *check++ == delim) - { - local_it = check; - ret = make_value<std::string>(ss.str()); - break; - } - } - - ss << *local_it++; - } - }; + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; // handle the remainder of the current line handle_line(it, end); @@ -2514,17 +2644,13 @@ class parser return value; } - bool is_hex(char c) - { - return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - uint32_t hex_to_digit(char c) { if (is_number(c)) return static_cast<uint32_t>(c - '0'); - return 10 + static_cast<uint32_t>( - c - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + return 10 + + static_cast<uint32_t>(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); } std::shared_ptr<base> parse_number(std::string::iterator& it, @@ -2538,17 +2664,23 @@ class parser ++check_it; }; - eat_sign(); + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; - auto eat_numbers = [&]() { + auto eat_digits = [&](bool (*check_char)(char)) { auto beg = check_it; - while (check_it != end && is_number(*check_it)) + while (check_it != end && check_char(*check_it)) { ++check_it; if (check_it != end && *check_it == '_') { ++check_it; - if (check_it == end || !is_number(*check_it)) + if (check_it == end || !check_char(*check_it)) throw_parse_exception("Malformed number"); } } @@ -2557,15 +2689,63 @@ class parser throw_parse_exception("Malformed number"); }; - auto check_no_leading_zero = [&]() { - if (check_it != end && *check_it == '0' && check_it + 1 != check_end - && check_it[1] != '.') + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') { - throw_parse_exception("Numbers may not have leading zeros"); + eat_hex(); + return parse_int(it, check_it, 16); } - }; + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + eat_sign(); check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits<double>::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits<double>::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + eat_numbers(); if (check_it != end @@ -2604,14 +2784,17 @@ class parser } std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, - const std::string::iterator& end) + const std::string::iterator& end, + int base = 10, + const char* prefix = "") { std::string v{it, end}; + v = prefix + v; v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); it = end; try { - return make_value<int64_t>(std::stoll(v)); + return make_value<int64_t>(std::stoll(v, nullptr, base)); } catch (const std::invalid_argument& ex) { @@ -2674,18 +2857,33 @@ class parser std::string::iterator find_end_of_number(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { + auto ret = std::find_if(it, end, [](char c) { return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' - && c != '-' && c != '+'; + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; } std::string::iterator find_end_of_date(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { - return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-' - && c != '+' && c != '.'; + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; }); } @@ -2754,7 +2952,7 @@ class parser if (it == date_end) return make_value(ldate); - eat('T'); + eat.eat_or('T', ' '); local_datetime ldt; static_cast<local_date&>(ldt) = ldate; @@ -2850,9 +3048,9 @@ class parser auto arr = make_array(); while (it != end && *it != ']') { - auto value = parse_value(it, end); - if (auto v = value->as<Value>()) - arr->get().push_back(value); + auto val = parse_value(it, end); + if (auto v = val->as<Value>()) + arr->get().push_back(val); else throw_parse_exception("Arrays must be homogeneous"); skip_whitespace_and_comments(it, end); @@ -2871,7 +3069,7 @@ class parser std::string::iterator& it, std::string::iterator& end) { - auto arr = make_element<Object>(); + auto arr = detail::make_element<Object>(); while (it != end && *it != ']') { @@ -2881,7 +3079,7 @@ class parser arr->get().push_back(((*this).*fun)(it, end)); skip_whitespace_and_comments(it, end); - if (*it != ',') + if (it == end || *it != ',') break; ++it; @@ -2906,8 +3104,11 @@ class parser throw_parse_exception("Unterminated inline table"); consume_whitespace(it, end); - parse_key_value(it, end, tbl.get()); - consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } } while (*it == ','); if (it == end || *it != '}') @@ -2987,7 +3188,8 @@ class parser if (it[4] != '-' || it[7] != '-') return {}; - if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end)) + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) { // datetime type auto time_end = find_end_of_time(it + 11, date_end); @@ -3243,7 +3445,7 @@ class toml_writer { res += "\\\\"; } - else if ((const uint32_t)*it <= 0x001f) + else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f)) { res += "\\u"; std::stringstream ss; @@ -3274,12 +3476,21 @@ class toml_writer */ void write(const value<double>& v) { - std::ios::fmtflags flags{stream_.flags()}; - - stream_ << std::showpoint; - write(v.get()); - - stream_.flags(flags); + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits<double>::max_digits10) + << v.get(); + + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); + + stream_ << double_str; + has_naked_endline_ = false; } /** @@ -3287,9 +3498,9 @@ class toml_writer * offset_datetime. */ template <class T> - typename std::enable_if<is_one_of<T, int64_t, local_date, local_time, - local_datetime, - offset_datetime>::value>::type + typename std::enable_if< + is_one_of<T, int64_t, local_date, local_time, local_datetime, + offset_datetime>::value>::type write(const value<T>& v) { write(v.get()); @@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a) a.accept(writer); return stream; } -} -#endif +} // namespace cpptoml +#endif // CPPTOML_H diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f280f7..13950ab8d169 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 7870393076b8..967c88d9bc80 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -81,6 +81,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); @@ -102,11 +104,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, attrs = nested; } } + // Expr insertion. + // ========================== if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast<ExprAttrs *>(e); + auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } } else { + // This attr path is not defined. Let's create it. attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); e->setName(i->symbol); } @@ -657,7 +677,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 06f577f36fce..070e72f3a966 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2050,9 +2050,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2063,27 +2063,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, request).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index aaf02c856d4f..6229fef8d02e 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -94,7 +94,11 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, runProgram("git", true, { "init", "--bare", cacheDir }); } - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; bool doFetch; time_t now = time(0); @@ -116,7 +120,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; } if (doFetch) { @@ -235,7 +239,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 66f49f374321..a907d0e1cd82 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now) + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ @@ -96,17 +96,14 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, try { runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); } - catch (ExecError & e){ + catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) - { + if (pathExists(transJournal)) { runProgram("hg", true, { "recover", "-R", cacheDir }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - else - { - throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status))); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { @@ -214,7 +211,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4128de05d0cf..a84e569e944d 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); } + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + else if (t->is_value()) { if (auto val = t->as<int64_t>()) mkInt(v, val->get()); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 4c35a4199590..9e1d7cee60e6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { + settings.set("max-jobs", s); + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54dc55..cd752f4678a0 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -175,10 +175,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description("build from source if substitution fails") .set(&(bool&) settings.tryFallback, true); - mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { - settings.set("max-jobs", s); - }); - auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba660..8b736056e01d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,6 +10,8 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "retry.hh" +#include "download.hh" #include <chrono> @@ -79,13 +81,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; + return retry<std::shared_ptr<std::string>>(downloadSettings.tries, [&]() -> std::shared_ptr<std::string> { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + }); } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0bd7388097c6..350ac4092854 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -38,6 +38,7 @@ #include <unistd.h> #include <errno.h> #include <cstring> +#include <termios.h> #include <pwd.h> #include <grp.h> @@ -265,6 +266,12 @@ public: /* Set if at least one derivation had a timeout. */ bool timedOut; + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + LocalStore & store; std::unique_ptr<HookInstance> hook; @@ -1558,8 +1565,8 @@ void DerivationGoal::buildDone() if (hook) { hook->builderOut.readSide = -1; hook->fromHook.readSide = -1; - } - else builderOut.readSide = -1; + } else + builderOut.readSide = -1; /* Close the log file. */ closeLogFile(); @@ -2181,7 +2188,48 @@ void DerivationGoal::startBuilder() Path logFile = openLogFile(); /* Create a pipe to get the output of the builder. */ - builderOut.create(); + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); result.startTime = time(0); @@ -2406,6 +2454,9 @@ void DerivationGoal::initEnv() may change that in the future. So tell the builder which file descriptor to use for that. */ env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; } @@ -3168,6 +3219,7 @@ void DerivationGoal::registerOutputs() /* Throw an error after registering the path as valid. */ + worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", dest, h.to_string(), h2.to_string())); @@ -3210,6 +3262,7 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { + worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); @@ -3221,10 +3274,10 @@ void DerivationGoal::registerOutputs() buildUser ? buildUser->getGID() : getgid(), path, dst, drvPath, tmpDir); - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") % drvPath % path); } @@ -4056,6 +4109,8 @@ Worker::Worker(LocalStore & store) lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; + hashMismatch = false; + checkMismatch = false; } @@ -4361,14 +4416,15 @@ void Worker::waitForInput() for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { ssize_t rd = read(k, buffer.data(), buffer.size()); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); @@ -4415,7 +4471,29 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return timedOut ? 101 : (permanentFailure ? 100 : 1); + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 + if (timedOut) + mask |= 0x01; // 101 + if (hashMismatch) + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) + mask |= 0x60; + return mask ? mask : 1; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 92aec63a0379..b1af3b4fc316 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -64,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 22382ab1d6e8..7a2af237ee8f 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "retry.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -19,34 +20,16 @@ #include <curl/curl.h> #include <algorithm> -#include <cmath> #include <cstring> #include <iostream> #include <queue> -#include <random> #include <thread> using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting<bool> enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting<size_t> httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); @@ -62,9 +45,6 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; - std::random_device rd; - std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this<DownloadItem> { CurlDownloader & downloader; @@ -77,12 +57,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - struct curl_slist * requestHeaders = 0; std::string encoding; @@ -270,6 +244,8 @@ struct CurlDownloader : public Downloader #if LIBCURL_VERSION_NUM >= 0x072f00 if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); @@ -319,16 +295,21 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; @@ -396,9 +377,7 @@ struct CurlDownloader : public Downloader } } - attempt++; - - auto exc = + fail( code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -409,31 +388,15 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); - - /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); + request.verb(), request.uri, curl_easy_strerror(code), code))); } } }; struct State { - struct EmbargoComparator { - bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) { - return i1->embargo > i2->embargo; - } - }; bool quit = false; - std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming; + std::vector<std::shared_ptr<DownloadItem>> incoming; }; Sync<State> state_; @@ -446,7 +409,6 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() - : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -540,9 +502,7 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ + /* Add new curl requests from the incoming requests queue. */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -551,22 +511,9 @@ struct CurlDownloader : public Downloader } std::vector<std::shared_ptr<DownloadItem>> incoming; - auto now = std::chrono::steady_clock::now(); - { auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } + std::swap(state->incoming, incoming); quit = state->quit; } @@ -593,7 +540,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); + state->incoming.clear(); state->quit = true; } } @@ -609,7 +556,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); + state->incoming.push_back(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -692,7 +639,9 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return enqueueDownload(request).get(); + return retry<DownloadResult>(request.tries, [&]() { + return enqueueDownload(request).get(); + }); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -790,20 +739,26 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref<Store> store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -822,6 +777,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -829,10 +786,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -845,17 +802,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -870,11 +827,12 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("%s; using cached result", e.msg()); + result.etag = expectedETag; } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -897,14 +855,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + unsigned int statusCode = 102; + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + url, request.expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; } @@ -917,5 +878,4 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } - } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d053a..9e965b506d0a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,34 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting<unsigned int> tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -36,11 +57,33 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr<std::string> data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + +struct CachedDownloadResult +{ + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional<std::string> etag; + std::string effectiveUri; +}; + class Store; struct Downloader @@ -53,19 +96,20 @@ struct Downloader std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. */ + /* Synchronously download a file. The request will be retried in + case of transient failures. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + invoked on the thread of the caller. The request will not be + retried in case of transient failures. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; @@ -84,6 +128,11 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } + + bool isTransient() override + { + return error == Downloader::Error::Transient; + } }; bool isUri(const string & s); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90fb6..0af8215d1fd8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,7 +255,7 @@ public: "Secret keys with which to sign local builds."}; Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting<bool> requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d82a..5633b4355d25 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "retry.hh" namespace nix { @@ -84,7 +85,6 @@ protected: try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { @@ -114,7 +114,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } @@ -137,21 +136,46 @@ protected: { checkEnabled(); - auto request(makeRequest(path)); - - getDownloader()->enqueueDownload(request, - {[callback, this](std::future<DownloadResult> result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr<std::string>()); - maybeDisable(); - callback.rethrow(); - } catch (...) { - callback.rethrow(); - } - }}); + struct State + { + DownloadRequest request; + std::function<void()> tryDownload; + unsigned int attempt = 0; + State(DownloadRequest && request) : request(request) {} + }; + + auto state = std::make_shared<State>(makeRequest(path)); + + state->tryDownload = [callback, state, this]() { + getDownloader()->enqueueDownload(state->request, + {[callback, state, this](std::future<DownloadResult> result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr<std::string>()); + ++state->attempt; + if (state->attempt < state->request.tries && e.isTransient()) { + auto ms = retrySleepTime(state->attempt); + warn("%s; retrying in %d ms", e.what(), ms); + /* We can't sleep here because that would + block the download thread. So use a + separate thread for sleeping. */ + std::thread([state, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + state->tryDownload(); + }).detach(); + } else { + maybeDisable(); + callback.rethrow(); + } + } catch (...) { + callback.rethrow(); + } + }}); + }; + + state->tryDownload(); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff11564ec..92f01fd2e0c7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,10 +6,11 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "retry.hh" +#include "download.hh" #include <future> - namespace nix { @@ -85,18 +86,25 @@ string storePathToHash(const Path & path) void checkStoreName(const string & name) { string validChars = "+-._?="; + + auto baseError = format("The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % validChars % name; + /* Disallow names starting with a dot for possible security reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") - throw Error(format("illegal name: '%1%'") % name); + throw Error(baseError % "it is illegal to start the name with a period"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { - throw Error(format("invalid character '%1%' in name '%2%'") - % i % name); + throw Error(baseError % (format("the '%1%' character is invalid") % i)); } } @@ -572,54 +580,57 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); - - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); - - auto info = srcStore->queryPathInfo(storePath); - - uint64_t total = 0; - - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; - - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + retry<void>(downloadSettings.tries, [&]() { - if (info->ultimate) { - auto info2 = make_ref<ValidPathInfo>(*info); - info2->ultimate = false; - info = info2; - } + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore->queryPathInfo(storePath); + + uint64_t total = 0; - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref<ValidPathInfo>(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); + }); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7a1b31d0ff59..59967737670d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -766,8 +766,7 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(), const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options like - ‘binary-caches’. */ + ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 3ccc23fd5c1b..e41a67d1f9e1 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,4 +6,4 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 799c6e1ae441..b379306f6ec0 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger(); void Logger::warn(const std::string & msg) { - log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } class SimpleLogger : public Logger @@ -46,6 +46,7 @@ public: char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 678703102e9b..5f221944594b 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -6,6 +6,7 @@ namespace nix { typedef enum { lvlError = 0, + lvlWarn, lvlInfo, lvlTalkative, lvlChatty, diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh new file mode 100644 index 000000000000..b45cb37f736b --- /dev/null +++ b/src/libutil/retry.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "logging.hh" + +#include <functional> +#include <cmath> +#include <random> +#include <thread> + +namespace nix { + +inline unsigned int retrySleepTime(unsigned int attempt) +{ + std::random_device rd; + std::mt19937 mt19937; + return 250.0 * std::pow(2.0f, + attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937)); +} + +template<typename C> +C retry(unsigned int attempts, std::function<C()> && f) +{ + unsigned int attempt = 0; + while (true) { + try { + return f(); + } catch (BaseError & e) { + ++attempt; + if (attempt >= attempts || !e.isTransient()) + throw; + auto ms = retrySleepTime(attempt); + warn("%s; retrying in %d ms", e.what(), ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 92bf469b5c6f..88e3243f47a5 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -109,6 +109,8 @@ public: const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); + + virtual bool isTransient() { return false; } }; #define MakeError(newClass, superClass) \ diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e314e..06eb3d23ba83 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -86,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -111,22 +113,11 @@ static void update(const StringSet & channelNames) } if (!unpacked) { - // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - // Check if the channel advertises a binary cache. - DownloadRequest request(url + "/binary-cache-url"); - try { - auto dlRes = dl->download(request); - extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; - } catch (DownloadError & e) { - } - // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8d63b8f362ec..e88aaf636444 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else if (setSubstituters(settings.extraSubstituters)) ; else - debug("ignoring untrusted setting '%s'", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } diff --git a/src/nix/local.mk b/src/nix/local.mk index ca4604d566c3..c09efd1fc895 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -17,7 +17,7 @@ nix_SOURCES := \ nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/src/nix/main.cc b/src/nix/main.cc index 4f87ad72b65c..a80fd0ea62fc 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,19 +8,52 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "download.hh" #include "finally.hh" +#include <sys/types.h> +#include <sys/socket.h> +#include <ifaddrs.h> +#include <netdb.h> + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) + return true; + } + } + + return false; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; + bool useNet = true; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -45,6 +78,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs mkFlag() .longName("print-build-logs") + .shortName('L') .description("print full build logs on stderr") .set(&printBuildLogs, true); @@ -52,6 +86,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); } void printFlags(std::ostream & out) override @@ -92,7 +131,7 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlError; + verbosity = lvlWarn; settings.verboseBuild = false; NixArgs args; @@ -107,6 +146,23 @@ void mainWrapped(int argc, char * * argv) startProgressBar(args.printBuildLogs); + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits<unsigned int>::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } + args.command->prepare(); args.command->run(); } diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index e7104540816b..b1c1d87de1a2 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -62,6 +62,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool haveUpdate = true; }; Sync<State> state_; @@ -83,7 +84,8 @@ public: updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { - state.wait(updateCV); + if (!state->haveUpdate) + state.wait(updateCV); draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } @@ -178,7 +180,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(); + update(*state); } /* Check whether an activity has an ancestore with the specified @@ -213,7 +215,7 @@ public: state->its.erase(i); } - update(); + update(*state); } void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override @@ -223,7 +225,7 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(); + update(*state); } else if (type == resBuildLogLine) { @@ -239,26 +241,26 @@ public: info.lastLine = lastLine; state->activities.emplace_back(info); i->second = std::prev(state->activities.end()); - update(); + update(*state); } } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(); + update(*state); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(); + update(*state); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(); + update(*state); } else if (type == resProgress) { @@ -269,7 +271,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(); + update(*state); } else if (type == resSetExpected) { @@ -281,17 +283,19 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(); + update(*state); } } - void update() + void update(State & state) { + state.haveUpdate = true; updateCV.notify_one(); } void draw(State & state) { + state.haveUpdate = false; if (!state.active) return; std::string line; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d8f812149069..f857b2e89c29 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -9,7 +9,14 @@ #include <readline/history.h> #include <readline/readline.h> #else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { #include <editline.h> +} #endif #include "shared.hh" diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 610e6070c5d7..e72533e70614 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -8,13 +8,13 @@ clearStore clearCache # Ensure this builds successfully first -nix build -f dependencies.nix +nix build --no-link -f dependencies.nix clearStore clearCache # Try --dry-run using old command first -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" # Now new command: nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" @@ -27,7 +27,7 @@ clearCache # Try --dry-run using new command first nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" fi ################################################### diff --git a/tests/check.nix b/tests/check.nix index 08aac2fb0a77..56c82e565a8f 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -10,6 +10,11 @@ with import ./config.nix; ''; }; + hashmismatch = import <nix/fetchurl.nix> { + url = "file://" + toString ./dummy; + sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; + }; + fetchurl = import <nix/fetchurl.nix> { url = "file://" + toString ./lang/eval-okay-xml.exp.xml; sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v"; diff --git a/tests/check.sh b/tests/check.sh index b05e40ffbeea..bc23a6634ca0 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -6,14 +6,16 @@ nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check nix-build check.nix -A nondeterministic --no-out-link -(! nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] clearStore nix-build dependencies.nix --no-out-link --repeat 3 -(! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? +[ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') @@ -23,10 +25,23 @@ echo foo > $path chmod -w $path nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' - # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' - [[ $(cat $path) != foo ]] + +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +echo -n > ./dummy +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +echo 'Hello World' > ./dummy + +nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +# Multiple failures with --keep-going +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +[ "$status" = "110" ] diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index ec3399b08d00..7319ced2b599 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -42,6 +42,10 @@ ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +# Test hashed mirrors with an SRI hash. +nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ + --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + # Test unpacking a NAR. rm -rf $TEST_ROOT/archive mkdir -p $TEST_ROOT/archive diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 5b9d47122cc5..d0dd3af2c814 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index 8e7cbd1c61e9..963932689942 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -46,15 +46,15 @@ "character encoding" = "value" "ʎǝʞ" = "value" 'key2' = "value" - #'quoted "value"' = "value" + 'quoted "value"' = "value" name = "Orange" - # FIXME: cpptoml doesn't handle dotted keys properly yet. - #physical.color = "orange" - #physical.shape = "round" - #site."google.com" = true + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + # This is legal according to the spec, but cpptoml doesn't handle it. #a.b.c = 1 #a.d = 2 @@ -68,16 +68,14 @@ int6 = 5_349_221 int7 = 1_2_3_4_5 - # FIXME: cpptoml doesn't support these yet: + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef - #hex1 = 0xDEADBEEF - #hex2 = 0xdeadbeef - #hex3 = 0xdead_beef + oct1 = 0o01234567 + oct2 = 0o755 - #oct1 = 0o01234567 - #oct2 = 0o755 - - #bin1 = 0b11010110 + bin1 = 0b11010110 flt1 = +1.0 flt2 = 3.1415 @@ -126,8 +124,8 @@ key1 = "another string" key2 = 456 - #[dog."tater.man"] - #type.name = "pug" + [dog."tater.man"] + type.name = "pug" [a.b.c] [ d.e.f ] @@ -137,7 +135,7 @@ name = { first = "Tom", last = "Preston-Werner" } point = { x = 1, y = 2 } - #animal = { type.name = "pug" } + animal = { type.name = "pug" } [[products]] name = "Hammer" @@ -149,6 +147,25 @@ name = "Nail" sku = 284758393 color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" '') (builtins.fromTOML '' @@ -181,4 +198,11 @@ "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" '') + + (builtins.fromTOML '' + a = [[{ b = true }]] + c = [ [ { d = true } ] ] + e = [[123]] + '') + ] diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/lang/parse-fail-mixed-nested-attrs1.nix new file mode 100644 index 000000000000..11e40e66fd1b --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.nix @@ -0,0 +1,4 @@ +{ + x.z = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/lang/parse-fail-mixed-nested-attrs2.nix new file mode 100644 index 000000000000..17da82e5f0c7 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.nix @@ -0,0 +1,4 @@ +{ + x.y.y = 3; + x = { y.y= 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-dup-attrs-6.nix b/tests/lang/parse-okay-dup-attrs-6.nix index ae6d7a769305..ae6d7a769305 100644 --- a/tests/lang/parse-fail-dup-attrs-6.nix +++ b/tests/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/lang/parse-okay-mixed-nested-attrs-1.nix new file mode 100644 index 000000000000..fd1001c8cafc --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/lang/parse-okay-mixed-nested-attrs-2.nix new file mode 100644 index 000000000000..ad066b680384 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.nix @@ -0,0 +1,4 @@ +{ + x.q = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/lang/parse-okay-mixed-nested-attrs-3.nix new file mode 100644 index 000000000000..45a33e480373 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.nix @@ -0,0 +1,7 @@ +{ + services.ssh.enable = true; + services.ssh = { port = 123; }; + services = { + httpd.enable = true; + }; +} diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh index 6aba667a45a6..eb801548d2f1 100644 --- a/tests/nix-copy-ssh.sh +++ b/tests/nix-copy-ssh.sh @@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2 chmod -R u+w "$remoteRoot" || true rm -rf "$remoteRoot" -outPath=$(nix-build dependencies.nix) +outPath=$(nix-build --no-out-link dependencies.nix) nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 6024ea399750..ee502dddb955 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv -[[ $(nix-shell --pure $PWD/shell.drv --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell -[[ $(nix-shell --pure shell --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc893..cd1bb7bc2aac 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY diff --git a/tests/timeout.sh b/tests/timeout.sh index 39ecf0a1a30c..eea9b5731da0 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -2,10 +2,14 @@ source common.sh -failed=0 -messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`" -if [ $failed -ne 0 ]; then - echo "error: 'nix-store' succeeded; should have timed out" + +set +e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) +status=$? +set -e + +if [ $status -ne 101 ]; then + echo "error: 'nix-store' exited with '$status'; should have exited 101" exit 1 fi |