diff options
52 files changed, 761 insertions, 389 deletions
diff --git a/Makefile b/Makefile index 8815d5c8e591..14be271bb107 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ makefiles = \ tests/local.mk #src/download-via-ssh/local.mk \ -GLOBAL_CXXFLAGS += -std=c++11 -g -Wall +GLOBAL_CXXFLAGS += -std=c++14 -g -Wall -include Makefile.config diff --git a/Makefile.config.in b/Makefile.config.in index 2db7172b15c9..a03776d572ae 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -18,7 +18,6 @@ bsddiff_compat_include = @bsddiff_compat_include@ curl = @curl@ datadir = @datadir@ datarootdir = @datarootdir@ -dblatex = @dblatex@ docdir = @docdir@ exec_prefix = @exec_prefix@ includedir = @includedir@ diff --git a/README.md b/README.md index 6d1b3243f5dd..1eb73b256f55 100644 --- a/README.md +++ b/README.md @@ -19,4 +19,4 @@ of the manual. It helps you to get started with building Nix from source. Nix is released under the LGPL v2.1 This product includes software developed by the OpenSSL Project for -use in the OpenSSL Toolkit (http://www.OpenSSL.org/). +use in the [OpenSSL Toolkit](http://www.OpenSSL.org/). diff --git a/configure.ac b/configure.ac index 91ed9947abdd..e6b11be2df19 100644 --- a/configure.ac +++ b/configure.ac @@ -128,7 +128,6 @@ NEED_PROG(bzip2, bzip2) NEED_PROG(gzip, gzip) NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) -AC_PATH_PROG(dblatex, dblatex) AC_PATH_PROG(pv, pv, pv) @@ -214,7 +213,7 @@ if test "$gc" = yes; then fi -# Check for the required Perl dependencies (DBI, DBD::SQLite and WWW::Curl). +# Check for the required Perl dependencies (DBI, DBD::SQLite). perlFlags="-I$perllibdir" AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], @@ -225,10 +224,6 @@ AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], [prefix of the Perl DBD::SQLite library]), perlFlags="$perlFlags -I$withval") -AC_ARG_WITH(www-curl, AC_HELP_STRING([--with-www-curl=PATH], - [prefix of the Perl WWW::Curl library]), - perlFlags="$perlFlags -I$withval") - AC_MSG_CHECKING([whether DBD::SQLite works]) if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then AC_MSG_RESULT(no) @@ -236,13 +231,6 @@ if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then fi AC_MSG_RESULT(yes) -AC_MSG_CHECKING([whether WWW::Curl works]) -if ! $perl $perlFlags -e 'use WWW::Curl;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl module WWW::Curl is missing.]) -fi -AC_MSG_RESULT(yes) - AC_SUBST(perlFlags) diff --git a/dev-shell b/dev-shell deleted file mode 100755 index 5a090ded6080..000000000000 --- a/dev-shell +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -if [ -e tests/test-tmp ]; then - chmod -R u+w tests/test-tmp - rm -rf tests/test-tmp -fi - -s=$(type -p nix-shell) -exec $s release.nix -A tarball --command " - unset http_proxy - export NIX_REMOTE=$NIX_REMOTE - export NIX_PATH='$NIX_PATH' - export NIX_BUILD_SHELL=$(type -p bash) - export c=\$configureFlags - exec $s release.nix -A build.$(if [ $(uname -s) = Darwin ]; then echo x86_64-darwin; elif [[ $(uname -m) =~ ^i[3456]86$ ]]; then echo i686-linux; else echo x86_64-linux; fi) --exclude tarball --command ' - configureFlags+=\" \$c --prefix=$(pwd)/inst --sysconfdir=$(pwd)/inst/etc\" - return - '" \ - "$@" diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index d2c9145e0505..6c0af39ecda9 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -18,7 +18,8 @@ <refsection><title>Description</title> <para>A number of persistent settings of Nix are stored in the file -<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>. +<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> or +<filename>$NIX_CONF_DIR/nix.conf</filename> if <envar>NIX_CONF_DIR</envar> is set. This file is a list of <literal><replaceable>name</replaceable> = <replaceable>value</replaceable></literal> pairs, one per line. Comments start with a <literal>#</literal> character. Here is an example diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 2ed4a5d9f666..85f10e0760bc 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -493,17 +493,11 @@ set returned by calling the function defined in source: <screen> -$ nix-env -f pkgs/top-level/all-packages.nix -i f-spot --dry-run +$ nix-env -f '<nixpkgs>' -iA hello --dry-run (dry run; not doing anything) -installing `f-spot-0.0.10' -the following derivations will be built: - /nix/store/0g63jv9aagwbgci4nnzs2dkxqz84kdja-libgnomeprintui-2.12.1.tar.bz2.drv - /nix/store/0gfarvxq6sannsdw8a1ir40j1ys2mqb4-ORBit2-2.14.2.tar.bz2.drv - /nix/store/0i9gs5zc04668qiy60ga2rc16abkj7g8-sqlite-2.8.17.drv - <replaceable>...</replaceable> -the following paths will be substituted: - /nix/store/8zbipvm4gp9jfqh9nnk1n3bary1a37gs-perl-XML-Parser-2.34 - /nix/store/b8a2bg7gnyvvvjjibp4axg9x1hzkw36c-mono-1.1.4 +installing ‘hello-2.10’ +these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): + /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 <replaceable>...</replaceable></screen> </para> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index bc26a90616e4..2a076877a1b4 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -191,6 +191,23 @@ </varlistentry> +<varlistentry><term><option>--no-build-hook</option></term> + + <listitem> + + <para>Disables the build hook mechanism. This allows to ignore remote + builders if they are setup on the machine.</para> + + <para>It's useful in cases where the bandwidth between the client and the + remote builder is too low. In that case it can take more time to upload the + sources to the remote builder and fetch back the result than to do the + computation locally.</para> + + </listitem> + +</varlistentry> + + <varlistentry><term><option>--readonly-mode</option></term> @@ -218,9 +235,8 @@ named <replaceable>name</replaceable>, it will call it with value <replaceable>value</replaceable>.</para> - <para>For instance, the file - <literal>pkgs/top-level/all-packages.nix</literal> in Nixpkgs is - actually a function: + <para>For instance, the top-level <literal>default.nix</literal> in + Nixpkgs is actually a function: <programlisting> { # The system (e.g., `i686-linux') for which to build the packages. diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 9517f20106ef..063bc04be483 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -210,6 +210,35 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> + <varlistentry><term><function>builtins.match</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + + <listitem><para>Returns a list if + <replaceable>regex</replaceable> matches + <replaceable>str</replaceable> precisely, otherwise returns <literal>null</literal>. + Each item in the list is a regex group. + +<programlisting> +builtins.match "ab" "abc" +</programlisting> + +Evaluates to <literal>null</literal>. + +<programlisting> +builtins.match "abc" "abc" +</programlisting> + +Evaluates to <literal>[ ]</literal>. + +<programlisting> +builtins.match "a(b)(c)" "abc" +</programlisting> + +Evaluates to <literal>[ "b" "c" ]</literal>. + + + </para></listitem> + </varlistentry> <varlistentry><term><function>builtins.elem</function> <replaceable>x</replaceable> <replaceable>xs</replaceable></term> diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml index b90baac5054c..67da688a4fc5 100644 --- a/doc/manual/expressions/language-values.xml +++ b/doc/manual/expressions/language-values.xml @@ -167,7 +167,16 @@ stdenv.mkDerivation { user's home directory. e.g. <filename>~/foo</filename> would be equivalent to <filename>/home/edolstra/foo</filename> for a user whose home directory is <filename>/home/edolstra</filename>. - </para></listitem> + </para> + + <para>Paths can also be specified between angle brackets, e.g. + <literal><nixpkgs></literal>. This means that the directories + listed in the environment variable + <envar linkend="env-NIX_PATH">NIX_PATH</envar> will be searched + for the given file or directory name. + </para> + + </listitem> <listitem><para><emphasis>Booleans</emphasis> with values <literal>true</literal> and diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index 11af0998f982..183aed7adff2 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -22,7 +22,7 @@ $ nix-build release.nix -A build.x86_64-linux environment variables are set up so that those dependencies can be found: <screen> -$ ./dev-shell +$ nix-shell </screen> To build Nix itself in this shell: <screen> @@ -30,7 +30,7 @@ To build Nix itself in this shell: [nix-shell]$ configurePhase [nix-shell]$ make </screen> -To test it: +To install it in <literal>$(pwd)/nix</literal> and test it: <screen> [nix-shell]$ make install [nix-shell]$ make installcheck diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index c0065f133700..cd6d61356ba1 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -34,7 +34,7 @@ or higher. If your distribution does not provide it, please install it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem> - <listitem><para>The Perl DBI, DBD::SQLite, and WWW::Curl libraries, which are + <listitem><para>The Perl DBI and DBD::SQLite libraries, which are available from <link xlink:href="http://search.cpan.org/">CPAN</link> if your distribution does not provide them.</para></listitem> diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 6c9a724dd6a8..743829e3f5f3 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -86,6 +86,7 @@ my ($tarball_x86_64_linux, $tarball_x86_64_linux_hash) = downloadFile("binaryTar my ($tarball_x86_64_darwin, $tarball_x86_64_darwin_hash) = downloadFile("binaryTarball.x86_64-darwin", "1"); # Update Nixpkgs in a very hacky way. +system("cd $nixpkgsDir && git pull") == 0 or die; my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName; my $oldHash = `nix-instantiate --eval $nixpkgsDir -A nix.src.outputHash`; chomp $oldHash; print STDERR "old stable version in Nixpkgs = $oldName / $oldHash\n"; @@ -135,6 +136,9 @@ system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") # Update the website. my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine"; + +system("cd $siteDir && git pull") == 0 or die; + write_file("$siteDir/nix-release.tt", "[%-\n" . "latestNixVersion = \"$version\"\n" . diff --git a/nix.spec.in b/nix.spec.in index edbc12d8f05c..401a2dc8a1f9 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -16,7 +16,6 @@ BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) %endif BuildRequires: perl(DBD::SQLite) BuildRequires: perl(DBI) -BuildRequires: perl(WWW::Curl) BuildRequires: perl(ExtUtils::ParseXS) Requires: /usr/bin/perl Requires: curl diff --git a/release.nix b/release.nix index 6b16bc718a31..d825dd583768 100644 --- a/release.nix +++ b/release.nix @@ -33,7 +33,6 @@ let configureFlags = '' --with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} - --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix} --enable-gc ''; @@ -85,7 +84,6 @@ let --disable-init-state --with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} - --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix} --enable-gc --sysconfdir=/etc ''; @@ -157,7 +155,6 @@ let --disable-init-state --with-dbi=${perlPackages.DBI}/${perl.libPrefix} --with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix} - --with-www-curl=${perlPackages.WWWCurl}/${perl.libPrefix} ''; dontInstall = false; @@ -284,7 +281,7 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "perl-WWW-Curl" "libcurl-devel" "openssl-devel" "xz-devel" ] + [ "perl-DBD-SQLite" "perl-devel" "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ] ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index fd38a4528cd7..1b4d92632f8e 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -7,7 +7,7 @@ self="$(dirname "$0")" nix="@nix@" cacert="@cacert@" -if ! [ -e $self/.reginfo ]; then +if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 exit 1 fi @@ -39,10 +39,10 @@ fi mkdir -p $dest/store -echo -n "copying Nix to $dest/store..." >&2 +printf "copying Nix to %s..." "${dest}/store" >&2 -for i in $(cd $self/store >/dev/null && echo *); do - echo -n "." >&2 +for i in $(cd "$self/store" >/dev/null && echo ./*); do + printf "." >&2 i_tmp="$dest/store/$i.$$" if [ -e "$i_tmp" ]; then rm -rf "$i_tmp" @@ -63,20 +63,20 @@ if ! $nix/bin/nix-store --init; then exit 1 fi -if ! $nix/bin/nix-store --load-db < $self/.reginfo; then +if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then echo "$0: unable to register valid paths" >&2 exit 1 fi -. $nix/etc/profile.d/nix.sh +. "$nix/etc/profile.d/nix.sh" -if ! $nix/bin/nix-env -i "$nix"; then +if ! "$nix/bin/nix-env" -i "$nix"; then echo "$0: unable to install Nix into your default profile" >&2 exit 1 fi # Install an SSL certificate bundle. -if [ -z "$NIX_SSL_CERT_FILE" -o ! -f "$NIX_SSL_CERT_FILE" ]; then +if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then $nix/bin/nix-env -i "$cacert" export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" fi @@ -100,7 +100,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> $fn + echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" fi added=1 break diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000000..dd448410554e --- /dev/null +++ b/shell.nix @@ -0,0 +1,34 @@ +with import <nixpkgs> {}; + +stdenv.mkDerivation { + name = "nix"; + + buildInputs = + [ curl bison flex perl libxml2 libxslt bzip2 xz + pkgconfig sqlite libsodium boehmgc + docbook5 docbook5_xsl + autoconf-archive + (aws-sdk-cpp.override { + apis = ["s3"]; + customMemoryManagement = false; + }) + autoreconfHook + perlPackages.DBDSQLite + ]; + + configureFlags = + [ "--disable-init-state" + "--enable-gc" + "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" + ]; + + enableParallelBuilding = true; + + installFlags = "sysconfdir=$(out)/etc"; + + shellHook = + '' + configureFlags+=" --prefix=$(pwd)/inst" + ''; +} diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index f3660ab43723..5b1ff0350cd1 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -87,8 +87,8 @@ static Expr * unescapeStr(SymbolTable & symbols, const char * s) ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* INT [0-9]+ FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? -PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+ -HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+ +PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ @@ -182,14 +182,22 @@ or { return OR_KW; } <INITIAL,INSIDE_DOLLAR_CURLY>{ -{PATH} { yylval->path = strdup(yytext); return PATH; } -{HPATH} { yylval->path = strdup(yytext); return HPATH; } +{PATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path ‘%s’ has a trailing slash", yytext); + yylval->path = strdup(yytext); + return PATH; + } +{HPATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path ‘%s’ has a trailing slash", yytext); + yylval->path = strdup(yytext); + return HPATH; + } {SPATH} { yylval->path = strdup(yytext); return SPATH; } {URI} { yylval->uri = strdup(yytext); return URI; } [ \t\r\n]+ /* eat up whitespace */ \#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*[^\/])*\*\/ /* long comments */ +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ . return yytext[0]; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4398cc951da2..5be61c647496 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -779,6 +779,9 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va string s = readFile(state.checkSourcePath(path)); if (s.find((char) 0) != string::npos) throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path); + context = state.store->isInStore(path) ? + state.store->queryPathInfo(state.store->toStorePath(path))->references : + PathSet{}; mkString(v, s.c_str(), context); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0c6e3fb76d64..12f083c7f794 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -24,12 +24,6 @@ namespace nix { -static void sigintHandler(int signo) -{ - _isInterrupted = 1; -} - - static bool gcWarning = true; void printGCWarning() @@ -120,19 +114,11 @@ void initNix() settings.processEnvironment(); settings.loadConfFile(); - /* Catch SIGINT. */ - struct sigaction act; - act.sa_handler = sigintHandler; - sigemptyset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, 0)) - throw SysError("installing handler for SIGINT"); - if (sigaction(SIGTERM, &act, 0)) - throw SysError("installing handler for SIGTERM"); - if (sigaction(SIGHUP, &act, 0)) - throw SysError("installing handler for SIGHUP"); + startSignalHandlerThread(); /* Ignore SIGPIPE. */ + struct sigaction act; + sigemptyset(&act.sa_mask); act.sa_handler = SIG_IGN; act.sa_flags = 0; if (sigaction(SIGPIPE, &act, 0)) @@ -347,7 +333,7 @@ RunPager::~RunPager() if (pid != -1) { std::cout.flush(); close(STDOUT_FILENO); - pid.wait(true); + pid.wait(); } } catch (...) { ignoreException(); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b61ea5298e1e..7fc6ff0df0f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,9 +17,9 @@ #include <sstream> #include <thread> #include <future> +#include <chrono> #include <limits.h> -#include <time.h> #include <sys/time.h> #include <sys/wait.h> #include <sys/types.h> @@ -187,6 +187,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) { } +typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; + + /* A mapping used to remember for each child process to what goal it belongs, and file descriptors for receiving log data and output path creation commands. */ @@ -197,8 +200,8 @@ struct Child set<int> fds; bool respectTimeouts; bool inBuildSlot; - time_t lastOutput; /* time we last got output on stdout/stderr */ - time_t timeStarted; + steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + steady_time_point timeStarted; }; @@ -238,7 +241,7 @@ private: WeakGoals waitingForAWhile; /* Last time the goals in `waitingForAWhile' where woken up. */ - time_t lastWokenUp; + steady_time_point lastWokenUp; /* Cache for pathContentsGood(). */ std::map<Path, bool> pathContentsGoodCache; @@ -254,7 +257,7 @@ public: LocalStore & store; - std::shared_ptr<HookInstance> hook; + std::unique_ptr<HookInstance> hook; Worker(LocalStore & store); ~Worker(); @@ -643,7 +646,7 @@ HookInstance::~HookInstance() { try { toHook.writeSide = -1; - pid.kill(true); + if (pid != -1) pid.kill(true); } catch (...) { ignoreException(); } @@ -748,7 +751,7 @@ private: Pipe userNamespaceSync; /* The build hook. */ - std::shared_ptr<HookInstance> hook; + std::unique_ptr<HookInstance> hook; /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -957,7 +960,7 @@ void DerivationGoal::killChild() child. */ ::kill(-pid, SIGKILL); /* ignore the result */ buildUser.kill(); - pid.wait(true); + pid.wait(); } else pid.kill(); @@ -1249,8 +1252,7 @@ void DerivationGoal::inputsRealised() } /* Second, the input sources. */ - for (auto & i : drv->inputSrcs) - worker.store.computeFSClosure(i, inputPaths); + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1270,6 +1272,8 @@ void DerivationGoal::inputsRealised() build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + + result = BuildResult(); } @@ -1343,6 +1347,7 @@ void DerivationGoal::tryToBuild() case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; return; case rpPostpone: @@ -1411,14 +1416,15 @@ void DerivationGoal::buildDone() /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have - simply have closed its end of the pipe --- just don't do that - :-) */ - /* !!! this could block! security problem! solution: kill the - child */ - int status = hook ? hook->pid.wait(true) : pid.wait(true); + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill(true) : pid.kill(true); debug(format("builder process for ‘%1%’ finished") % drvPath); + result.timesBuilt++; + result.stopTime = time(0); + /* So the child is gone now. */ worker.childTerminated(this); @@ -1558,7 +1564,7 @@ HookReply DerivationGoal::tryBuildHook() if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; if (!worker.hook) - worker.hook = std::make_shared<HookInstance>(); + worker.hook = std::make_unique<HookInstance>(); /* Tell the hook about system features (beyond the system type) required from the build machine. (The hook could parse the @@ -1593,8 +1599,7 @@ HookReply DerivationGoal::tryBuildHook() printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - hook = worker.hook; - worker.hook.reset(); + hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the remote system. This unfortunately has to contain the entire @@ -2102,7 +2107,11 @@ void DerivationGoal::startBuilder() /* Create a pipe to get the output of the builder. */ builderOut.create(); + result.startTime = time(0); + /* Fork a child to build the package. */ + ProcessOptions options; + #if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2144,7 +2153,6 @@ void DerivationGoal::startBuilder() userNamespaceSync.create(); - ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { @@ -2155,7 +2163,8 @@ void DerivationGoal::startBuilder() namespace, we can't drop additional groups; they will be mapped to nogroup in the child namespace. There does not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?)*/ + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); @@ -2179,7 +2188,7 @@ void DerivationGoal::startBuilder() _exit(0); }, options); - if (helper.wait(true) != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2210,7 +2219,6 @@ void DerivationGoal::startBuilder() } else #endif { - ProcessOptions options; options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); @@ -2284,12 +2292,8 @@ void DerivationGoal::runChild() outside of the namespace. Making a subtree private is local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ - Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); - for (auto & i : mounts) { - vector<string> fields = tokenizeString<vector<string> >(i, " "); - string fs = decodeOctalEscaped(fields.at(4)); - if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) - throw SysError(format("unable to make filesystem ‘%1%’ private") % fs); + if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { + throw SysError("unable to make ‘/’ private mount"); } /* Bind-mount chroot directory to itself, to treat it as a @@ -2329,6 +2333,7 @@ void DerivationGoal::runChild() ss.push_back("/etc/nsswitch.conf"); ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); + ss.push_back("/var/run/nscd/socket"); } for (auto & i : ss) dirsInChroot[i] = i; @@ -2555,15 +2560,18 @@ void DerivationGoal::runChild() */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { - if (i.first != i.second) + if (i.first != i.second.source) throw Error(format( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second); + % i.first % i.second.source); string path = i.first; struct stat st; - if (lstat(path.c_str(), &st)) + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; throw SysError(format("getting attributes of path ‘%1%’") % path); + } if (S_ISDIR(st.st_mode)) sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); else @@ -2674,7 +2682,9 @@ void DerivationGoal::registerOutputs() outputs to allow hard links between outputs. */ InodesSeen inodesSeen; - Path checkSuffix = "-check"; + Path checkSuffix = ".check"; + bool runDiffHook = settings.get("run-diff-hook", false); + bool keepPreviousRound = settings.keepFailed || runDiffHook; /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -2904,30 +2914,42 @@ void DerivationGoal::registerOutputs() assert(prevInfos.size() == infos.size()); for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { + result.isNonDeterministic = true; Path prev = i->path + checkSuffix; - if (pathExists(prev)) - throw NotDeterministic( - format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") - % i->path % drvPath % prev); - else - throw NotDeterministic( - format("output ‘%1%’ of ‘%2%’ differs from previous round") - % i->path % drvPath); + bool prevExists = keepPreviousRound && pathExists(prev); + auto msg = prevExists + ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) + : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); + + auto diffHook = settings.get("diff-hook", std::string("")); + if (prevExists && diffHook != "" && runDiffHook) { + try { + auto diff = runProgram(diffHook, true, {prev, i->path}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } + + if (settings.get("enforce-determinism", true)) + throw NotDeterministic(msg); + + printError(msg); + curRound = nrRounds; // we know enough, bail out early } - abort(); // shouldn't happen } - if (settings.keepFailed) { + /* If this is the first round of several, then move the output out + of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { Path prev = i.second.path + checkSuffix; deletePath(prev); - if (curRound < nrRounds) { - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); - } + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); } - } if (curRound < nrRounds) { @@ -2935,6 +2957,15 @@ void DerivationGoal::registerOutputs() return; } + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + } + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -3045,7 +3076,8 @@ void DerivationGoal::handleEOF(int fd) void DerivationGoal::flushLine() { - if (settings.verboseBuild) + if (settings.verboseBuild && + (settings.printRepeatedBuilds || curRound == 1)) printError(filterANSIEscapes(currentLogLine, true)); else { logTail.push_back(currentLogLine); @@ -3387,7 +3419,7 @@ Worker::Worker(LocalStore & store) if (working) abort(); working = true; nrLocalBuilds = 0; - lastWokenUp = 0; + lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; } @@ -3496,7 +3528,7 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds, child.goal = goal; child.goal2 = goal.get(); child.fds = fds; - child.timeStarted = child.lastOutput = time(0); + child.timeStarted = child.lastOutput = steady_time_point::clock::now(); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; children.emplace_back(child); @@ -3615,35 +3647,38 @@ void Worker::waitForInput() bool useTimeout = false; struct timeval timeout; timeout.tv_usec = 0; - time_t before = time(0); + auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there is a build timeout, then wait for input until the first deadline for any child. */ - assert(sizeof(time_t) >= sizeof(long)); - time_t nearest = LONG_MAX; // nearest deadline + auto nearest = steady_time_point::max(); // nearest deadline for (auto & i : children) { if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } - if (nearest != LONG_MAX) { - timeout.tv_sec = std::max((time_t) 1, nearest - before); + if (nearest != steady_time_point::max()) { + timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); useTimeout = true; - printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); } /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == 0) + if (lastWokenUp == steady_time_point::min()) printError("waiting for locks or build slots..."); - if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); - } else lastWokenUp = 0; + if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; + timeout.tv_sec = std::max(1L, + (long) std::chrono::duration_cast<std::chrono::seconds>( + lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); + } else lastWokenUp = steady_time_point::min(); + + if (useTimeout) + vomit("sleeping %d seconds", timeout.tv_sec); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) @@ -3663,9 +3698,10 @@ void Worker::waitForInput() throw SysError("waiting for input"); } - time_t after = time(0); + auto after = steady_time_point::clock::now(); - /* Process all available file descriptors. */ + /* Process all available file descriptors. FIXME: this is + O(children * fds). */ decltype(children)::iterator i; for (auto j = children.begin(); j != children.end(); j = i) { i = std::next(j); @@ -3701,7 +3737,7 @@ void Worker::waitForInput() if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && j->respectTimeouts && - after - j->lastOutput >= (time_t) settings.maxSilentTime) + after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { printError( format("%1% timed out after %2% seconds of silence") @@ -3712,7 +3748,7 @@ void Worker::waitForInput() else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && j->respectTimeouts && - after - j->timeStarted >= (time_t) settings.buildTimeout) + after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { printError( format("%1% timed out after %2% seconds") @@ -3721,7 +3757,7 @@ void Worker::waitForInput() } } - if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { GoalPtr goal = i.lock(); @@ -3783,12 +3819,13 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) worker.run(goals); PathSet failed; - for (auto & i : goals) - if (i->getExitCode() == Goal::ecFailed) { + for (auto & i : goals) { + if (i->getExitCode() != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } + } if (!failed.empty()) throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 954044c2344f..42873d9e8a10 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -324,20 +324,30 @@ struct CurlDownloader : public Downloader ~CurlDownloader() { + stopWorkerThread(); + + workerThread.join(); + + if (curlm) curl_multi_cleanup(curlm); + } + + void stopWorkerThread() + { /* Signal the worker thread to exit. */ { auto state(state_.lock()); state->quit = true; } - writeFull(wakeupPipe.writeSide.get(), " "); - - workerThread.join(); - - if (curlm) curl_multi_cleanup(curlm); + writeFull(wakeupPipe.writeSide.get(), " ", false); } void workerThreadMain() { + /* Cause this thread to be notified on SIGINT. */ + auto callback = createInterruptCallback([&]() { + stopWorkerThread(); + }); + std::map<CURL *, std::shared_ptr<DownloadItem>> items; bool quit = false; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ae03604faf98..8e90913cc3f1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -379,7 +379,7 @@ void LocalStore::findRuntimeRoots(PathSet & roots) auto digitsRegex = std::regex(R"(^\d+$)"); auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); - while (errno = 0, ent = readdir(procDir)) { + while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); @@ -393,14 +393,14 @@ void LocalStore::findRuntimeRoots(PathSet & roots) throw SysError(format("opening %1%") % fdStr); } struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir)) { + while (errno = 0, fd_ent = readdir(fdDir.get())) { if (fd_ent->d_name[0] != '.') { readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); } } if (errno) throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); - fdDir.close(); + fdDir.reset(); auto mapLines = tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); @@ -621,6 +621,11 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) /* Don't delete .chroot directories for derivations that are currently being built. */ if (isActiveTempFile(state, path, ".chroot")) return; + + /* Don't delete .check directories for derivations that are + currently being built, because we may need to run + diff-hook. */ + if (isActiveTempFile(state, path, ".check")) return; } PathSet visited; @@ -646,13 +651,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) the link count. */ void LocalStore::removeUnusedLinks(const GCState & state) { - AutoCloseDir dir = opendir(linksDir.c_str()); + AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir); long long actualSize = 0, unsharedSize = 0; struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { + while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -771,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) try { - AutoCloseDir dir = opendir(realStoreDir.c_str()); + AutoCloseDir dir(opendir(realStoreDir.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir); /* Read the store and immediately delete all paths that @@ -782,7 +787,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) can start faster. */ Paths entries; struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { + while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -793,7 +798,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) tryToDelete(state, path); } - dir.close(); + dir.reset(); /* Now delete the unreachable valid paths. Randomise the order in which we delete entries to make the collector diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3194193bc842..a423b4e5c0f4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -149,6 +149,11 @@ struct Settings { before being killed (0 means no limit). */ unsigned long maxLogSize; + /* When build-repeat > 0 and verboseBuild == true, whether to + print repeated builds (i.e. builds other than the first one) to + stderr. Hack to prevent Hydra logs from being polluted. */ + bool printRepeatedBuilds = true; + /* How often (in seconds) to poll for locks. */ unsigned int pollInterval; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0c2c49e5531f..9a88cdc317b6 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -8,7 +8,7 @@ namespace nix { -void Store::computeFSClosure(const Path & startPath, +void Store::computeFSClosure(const PathSet & startPaths, PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { struct State @@ -85,7 +85,8 @@ void Store::computeFSClosure(const Path & startPath, }); }; - enqueue(startPath); + for (auto & startPath : startPaths) + enqueue(startPath); { auto state(state_.lock()); @@ -95,6 +96,13 @@ void Store::computeFSClosure(const Path & startPath, } +void Store::computeFSClosure(const Path & startPath, + PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +{ + computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); +} + + void Store::queryMissing(const PathSet & targets, PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ded19c05d2cd..4cb5de7449ea 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -52,8 +52,9 @@ struct NarIndexer : ParseSink, StringSource void preallocateContents(unsigned long long size) override { currentStart = string(s, pos, 16); + assert(size <= std::numeric_limits<size_t>::max()); members.emplace(currentPath, - NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); + NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size}); } void receiveContents(unsigned char * data, unsigned int len) override diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 1bf8b7d83bbc..b71c7e905ff1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -5,6 +5,7 @@ #include "globals.hh" #include <cstdlib> +#include <cstring> #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> @@ -46,11 +47,11 @@ LocalStore::InodeHash LocalStore::loadInodeHash() debug("loading hash inodes in memory"); InodeHash inodeHash; - AutoCloseDir dir = opendir(linksDir.c_str()); + AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir); struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ checkInterrupt(); // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); @@ -67,11 +68,11 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa { Strings names; - AutoCloseDir dir = opendir(path.c_str()); + AutoCloseDir dir(opendir(path.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % path); struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { @@ -148,10 +149,24 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa inodeHash.insert(st.st_ino); return; } - if (errno != EEXIST) - throw SysError(format("cannot link ‘%1%’ to ‘%2%’") % linkPath % path); - /* Fall through if another process created ‘linkPath’ before - we did. */ + + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link ‘%s’ to ‘%s’: %s", linkPath, path, strerror(errno)); + return; + + default: + throw SysError("cannot link ‘%1%’ to ‘%2%’", linkPath, path); + } } /* Yes! We've seen a file with the same contents. Replace the @@ -195,7 +210,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa printInfo(format("‘%1%’ has maximum number of links") % linkPath); return; } - throw SysError(format("cannot link ‘%1%’ to ‘%2%’") % tempLink % linkPath); + throw SysError("cannot link ‘%1%’ to ‘%2%’", tempLink, linkPath); } /* Atomically replace the old file with the new hard link. */ diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 8fc862073030..620c9a6b752d 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,5 +1,6 @@ #include "pathlocks.hh" #include "util.hh" +#include "sync.hh" #include <cerrno> #include <cstdlib> @@ -74,7 +75,7 @@ bool lockFile(int fd, LockType lockType, bool wait) close a descriptor, the previous lock will be closed as well. And there is no way to query whether we already have a lock (F_GETLK only works on locks held by other processes). */ -static StringSet lockedPaths; /* !!! not thread-safe */ +static Sync<StringSet> lockedPaths_; PathLocks::PathLocks() @@ -110,49 +111,60 @@ bool PathLocks::lockPaths(const PathSet & _paths, debug(format("locking path ‘%1%’") % path); - if (lockedPaths.find(lockPath) != lockedPaths.end()) - throw Error("deadlock: trying to re-acquire self-held lock"); + { + auto lockedPaths(lockedPaths_.lock()); + if (lockedPaths->count(lockPath)) + throw Error("deadlock: trying to re-acquire self-held lock ‘%s’", lockPath); + lockedPaths->insert(lockPath); + } + + try { - AutoCloseFD fd; + AutoCloseFD fd; - while (1) { + while (1) { - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } } + + debug(format("lock acquired on ‘%1%’") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file ‘%1%’") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file ‘%1%’ has become stale") % lockPath); + else + break; } - debug(format("lock acquired on ‘%1%’") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file ‘%1%’") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file ‘%1%’ has become stale") % lockPath); - else - break; + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); + + } catch (...) { + lockedPaths_.lock()->erase(lockPath); + throw; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - lockedPaths.insert(lockPath); } return true; @@ -174,7 +186,8 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths.erase(i.second); + lockedPaths_.lock()->erase(i.second); + if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on ‘%1%’") % i.second); @@ -195,7 +208,7 @@ void PathLocks::setDeletion(bool deletePaths) bool pathIsLockedByMe(const Path & path) { Path lockPath = path + ".lock"; - return lockedPaths.find(lockPath) != lockedPaths.end(); + return lockedPaths_.lock()->count(lockPath); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 77faa2f801f1..816d95ba6075 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -599,9 +599,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else if (msg == STDERR_READ) { if (!source) throw Error("no source"); size_t len = readInt(from); - unsigned char * buf = new unsigned char[len]; - AutoDeleteArray<unsigned char> d(buf); - writeString(buf, source->read(buf, len), to); + auto buf = std::make_unique<unsigned char[]>(len); + writeString(buf.get(), source->read(buf.get(), len), to); to.flush(); } else diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index c11f2b06b990..ccb71f1eefe5 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,23 +1,34 @@ #include "config.h" #if ENABLE_S3 +#if __linux__ #include "s3-binary-cache-store.hh" #include "nar-info.hh" #include "nar-info-disk-cache.hh" #include "globals.hh" +#include <aws/core/Aws.h> #include <aws/core/client/ClientConfiguration.h> #include <aws/s3/S3Client.h> #include <aws/s3/model/CreateBucketRequest.h> #include <aws/s3/model/GetBucketLocationRequest.h> #include <aws/s3/model/GetObjectRequest.h> #include <aws/s3/model/HeadObjectRequest.h> -#include <aws/s3/model/PutObjectRequest.h> #include <aws/s3/model/ListObjectsRequest.h> +#include <aws/s3/model/PutObjectRequest.h> namespace nix { +struct istringstream_nocopy : public std::stringstream +{ + istringstream_nocopy(const std::string & s) + { + rdbuf()->pubsetbuf( + (char *) s.data(), s.size()); + } +}; + struct S3Error : public Error { Aws::S3::S3Errors err; @@ -37,6 +48,20 @@ R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) return outcome.GetResultWithOwnership(); } +static void initAWS() +{ + static std::once_flag flag; + std::call_once(flag, []() { + Aws::SDKOptions options; + + /* We install our own OpenSSL locking function (see + shared.cc), so don't let aws-sdk-cpp override it. */ + options.cryptoOptions.initAndCleanupOpenSSL = false; + + Aws::InitAPI(options); + }); +} + struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { std::string bucketName; @@ -63,6 +88,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore ref<Aws::Client::ClientConfiguration> makeConfig() { + initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); res->region = Aws::Region::US_EAST_1; // FIXME: make configurable res->requestTimeoutMs = 600 * 1000; @@ -145,7 +171,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore .WithBucket(bucketName) .WithKey(path); - auto stream = std::make_shared<std::stringstream>(data); + auto stream = std::make_shared<istringstream_nocopy>(data); request.SetBody(stream); @@ -260,3 +286,4 @@ static RegisterStoreImplementation regStore([]( } #endif +#endif diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 32523dc785aa..ec3bf5a6fd83 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -208,7 +208,20 @@ struct BuildResult NotDeterministic, } status = MiscFailure; std::string errorMsg; - //time_t startTime = 0, stopTime = 0; + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + bool success() { return status == Built || status == Substituted || status == AlreadyValid; } @@ -477,15 +490,19 @@ public: ensurePath(). */ Derivation derivationFromPath(const Path & drvPath); - /* Place in `paths' the set of all store paths in the file system + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `paths' is not cleared. If + or indirectly reached from it. `out' is not cleared. If `flipDirection' is true, the set of paths that can reach `storePath' is returned; that is, the closures under the `referrers' relation instead of the `references' relation is returned. */ + void computeFSClosure(const PathSet & paths, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + void computeFSClosure(const Path & path, - PathSet & paths, bool flipDirection = false, + PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); /* Given a set of paths that are to be built, return the set of diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 81aced0fde16..aa50fceb9e3e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -165,7 +165,13 @@ Hash parseHash32(HashType ht, const string & s) unsigned int i = b / 8; unsigned int j = b % 8; hash.hash[i] |= digit << j; - if (i < hash.hashSize - 1) hash.hash[i + 1] |= digit >> (8 - j); + + if (i < hash.hashSize - 1) { + hash.hash[i + 1] |= digit >> (8 - j); + } else { + if (digit >> (8 - j)) + throw BadHash(format("invalid base-32 hash ‘%1%’") % s); + } } return hash; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index ba99a81c3826..3e6c4b54853c 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -79,6 +79,7 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define printError(args...) printMsg(lvlError, args) #define printInfo(args...) printMsg(lvlInfo, args) #define debug(args...) printMsg(lvlDebug, args) +#define vomit(args...) printMsg(lvlVomit, args) void warnOnce(bool & haveWarned, const FormatOrString & fs); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 24c6d107359e..a68f7a0fa8ee 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -3,6 +3,7 @@ #include <cstring> #include <cerrno> +#include <memory> namespace nix { @@ -236,11 +237,10 @@ size_t readString(unsigned char * buf, size_t max, Source & source) string readString(Source & source) { size_t len = readInt(source); - unsigned char * buf = new unsigned char[len]; - AutoDeleteArray<unsigned char> d(buf); - source(buf, len); + auto buf = std::make_unique<unsigned char[]>(len); + source(buf.get(), len); readPadding(len, source); - return string((char *) buf, len); + return string((char *) buf.get(), len); } Source & operator >> (Source & in, string & s) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ce16cc30a5c7..e9457582810a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -2,14 +2,16 @@ #include "util.hh" #include "affinity.hh" +#include "sync.hh" -#include <iostream> +#include <cctype> #include <cerrno> #include <cstdio> #include <cstdlib> -#include <sstream> #include <cstring> -#include <cctype> +#include <iostream> +#include <sstream> +#include <thread> #include <sys/wait.h> #include <unistd.h> @@ -234,11 +236,11 @@ DirEntries readDirectory(const Path & path) DirEntries entries; entries.reserve(64); - AutoCloseDir dir = opendir(path.c_str()); + AutoCloseDir dir(opendir(path.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % path); struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -272,11 +274,10 @@ string readFile(int fd) if (fstat(fd, &st) == -1) throw SysError("statting file"); - unsigned char * buf = new unsigned char[st.st_size]; - AutoDeleteArray<unsigned char> d(buf); - readFull(fd, buf, st.st_size); + auto buf = std::make_unique<unsigned char[]>(st.st_size); + readFull(fd, buf.get(), st.st_size); - return string((char *) buf, st.st_size); + return string((char *) buf.get(), st.st_size); } @@ -646,69 +647,26 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// -AutoCloseDir::AutoCloseDir() -{ - dir = 0; -} - - -AutoCloseDir::AutoCloseDir(DIR * dir) -{ - this->dir = dir; -} - - -AutoCloseDir::~AutoCloseDir() -{ - close(); -} - - -void AutoCloseDir::operator =(DIR * dir) -{ - this->dir = dir; -} - - -AutoCloseDir::operator DIR *() -{ - return dir; -} - - -void AutoCloseDir::close() -{ - if (dir) { - closedir(dir); - dir = 0; - } -} - - -////////////////////////////////////////////////////////////////////// - - Pid::Pid() - : pid(-1), separatePG(false), killSignal(SIGKILL) { } Pid::Pid(pid_t pid) - : pid(pid), separatePG(false), killSignal(SIGKILL) + : pid(pid) { } Pid::~Pid() { - kill(); + if (pid != -1) kill(); } void Pid::operator =(pid_t pid) { - if (this->pid != pid) kill(); + if (this->pid != -1 && this->pid != pid) kill(); this->pid = pid; killSignal = SIGKILL; // reset signal to default } @@ -720,9 +678,9 @@ Pid::operator pid_t() } -void Pid::kill(bool quiet) +int Pid::kill(bool quiet) { - if (pid == -1 || pid == 0) return; + assert(pid != -1); if (!quiet) printError(format("killing process %1%") % pid); @@ -733,32 +691,20 @@ void Pid::kill(bool quiet) if (::kill(separatePG ? -pid : pid, killSignal) != 0) printError((SysError(format("killing process %1%") % pid).msg())); - /* Wait until the child dies, disregarding the exit status. */ - int status; - while (waitpid(pid, &status, 0) == -1) { - checkInterrupt(); - if (errno != EINTR) { - printError( - (SysError(format("waiting for process %1%") % pid).msg())); - break; - } - } - - pid = -1; + return wait(); } -int Pid::wait(bool block) +int Pid::wait() { assert(pid != -1); while (1) { int status; - int res = waitpid(pid, &status, block ? 0 : WNOHANG); + int res = waitpid(pid, &status, 0); if (res == pid) { pid = -1; return status; } - if (res == 0 && !block) return -1; if (errno != EINTR) throw SysError("cannot get child exit status"); checkInterrupt(); @@ -823,7 +769,7 @@ void killUser(uid_t uid) _exit(0); }, options); - int status = pid.wait(true); + int status = pid.wait(); if (status != 0) throw Error(format("cannot kill processes for uid ‘%1%’: %2%") % uid % statusToString(status)); @@ -934,7 +880,7 @@ string runProgram(Path program, bool searchPath, const Strings & args, string result = drainFD(out.readSide.get()); /* Wait for the child to finish. */ - int status = pid.wait(true); + int status = pid.wait(); if (!statusOk(status)) throw ExecError(status, format("program ‘%1%’ %2%") % program % statusToString(status)); @@ -976,7 +922,7 @@ void restoreSIGPIPE() ////////////////////////////////////////////////////////////////////// -volatile sig_atomic_t _isInterrupted = 0; +bool _isInterrupted = false; thread_local bool interruptThrown = false; @@ -1243,4 +1189,64 @@ void callFailure(const std::function<void(std::exception_ptr exc)> & failure, st } +static Sync<std::list<std::function<void()>>> _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) { + _isInterrupted = 1; + + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto & callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } + } + } +} + +void startSignalHandlerThread() +{ + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + std::list<std::function<void()>>::iterator it; + ~InterruptCallbackImpl() override + { + _interruptCallbacks.lock()->erase(it); + } +}; + +std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->push_back(callback); + + auto res = std::make_unique<InterruptCallbackImpl>(); + res->it = interruptCallbacks->end(); + res->it--; + + return res; +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 2e48034ae86c..b68d48582b34 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -139,18 +139,6 @@ string drainFD(int fd); /* Automatic cleanup of resources. */ -template <class T> -struct AutoDeleteArray -{ - T * p; - AutoDeleteArray(T * p) : p(p) { } - ~AutoDeleteArray() - { - delete [] p; - } -}; - - class AutoDelete { Path path; @@ -192,32 +180,30 @@ public: }; -class AutoCloseDir +struct DIRDeleter { - DIR * dir; -public: - AutoCloseDir(); - AutoCloseDir(DIR * dir); - ~AutoCloseDir(); - void operator =(DIR * dir); - operator DIR *(); - void close(); + void operator()(DIR * dir) const { + closedir(dir); + } }; +typedef std::unique_ptr<DIR, DIRDeleter> AutoCloseDir; + class Pid { - pid_t pid; - bool separatePG; - int killSignal; + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; public: Pid(); Pid(pid_t pid); ~Pid(); void operator =(pid_t pid); operator pid_t(); - void kill(bool quiet = false); - int wait(bool block); + int kill(bool quiet = false); + int wait(); + void setSeparatePG(bool separatePG); void setKillSignal(int signal); pid_t release(); @@ -233,11 +219,10 @@ void killUser(uid_t uid); pid to the caller. */ struct ProcessOptions { - string errorPrefix; - bool dieWithParent; - bool runExitHandlers; - bool allowVfork; - ProcessOptions() : errorPrefix("error: "), dieWithParent(true), runExitHandlers(false), allowVfork(true) { }; + string errorPrefix = "error: "; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = true; }; pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); @@ -278,7 +263,7 @@ void restoreSIGPIPE(); /* User interruption. */ -extern volatile sig_atomic_t _isInterrupted; +extern bool _isInterrupted; extern thread_local bool interruptThrown; @@ -434,4 +419,19 @@ void callSuccess( } +/* Start a thread that handles various signals. Also block those signals + on the current thread (and thus any threads created by it). */ +void startSignalHandlerThread(); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/* Register a function that gets called on SIGINT (in a non-signal + context). */ +std::unique_ptr<InterruptCallback> createInterruptCallback( + std::function<void()> callback); + + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 08c6793577a4..3eb2d2c0b7a9 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -81,7 +81,8 @@ int main(int argc, char ** argv) auto pure = false; auto fromArgs = false; auto packages = false; - auto interactive = true; + // Same condition as bash uses for interactive shells + auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings instArgs; Strings buildArgs; @@ -105,6 +106,7 @@ int main(int argc, char ** argv) std::vector<string> args; for (int i = 1; i < argc; ++i) args.push_back(argv[i]); + // Heuristic to see if we're invoked as a shebang script, namely, if we // have a single argument, it's the name of an executable file, and it // starts with "#!". @@ -115,9 +117,9 @@ int main(int argc, char ** argv) if (std::regex_search(lines.front(), std::regex("^#!"))) { lines.pop_front(); inShebang = true; - for (int i = 2; i < argc - 1; ++i) + for (int i = 2; i < argc; ++i) savedArgs.push_back(argv[i]); - std::vector<string> args; + args.clear(); for (auto line : lines) { line = chomp(line); std::smatch match; @@ -134,15 +136,11 @@ int main(int argc, char ** argv) if (arg == "--help") { deletePath(tmpDir); - tmpDir.cancel(); - execlp("man", "man", myName, NULL); - throw SysError("executing man"); + showManPage(myName); } - else if (arg == "--version") { - std::cout << myName << " (Nix) " << nixVersion << '\n'; - return; - } + else if (arg == "--version") + printVersion(myName); else if (arg == "--add-drv-link") { drvLink = "./derivation"; @@ -276,6 +274,7 @@ int main(int argc, char ** argv) if (n >= args.size()) { throw UsageError(format("%1% requires an argument") % arg); } + interactive = false; auto interpreter = args[n]; auto execArgs = ""; @@ -287,9 +286,8 @@ int main(int argc, char ** argv) // executes it unless it contains the string "perl" or "indir", // or (undocumented) argv[0] does not contain "perl". Exploit // the latter by doing "exec -a". - if (std::regex_search(interpreter, std::regex("perl"))) { - execArgs = "-a PERL"; - } + if (std::regex_search(interpreter, std::regex("perl"))) + execArgs = "-a PERL"; std::ostringstream joined; for (const auto & i : savedArgs) @@ -300,7 +298,6 @@ int main(int argc, char ** argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\") -- %4%") % execArgs % interpreter % script % joined.str()).str(); } else { envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); @@ -420,7 +417,7 @@ int main(int argc, char ** argv) // environment variables and shell functions. Also don't lose // the current $PATH directories. auto rcfile = (Path) tmpDir + "/rc"; - writeFile(rcfile, (format( + writeFile(rcfile, fmt( "rm -rf '%1%'; " "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " "%2%" @@ -434,13 +431,12 @@ int main(int argc, char ** argv) "unset NIX_INDENT_MAKE; " "shopt -u nullglob; " "unset TZ; %4%" - "%5%" - ) - % (Path) tmpDir - % (pure ? "" : "p=$PATH; ") - % (pure ? "" : "PATH=$PATH:$p; unset p; ") - % (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : "") - % envCommand).str()); + "%5%", + (Path) tmpDir, + (pure ? "" : "p=$PATH; "), + (pure ? "" : "PATH=$PATH:$p; unset p; "), + (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), + envCommand)); Strings envStrs; for (auto & i : env) @@ -450,10 +446,13 @@ int main(int argc, char ** argv) ? Strings{"bash", "--rcfile", rcfile} : Strings{"bash", rcfile}; - environ = stringsToCharPtrs(envStrs).data(); + auto envPtrs = stringsToCharPtrs(envStrs); + + environ = envPtrs.data(); + + auto argPtrs = stringsToCharPtrs(args); - execvp(getEnv("NIX_BUILD_SHELL", "bash").c_str(), - stringsToCharPtrs(args).data()); + execvp(getEnv("NIX_BUILD_SHELL", "bash").c_str(), argPtrs.data()); throw SysError("executing shell"); } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 5b4c2181996c..361627823126 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -76,10 +76,10 @@ static void update(const StringSet & channelNames) // Download each channel. auto exprs = Strings{}; for (const auto & channel : channels) { - if (!channelNames.empty() && channelNames.find(channel.first) != channelNames.end()) - continue; auto name = channel.first; auto url = channel.second; + if (!(channelNames.empty() || channelNames.count(name))) + continue; // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index c8fa81df13c8..90a7301873c4 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -32,14 +32,14 @@ using namespace nix; #ifndef __linux__ #define SPLICE_F_MOVE 0 -static ssize_t splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags) +static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) { /* We ignore most parameters, we just have them for conformance with the linux syscall */ char buf[8192]; auto read_count = read(fd_in, buf, sizeof(buf)); if (read_count == -1) return read_count; - auto write_count = decltype<read_count>(0); + auto write_count = decltype(read_count)(0); while (write_count < read_count) { auto res = write(fd_out, buf + write_count, read_count - write_count); if (res == -1) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a8cb46319abc..c1e6afef0e50 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -424,10 +424,9 @@ static void opQuery(Strings opFlags, Strings opArgs) case qRoots: { PathSet referrers; for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - for (auto & j : paths) - store->computeFSClosure(j, referrers, true, - settings.gcKeepOutputs, settings.gcKeepDerivations); + store->computeFSClosure( + maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), + referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); for (auto & i : roots) @@ -841,6 +840,12 @@ static void opServe(Strings opFlags, Strings opArgs) settings.buildTimeout = readInt(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.maxLogSize = readInt(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { + settings.set("build-repeat", std::to_string(readInt(in))); + settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); + settings.set("run-diff-hook", "true"); + } + settings.printRepeatedBuilds = false; }; while (true) { @@ -956,15 +961,17 @@ static void opServe(Strings opFlags, Strings opArgs) out << status.status << status.errorMsg; + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) + out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; + break; } case cmdQueryClosure: { bool includeOutputs = readInt(in); - PathSet paths = readStorePaths<PathSet>(*store, in); PathSet closure; - for (auto & i : paths) - store->computeFSClosure(i, closure, false, includeOutputs); + store->computeFSClosure(readStorePaths<PathSet>(*store, in), + closure, false, includeOutputs); out << closure; break; } diff --git a/src/nix-store/serve-protocol.hh b/src/nix-store/serve-protocol.hh index c4e2a370300b..f8cc9a4b6ebe 100644 --- a/src/nix-store/serve-protocol.hh +++ b/src/nix-store/serve-protocol.hh @@ -5,7 +5,7 @@ namespace nix { #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x202 +#define SERVE_PROTOCOL_VERSION 0x203 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix/command.cc b/src/nix/command.cc index fdf6ae6affaf..5a8288da912f 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -106,8 +106,8 @@ void StorePathsCommand::run(ref<Store> store) if (recursive) { PathSet closure; - for (auto & storePath : storePaths) - store->computeFSClosure(storePath, closure, false, false); + store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), + closure, false, false); storePaths = Paths(closure.begin(), closure.end()); } } diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/lang/eval-fail-path-slash.nix new file mode 100644 index 000000000000..530105b3210b --- /dev/null +++ b/tests/lang/eval-fail-path-slash.nix @@ -0,0 +1,6 @@ +# Trailing slashes in paths are not allowed. +# This restriction could be lifted sometime, +# for example if we make '/' a path concatenation operator. +# See https://github.com/NixOS/nix/issues/1138 +# and http://lists.science.uu.nl/pipermail/nix-dev/2016-June/020829.html +/nix/store/ diff --git a/tests/lang/eval-okay-comments.exp b/tests/lang/eval-okay-comments.exp new file mode 100644 index 000000000000..7182dc2f9b8e --- /dev/null +++ b/tests/lang/eval-okay-comments.exp @@ -0,0 +1 @@ +"abcdefghijklmnopqrstuvwxyz" diff --git a/tests/lang/eval-okay-comments.nix b/tests/lang/eval-okay-comments.nix new file mode 100644 index 000000000000..cb2cce218029 --- /dev/null +++ b/tests/lang/eval-okay-comments.nix @@ -0,0 +1,59 @@ +# A simple comment +"a"+ # And another +## A double comment +"b"+ ## And another +# Nested # comments # +"c"+ # and # some # other # +# An empty line, following here: + +"d"+ # and a comment not starting the line ! + +"e"+ +/* multiline comments */ +"f" + +/* multiline + comments, + on + multiple + lines +*/ +"g" + +# Small, tricky comments +/**/ "h"+ /*/*/ "i"+ /***/ "j"+ /* /*/ "k"+ /*/* /*/ "l"+ +# Comments with an even number of ending '*' used to fail: +"m"+ +/* */ /* **/ /* ***/ /* ****/ "n"+ +/* */ /** */ /*** */ /**** */ "o"+ +/** **/ /*** ***/ /**** ****/ "p"+ +/* * ** *** **** ***** */ "q"+ +# Random comments +/* ***** ////// * / * / /* */ "r"+ +# Mixed comments +/* # */ +"s"+ +# /* # +"t"+ +# /* # */ +"u"+ +# /*********/ +"v"+ +## */* +"w"+ +/* + * Multiline, decorated comments + * # This ain't a nest'd comm'nt + */ +"x"+ +''${/** with **/"y" + # real + /* comments + inside ! # */ + + # (and empty lines) + +}''+ /* And a multiline comment, + on the same line, + after some spaces +*/ # followed by a one-line comment +"z" +/* EOF */ diff --git a/tests/local.mk b/tests/local.mk index 2ca52144baee..b3ce39cda806 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -11,7 +11,7 @@ nix_tests = \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ - placeholders.sh + placeholders.sh nix-shell.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh new file mode 100644 index 000000000000..26cc521bbcbf --- /dev/null +++ b/tests/nix-shell.sh @@ -0,0 +1,21 @@ +source common.sh + +clearStore + +# Test nix-shell -A +export IMPURE_VAR=foo +output=$(nix-shell --pure shell.nix -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') + +[ "$output" = " - foo - bar" ] + +# Test nix-shell -p +output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') +[ "$output" = "foo bar" ] + +# Test nix-shell shebang mode +sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh +chmod a+rx $TEST_ROOT/shell.shebang.sh + +output=$($TEST_ROOT/shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] diff --git a/tests/shell.nix b/tests/shell.nix new file mode 100644 index 000000000000..ed4d6fbaaa0b --- /dev/null +++ b/tests/shell.nix @@ -0,0 +1,46 @@ +{ }: + +with import ./config.nix; + +rec { + setupSh = builtins.toFile "setup" '' + export VAR_FROM_STDENV_SETUP=foo + for pkg in $buildInputs; do + export PATH=$PATH:$pkg/bin + done + ''; + + stdenv = mkDerivation { + name = "stdenv"; + buildCommand = '' + mkdir -p $out + ln -s ${setupSh} $out/setup + ''; + }; + + shellDrv = mkDerivation { + name = "shellDrv"; + builder = "/does/not/exist"; + VAR_FROM_NIX = "bar"; + inherit stdenv; + }; + + # Used by nix-shell -p + runCommand = name: args: buildCommand: mkDerivation (args // { + inherit name buildCommand stdenv; + }); + + foo = runCommand "foo" {} '' + mkdir -p $out/bin + echo 'echo foo' > $out/bin/foo + chmod a+rx $out/bin/foo + ''; + + bar = runCommand "bar" {} '' + mkdir -p $out/bin + echo 'echo bar' > $out/bin/bar + chmod a+rx $out/bin/bar + ''; + + bash = shell; +} diff --git a/tests/shell.shebang.sh b/tests/shell.shebang.sh new file mode 100755 index 000000000000..3dadd591572d --- /dev/null +++ b/tests/shell.shebang.sh @@ -0,0 +1,4 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --option use-binary-caches false +#! nix-shell --pure -i bash -p foo bar +echo "$(foo) $(bar) $@" diff --git a/tests/timeout.builder.sh b/tests/timeout.builder.sh deleted file mode 100644 index 3fbdd57946ad..000000000000 --- a/tests/timeout.builder.sh +++ /dev/null @@ -1,2 +0,0 @@ -echo "‘timeout’ builder entering an infinite loop" -while true ; do echo -n .; done diff --git a/tests/timeout.nix b/tests/timeout.nix index b41368bb38e2..540fba934ff6 100644 --- a/tests/timeout.nix +++ b/tests/timeout.nix @@ -1,6 +1,28 @@ with import ./config.nix; -mkDerivation { - name = "timeout"; - builder = ./timeout.builder.sh; +{ + + infiniteLoop = mkDerivation { + name = "timeout"; + buildCommand = '' + echo "‘timeout’ builder entering an infinite loop" + while true ; do echo -n .; done + ''; + }; + + silent = mkDerivation { + name = "silent"; + buildCommand = '' + sleep 60 + ''; + }; + + closeLog = mkDerivation { + name = "silent"; + buildCommand = '' + exec > /dev/null 2>&1 + sleep 1000000000 + ''; + }; + } diff --git a/tests/timeout.sh b/tests/timeout.sh index 2ebd06b9330c..ce1ae7d674a1 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -3,7 +3,7 @@ source common.sh failed=0 -messages="`nix-build -Q timeout.nix --timeout 2 2>&1 || failed=1`" +messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`" if [ $failed -ne 0 ]; then echo "error: ‘nix-store’ succeeded; should have timed out" exit 1 @@ -15,7 +15,17 @@ if ! echo "$messages" | grep -q "timed out"; then exit 1 fi -if nix-build -Q timeout.nix --option build-max-log-size 100; then +if nix-build -Q timeout.nix -A infiniteLoop --option build-max-log-size 100; then + echo "build should have failed" + exit 1 +fi + +if nix-build timeout.nix -A silent --max-silent-time 2; then + echo "build should have failed" + exit 1 +fi + +if nix-build timeout.nix -A closeLog; then echo "build should have failed" exit 1 fi |