diff options
| -rw-r--r-- | INSTALL | 229 | ||||
| -rw-r--r-- | README.md | 5 | ||||
| -rw-r--r-- | doc/manual/advanced-topics/distributed-builds.xml | 3 | ||||
| -rw-r--r-- | doc/manual/installation/prerequisites-source.xml | 4 | ||||
| -rw-r--r-- | misc/docker/Dockerfile | 3 | ||||
| -rw-r--r-- | nix.spec.in | 23 | ||||
| -rw-r--r-- | scripts/local.mk | 9 | ||||
| -rw-r--r-- | src/libstore/build.cc | 6 | ||||
| -rw-r--r-- | src/libstore/download.cc | 2 | ||||
| -rw-r--r-- | src/libstore/misc.cc | 266 | ||||
| -rw-r--r-- | src/libstore/nar-info.cc | 4 | ||||
| -rw-r--r-- | src/libutil/hash.cc | 1 | ||||
| -rw-r--r-- | src/libutil/thread-pool.cc | 6 | ||||
| -rw-r--r-- | src/libutil/thread-pool.hh | 2 | ||||
| -rw-r--r-- | tests/common.sh.in | 1 |
15 files changed, 201 insertions, 363 deletions
diff --git a/INSTALL b/INSTALL deleted file mode 100644 index a4b34144dcff..000000000000 --- a/INSTALL +++ /dev/null @@ -1,229 +0,0 @@ -Copyright 1994, 1995, 1996, 1999, 2000, 2001, 2002 Free Software -Foundation, Inc. - - This file is free documentation; the Free Software Foundation gives -unlimited permission to copy, distribute and modify it. - -Basic Installation -================== - - These are generic installation instructions. - - The `configure' shell script attempts to guess correct values for -various system-dependent variables used during compilation. It uses -those values to create a `Makefile' in each directory of the package. -It may also create one or more `.h' files containing system-dependent -definitions. Finally, it creates a shell script `config.status' that -you can run in the future to recreate the current configuration, and a -file `config.log' containing compiler output (useful mainly for -debugging `configure'). - - It can also use an optional file (typically called `config.cache' -and enabled with `--cache-file=config.cache' or simply `-C') that saves -the results of its tests to speed up reconfiguring. (Caching is -disabled by default to prevent problems with accidental use of stale -cache files.) - - If you need to do unusual things to compile the package, please try -to figure out how `configure' could check whether to do them, and mail -diffs or instructions to the address given in the `README' so they can -be considered for the next release. If you are using the cache, and at -some point `config.cache' contains results you don't want to keep, you -may remove or edit it. - - The file `configure.ac' (or `configure.in') is used to create -`configure' by a program called `autoconf'. You only need -`configure.ac' if you want to change it or regenerate `configure' using -a newer version of `autoconf'. - -The simplest way to compile this package is: - - 1. `cd' to the directory containing the package's source code and type - `./configure' to configure the package for your system. If you're - using `csh' on an old version of System V, you might need to type - `sh ./configure' instead to prevent `csh' from trying to execute - `configure' itself. - - Running `configure' takes awhile. While running, it prints some - messages telling which features it is checking for. - - 2. Type `make' to compile the package. - - 3. Optionally, type `make check' to run any self-tests that come with - the package. - - 4. Type `make install' to install the programs and any data files and - documentation. - - 5. You can remove the program binaries and object files from the - source code directory by typing `make clean'. To also remove the - files that `configure' created (so you can compile the package for - a different kind of computer), type `make distclean'. There is - also a `make maintainer-clean' target, but that is intended mainly - for the package's developers. If you use it, you may have to get - all sorts of other programs in order to regenerate files that came - with the distribution. - -Compilers and Options -===================== - - Some systems require unusual options for compilation or linking that -the `configure' script does not know about. Run `./configure --help' -for details on some of the pertinent environment variables. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is an example: - - ./configure CC=c89 CFLAGS=-O2 LIBS=-lposix - - *Note Defining Variables::, for more details. - -Compiling For Multiple Architectures -==================================== - - You can compile the package for more than one kind of computer at the -same time, by placing the object files for each architecture in their -own directory. To do this, you must use a version of `make' that -supports the `VPATH' variable, such as GNU `make'. `cd' to the -directory where you want the object files and executables to go and run -the `configure' script. `configure' automatically checks for the -source code in the directory that `configure' is in and in `..'. - - If you have to use a `make' that does not support the `VPATH' -variable, you have to compile the package for one architecture at a -time in the source code directory. After you have installed the -package for one architecture, use `make distclean' before reconfiguring -for another architecture. - -Installation Names -================== - - By default, `make install' will install the package's files in -`/usr/local/bin', `/usr/local/man', etc. You can specify an -installation prefix other than `/usr/local' by giving `configure' the -option `--prefix=PATH'. - - You can specify separate installation prefixes for -architecture-specific files and architecture-independent files. If you -give `configure' the option `--exec-prefix=PATH', the package will use -PATH as the prefix for installing programs and libraries. -Documentation and other data files will still use the regular prefix. - - In addition, if you use an unusual directory layout you can give -options like `--bindir=PATH' to specify different values for particular -kinds of files. Run `configure --help' for a list of the directories -you can set and what kinds of files go in them. - - If the package supports it, you can cause programs to be installed -with an extra prefix or suffix on their names by giving `configure' the -option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. - -Optional Features -================= - - Some packages pay attention to `--enable-FEATURE' options to -`configure', where FEATURE indicates an optional part of the package. -They may also pay attention to `--with-PACKAGE' options, where PACKAGE -is something like `gnu-as' or `x' (for the X Window System). The -`README' should mention any `--enable-' and `--with-' options that the -package recognizes. - - For packages that use the X Window System, `configure' can usually -find the X include and library files automatically, but if it doesn't, -you can use the `configure' options `--x-includes=DIR' and -`--x-libraries=DIR' to specify their locations. - -Specifying the System Type -========================== - - There may be some features `configure' cannot figure out -automatically, but needs to determine by the type of machine the package -will run on. Usually, assuming the package is built to be run on the -_same_ architectures, `configure' can figure that out, but if it prints -a message saying it cannot guess the machine type, give it the -`--build=TYPE' option. TYPE can either be a short name for the system -type, such as `sun4', or a canonical name which has the form: - - CPU-COMPANY-SYSTEM - -where SYSTEM can have one of these forms: - - OS KERNEL-OS - - See the file `config.sub' for the possible values of each field. If -`config.sub' isn't included in this package, then this package doesn't -need to know the machine type. - - If you are _building_ compiler tools for cross-compiling, you should -use the `--target=TYPE' option to select the type of system they will -produce code for. - - If you want to _use_ a cross compiler, that generates code for a -platform different from the build platform, you should specify the -"host" platform (i.e., that on which the generated programs will -eventually be run) with `--host=TYPE'. - -Sharing Defaults -================ - - If you want to set default values for `configure' scripts to share, -you can create a site shell script called `config.site' that gives -default values for variables like `CC', `cache_file', and `prefix'. -`configure' looks for `PREFIX/share/config.site' if it exists, then -`PREFIX/etc/config.site' if it exists. Or, you can set the -`CONFIG_SITE' environment variable to the location of the site script. -A warning: not all `configure' scripts look for a site script. - -Defining Variables -================== - - Variables not defined in a site shell script can be set in the -environment passed to `configure'. However, some packages may run -configure again during the build, and the customized values of these -variables may be lost. In order to avoid this problem, you should set -them in the `configure' command line, using `VAR=value'. For example: - - ./configure CC=/usr/local2/bin/gcc - -will cause the specified gcc to be used as the C compiler (unless it is -overridden in the site shell script). - -`configure' Invocation -====================== - - `configure' recognizes the following options to control how it -operates. - -`--help' -`-h' - Print a summary of the options to `configure', and exit. - -`--version' -`-V' - Print the version of Autoconf used to generate the `configure' - script, and exit. - -`--cache-file=FILE' - Enable the cache: use and save the results of the tests in FILE, - traditionally `config.cache'. FILE defaults to `/dev/null' to - disable caching. - -`--config-cache' -`-C' - Alias for `--cache-file=config.cache'. - -`--quiet' -`--silent' -`-q' - Do not print messages saying which checks are being made. To - suppress all normal output, redirect it to `/dev/null' (any error - messages will still be shown). - -`--srcdir=DIR' - Look for the package's source code in directory DIR. Usually - `configure' can determine that directory automatically. - -`configure' also accepts some other, not widely useful, options. Run -`configure --help' for more details. - diff --git a/README.md b/README.md index e77ce302f193..6d1b3243f5dd 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,11 @@ To find out more about the tool, usage and installation instructions, please read the manual, which is available on the Nix website at <http://nixos.org/nix/manual>. +## Contributing + +Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking) +of the manual. It helps you to get started with building Nix from source. + ## License Nix is released under the LGPL v2.1 diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index 70f396f81cdb..1f8d98f5d8fe 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -44,7 +44,8 @@ outputs and perform the remote build. To use it, you should set <envar>NIX_BUILD_HOOK</envar> to <filename><replaceable>prefix</replaceable>/libexec/nix/build-remote.pl</filename>. You should also define a list of available build machines and point -the environment variable <envar>NIX_REMOTE_SYSTEMS</envar> to it. An +the environment variable <envar>NIX_REMOTE_SYSTEMS</envar> to +it. <envar>NIX_REMOTE_SYSTEMS</envar> must be an absolute path. An example configuration is shown in <xref linkend='ex-remote-systems' />. Each line in the file specifies a machine, with the following bits of information: diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index 49036d940bd4..c0065f133700 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -19,6 +19,10 @@ it from <link xlink:href="http://www.freedesktop.org/wiki/Software/pkg-config" />.</para></listitem> + + <listitem><para>The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from <link + xlink:href="https://www.openssl.org"/>.</para></listitem> <listitem><para>The bzip2 compressor program and the <literal>libbz2</literal> library. Thus you must have bzip2 diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index bfa0c34d0947..098633fc3c86 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -17,4 +17,5 @@ ENV \ ENV=/etc/profile \ PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ - SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt + SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ + NIX_PATH=/nix/var/nix/profiles/per-user/root/channels/ diff --git a/nix.spec.in b/nix.spec.in index dff8efbdcbed..61da404a3844 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -1,16 +1,16 @@ %global nixbld_user "nix-builder-" -%global nixbld_group "nix-builders" +%global nixbld_group "nixbld" Summary: The Nix software deployment system Name: nix Version: @PACKAGE_VERSION@ Release: 2%{?dist} License: LGPLv2+ -%if 0%{?rhel} +%if 0%{?rhel} && 0%{?rhel} < 7 Group: Applications/System %endif URL: http://nixos.org/ -Source0: %{name}-%{version}.tar.bz2 +Source0: %{name}-%{version}.tar.xz %if 0%{?el5} BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) %endif @@ -41,7 +41,7 @@ it can be used equally well under other Unix systems. %package devel Summary: Development files for %{name} -%if 0%{?rhel} +%if 0%{?rhel} && 0%{?rhel} < 7 Group: Development/Libraries %endif Requires: %{name}%{?_isa} = %{version}-%{release} @@ -53,7 +53,7 @@ developing applications that use %{name}. %package doc Summary: Documentation files for %{name} -%if 0%{?rhel} +%if 0%{?rhel} && 0%{?rhel} < 7 Group: Documentation %endif BuildArch: noarch @@ -65,7 +65,7 @@ The %{name}-doc package contains documentation files for %{name}. %package -n emacs-%{name} Summary: Nix mode for Emacs -%if 0%{?rhel} +%if 0%{?rhel} && 0%{?rhel} < 7 Group: Applications/Editors %endif BuildArch: noarch @@ -77,7 +77,7 @@ This package provides a major mode for editing Nix expressions. %package -n emacs-%{name}-el Summary: Elisp source files for emacs-%{name} -%if 0%{?rhel} +%if 0%{?rhel} && 0%{?rhel} < 7 Group: Applications/Editors %endif BuildArch: noarch @@ -118,6 +118,10 @@ make DESTDIR=$RPM_BUILD_ROOT install find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' +# make the store +mkdir -p $RPM_BUILD_ROOT/nix/store +chmod 1775 $RPM_BUILD_ROOT/nix/store + # make per-user directories for d in profiles gcroots; do @@ -155,8 +159,7 @@ done %post chgrp %{nixbld_group} /nix/store -chmod 1775 /nix/store -%if ! 0%{?rhel} +%if ! 0%{?rhel} || 0%{?rhel} >= 7 # Enable and start Nix worker systemctl enable nix-daemon.socket nix-daemon.service systemctl start nix-daemon.socket @@ -168,7 +171,7 @@ systemctl start nix-daemon.socket %{perl_vendorarch}/* %exclude %dir %{perl_vendorarch}/auto/ %{_prefix}/libexec/* -%if ! 0%{?rhel} +%if ! 0%{?rhel} || 0%{?rhel} >= 7 %{_prefix}/lib/systemd/system/nix-daemon.socket %{_prefix}/lib/systemd/system/nix-daemon.service %endif diff --git a/scripts/local.mk b/scripts/local.mk index 86b6d08500aa..edaf44cc492d 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -9,18 +9,23 @@ bin-scripts += $(nix_bin_scripts) nix_noinst_scripts := \ $(d)/build-remote.pl \ - $(d)/resolve-system-dependencies.pl \ $(d)/nix-http-export.cgi \ $(d)/nix-profile.sh \ $(d)/nix-reduce-build +ifeq ($(OS), Darwin) + nix_noinst_scripts += $(d)/resolve-system-dependencies.pl +endif + noinst-scripts += $(nix_noinst_scripts) profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) -$(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) +ifeq ($(OS), Darwin) + $(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) +endif $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0a4df8ca7af4..af2c908c30db 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1075,8 +1075,10 @@ void DerivationGoal::outputsSubstituted() { trace("all outputs substituted (maybe)"); - if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) - throw Error(format("some substitutes for the outputs of derivation ‘%1%’ failed (usually happens due to networking issues); try ‘--fallback’ to build derivation from source ") % drvPath); + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { + done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation ‘%1%’ failed (usually happens due to networking issues); try ‘--fallback’ to build derivation from source ") % drvPath).str()); + return; + } /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 6e39330e40d9..04a2b325c651 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -157,7 +157,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); } - data->clear(); + data = make_ref<std::string>(); if (requestHeaders) { curl_slist_free_all(requestHeaders); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 4bb043aaf17e..da654ba0d2c3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "local-store.hh" #include "store-api.hh" +#include "thread-pool.hh" namespace nix { @@ -10,157 +11,196 @@ namespace nix { void Store::computeFSClosure(const Path & path, PathSet & paths, bool flipDirection, bool includeOutputs, bool includeDerivers) { - if (paths.find(path) != paths.end()) return; - paths.insert(path); + ThreadPool pool; - PathSet edges; + Sync<bool> state_; - if (flipDirection) { - queryReferrers(path, edges); + std::function<void(Path)> doPath; - if (includeOutputs) { - PathSet derivers = queryValidDerivers(path); - for (auto & i : derivers) - edges.insert(i); + doPath = [&](const Path & path) { + { + auto state(state_.lock()); + if (paths.count(path)) return; + paths.insert(path); } - if (includeDerivers && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - for (auto & i : outputs) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - edges.insert(i); - } - - } else { auto info = queryPathInfo(path); - edges = info->references; - if (includeOutputs && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - for (auto & i : outputs) - if (isValidPath(i)) edges.insert(i); + if (flipDirection) { + + PathSet referrers; + queryReferrers(path, referrers); + for (auto & ref : referrers) + if (ref != path) + pool.enqueue(std::bind(doPath, ref)); + + if (includeOutputs) { + PathSet derivers = queryValidDerivers(path); + for (auto & i : derivers) + pool.enqueue(std::bind(doPath, i)); + } + + if (includeDerivers && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + for (auto & i : outputs) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + pool.enqueue(std::bind(doPath, i)); + } + + } else { + + for (auto & ref : info->references) + if (ref != path) + pool.enqueue(std::bind(doPath, ref)); + + if (includeOutputs && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + for (auto & i : outputs) + if (isValidPath(i)) pool.enqueue(std::bind(doPath, i)); + } + + if (includeDerivers && isValidPath(info->deriver)) + pool.enqueue(std::bind(doPath, info->deriver)); + } + }; - if (includeDerivers && isValidPath(info->deriver)) - edges.insert(info->deriver); - } + pool.enqueue(std::bind(doPath, path)); - for (auto & i : edges) - computeFSClosure(i, paths, flipDirection, includeOutputs, includeDerivers); + pool.process(); } void Store::queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, + unsigned long long & downloadSize_, unsigned long long & narSize_) { - downloadSize = narSize = 0; + downloadSize_ = narSize_ = 0; - PathSet todo(targets.begin(), targets.end()), done; + ThreadPool pool; - /* Getting substitute info has high latency when using the binary - cache substituter. Thus it's essential to do substitute - queries in parallel as much as possible. To accomplish this - we do the following: - - - For all paths still to be processed (‘todo’), we add all - paths for which we need info to the set ‘query’. For an - unbuilt derivation this is the output paths; otherwise, it's - the path itself. - - - We get info about all paths in ‘query’ in parallel. + struct State + { + PathSet done; + PathSet & unknown, & willSubstitute, & willBuild; + unsigned long long & downloadSize; + unsigned long long & narSize; + }; - - We process the results and add new items to ‘todo’ if - necessary. E.g. if a path is substitutable, then we need to - get info on its references. + struct DrvState + { + size_t left; + bool done = false; + PathSet outPaths; + DrvState(size_t left) : left(left) { } + }; - - Repeat until ‘todo’ is empty. - */ + Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - while (!todo.empty()) { + std::function<void(Path)> doPath; - PathSet query, todoDrv, todoNonDrv; + auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) { + { + auto state(state_.lock()); + state->willBuild.insert(drvPath); + } - for (auto & i : todo) { - if (done.find(i) != done.end()) continue; - done.insert(i); + for (auto & i : drv.inputDrvs) + pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + }; - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); + auto checkOutput = [&]( + const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_) + { + if (drvState_->lock()->done) return; - if (isDerivation(i2.first)) { - if (!isValidPath(i2.first)) { - // FIXME: we could try to substitute p. - unknown.insert(i); - continue; + SubstitutablePathInfos infos; + querySubstitutablePathInfos({outPath}, infos); + + if (infos.empty()) { + drvState_->lock()->done = true; + mustBuildDrv(drvPath, *drv); + } else { + { + auto drvState(drvState_->lock()); + if (drvState->done) return; + assert(drvState->left); + drvState->left--; + drvState->outPaths.insert(outPath); + if (!drvState->left) { + for (auto & path : drvState->outPaths) + pool.enqueue(std::bind(doPath, path)); } - Derivation drv = derivationFromPath(i2.first); - - PathSet invalid; - for (auto & j : drv.outputs) - if (wantOutput(j.first, i2.second) - && !isValidPath(j.second.path)) - invalid.insert(j.second.path); - if (invalid.empty()) continue; - - todoDrv.insert(i); - if (settings.useSubstitutes && drv.substitutesAllowed()) - query.insert(invalid.begin(), invalid.end()); - } - - else { - if (isValidPath(i)) continue; - query.insert(i); - todoNonDrv.insert(i); } } + }; - todo.clear(); + doPath = [&](const Path & path) { - SubstitutablePathInfos infos; - querySubstitutablePathInfos(query, infos); + { + auto state(state_.lock()); + if (state->done.count(path)) return; + state->done.insert(path); + } + + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); - for (auto & i : todoDrv) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); + if (isDerivation(i2.first)) { + if (!isValidPath(i2.first)) { + // FIXME: we could try to substitute the derivation. + auto state(state_.lock()); + state->unknown.insert(path); + return; + } - // FIXME: cache this Derivation drv = derivationFromPath(i2.first); - PathSet outputs; - bool mustBuild = false; + PathSet invalid; + for (auto & j : drv.outputs) + if (wantOutput(j.first, i2.second) + && !isValidPath(j.second.path)) + invalid.insert(j.second.path); + if (invalid.empty()) return; + if (settings.useSubstitutes && drv.substitutesAllowed()) { - for (auto & j : drv.outputs) { - if (!wantOutput(j.first, i2.second)) continue; - if (!isValidPath(j.second.path)) { - if (infos.find(j.second.path) == infos.end()) - mustBuild = true; - else - outputs.insert(j.second.path); - } - } + auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); + for (auto & output : invalid) + pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); } else - mustBuild = true; + mustBuildDrv(i2.first, drv); - if (mustBuild) { - willBuild.insert(i2.first); - todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); - for (auto & j : drv.inputDrvs) - todo.insert(makeDrvPathWithOutputs(j.first, j.second)); - } else - todoNonDrv.insert(outputs.begin(), outputs.end()); - } + } else { - for (auto & i : todoNonDrv) { - done.insert(i); - SubstitutablePathInfos::iterator info = infos.find(i); - if (info != infos.end()) { - willSubstitute.insert(i); - downloadSize += info->second.downloadSize; - narSize += info->second.narSize; - todo.insert(info->second.references.begin(), info->second.references.end()); - } else - unknown.insert(i); + if (isValidPath(path)) return; + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({path}, infos); + + if (infos.empty()) { + auto state(state_.lock()); + state->unknown.insert(path); + return; + } + + auto info = infos.find(path); + assert(info != infos.end()); + + { + auto state(state_.lock()); + state->willSubstitute.insert(path); + state->downloadSize += info->second.downloadSize; + state->narSize += info->second.narSize; + } + + for (auto & ref : info->second.references) + pool.enqueue(std::bind(doPath, ref)); } - } + }; + + for (auto & path : targets) + pool.enqueue(std::bind(doPath, path)); + + pool.process(); } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1aacca84b800..b0a8d77c2fba 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -6,7 +6,7 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) { auto corrupt = [&]() { - throw Error("NAR info file ‘%1%’ is corrupt"); + throw Error(format("NAR info file ‘%1%’ is corrupt") % whence); }; auto parseHashField = [&](const string & s) { @@ -73,7 +73,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (path.empty() || url.empty()) corrupt(); + if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); } std::string NarInfo::to_string() const diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index fa42587777cd..81aced0fde16 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -121,6 +121,7 @@ const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; string printHash32(const Hash & hash) { + assert(hash.hashSize); size_t len = hash.base32Len(); assert(len); diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 32363ecf0098..696ecd6c38c8 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -36,7 +36,8 @@ ThreadPool::~ThreadPool() void ThreadPool::enqueue(const work_t & t) { auto state(state_.lock()); - assert(!state->quit); + if (state->quit) + throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); state->left.push(t); if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads) state->workers.emplace_back(&ThreadPool::workerEntry, this); @@ -84,7 +85,8 @@ void ThreadPool::workerEntry() } catch (std::exception & e) { auto state(state_.lock()); if (state->exception) { - if (!dynamic_cast<Interrupted*>(&e)) + if (!dynamic_cast<Interrupted*>(&e) && + !dynamic_cast<ThreadPoolShutDown*>(&e)) printMsg(lvlError, format("error: %s") % e.what()); } else { state->exception = std::current_exception(); diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 78b63467d62e..b64dc52d473e 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -10,6 +10,8 @@ namespace nix { +MakeError(ThreadPoolShutDown, Error) + /* A simple thread pool that executes a queue of work items (lambdas). */ class ThreadPool diff --git a/tests/common.sh.in b/tests/common.sh.in index e721c36967b9..097d94bb1eea 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -17,6 +17,7 @@ export NIX_CONF_DIR=$TEST_ROOT/etc export NIX_MANIFESTS_DIR=$TEST_ROOT/var/nix/manifests export _NIX_TEST_SHARED=$TEST_ROOT/shared export NIX_REMOTE=$NIX_REMOTE_ +unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME mkdir -p $TEST_HOME |