diff options
45 files changed, 491 insertions, 401 deletions
diff --git a/Makefile.config.in b/Makefile.config.in index 6948dad5a60b..3cae30d487d7 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -5,6 +5,7 @@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ +HAVE_READLINE = @HAVE_READLINE@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ diff --git a/configure.ac b/configure.ac index c7026cf954dd..ac37456ae5fe 100644 --- a/configure.ac +++ b/configure.ac @@ -196,6 +196,14 @@ if test "$gc" = yes; then fi +# Check for readline, needed by "nix repl". +AX_LIB_READLINE +if test "$ax_cv_lib_readline" != "no"; then + have_readline=1 +fi +AC_SUBST(HAVE_READLINE, [$have_readline]) + + AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], [do not initialise DB etc. in `make install']), init_state=$enableval, init_state=yes) diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index d5bc1c592553..1957e1105e68 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -22,10 +22,7 @@ will call whenever it wants to build a derivation. The build hook will perform it in the usual way if possible, or it can accept it, in which case it is responsible for somehow getting the inputs of the build to another machine, doing the build there, and getting the -results back. The details of the build hook protocol are described in -the documentation of the <link -linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar> -variable</link>.</para> +results back.</para> <example xml:id='ex-remote-systems'><title>Remote machine configuration: <filename>remote-systems.conf</filename></title> @@ -103,14 +100,6 @@ requiredSystemFeatures = [ "kvm" ]; </orderedlist> -You should also set up the environment variable -<envar>NIX_CURRENT_LOAD</envar> to point at a directory (e.g., -<filename>/var/run/nix/current-load</filename>) that -<filename>build-remote</filename> uses to remember how many builds -it is currently executing remotely. It doesn't look at the actual -load on the remote machine, so if you have multiple instances of Nix -running, they should use the same <envar>NIX_CURRENT_LOAD</envar> -file. Maybe in the future <filename>build-remote</filename> will -look at the actual remote load.</para> +</para> </chapter> diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index c757cb17bd10..a83aeaf2e575 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -148,139 +148,6 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen> </varlistentry> -<varlistentry xml:id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term> - - <listitem> - - <para>Specifies the location of the <emphasis>build hook</emphasis>, - which is a program (typically some script) that Nix will call - whenever it wants to build a derivation. This is used to implement - distributed builds<phrase condition="manual"> (see <xref - linkend="chap-distributed-builds" />)</phrase>.</para> - - <!-- - The protocol by - which the calling Nix process and the build hook communicate is as - follows. - - <para>The build hook is called with the following command-line - arguments: - - <orderedlist> - - <listitem><para>A boolean value <literal>0</literal> or - <literal>1</literal> specifying whether Nix can locally execute - more builds, as per the <link - linkend="opt-max-jobs"><option>- -max-jobs</option> option</link>. - The purpose of this argument is to allow the hook to not have to - maintain bookkeeping for the local machine.</para></listitem> - - <listitem><para>The Nix platform identifier for the local machine - (e.g., <literal>i686-linux</literal>).</para></listitem> - - <listitem><para>The Nix platform identifier for the derivation, - i.e., its <link linkend="attr-system"><varname>system</varname> - attribute</link>.</para></listitem> - - <listitem><para>The store path of the derivation.</para></listitem> - - </orderedlist> - - </para> - - <para>On the basis of this information, and whatever persistent - state the build hook keeps about other machines and their current - load, it has to decide what to do with the build. It should print - out on standard error one of the following responses (terminated by - a newline, <literal>"\n"</literal>): - - <variablelist> - - <varlistentry><term><literal># decline</literal></term> - - <listitem><para>The build hook is not willing or able to perform - the build; the calling Nix process should do the build itself, - if possible.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal># postpone</literal></term> - - <listitem><para>The build hook cannot perform the build now, but - can do so in the future (e.g., because all available build slots - on remote machines are in use). The calling Nix process should - postpone this build until at least one currently running build - has terminated.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal># accept</literal></term> - - <listitem><para>The build hook has accepted the - build.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - <para>After sending <literal># accept</literal>, the hook should - read one line from standard input, which will be the string - <literal>okay</literal>. It can then proceed with the build. - Before sending <literal>okay</literal>, Nix will store in the hook’s - current directory a number of text files that contain information - about the derivation: - - <variablelist> - - <varlistentry><term><filename>inputs</filename></term> - - <listitem><para>The set of store paths that are inputs to the - build process (one per line). These have to be copied - <emphasis>to</emphasis> the remote machine (in addition to the - store derivation itself).</para></listitem> - - </varlistentry> - - <varlistentry><term><filename>outputs</filename></term> - - <listitem><para>The set of store paths that are outputs of the - derivation (one per line). These have to be copied - <emphasis>from</emphasis> the remote machine if the build - succeeds.</para></listitem> - - </varlistentry> - - <varlistentry><term><filename>references</filename></term> - - <listitem><para>The reference graph of the inputs, in the format - accepted by the command <command>nix-store - - -register-validity</command>. It is necessary to run this - command on the remote machine after copying the inputs to inform - Nix on the remote machine that the inputs are valid - paths.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - <para>The hook should copy the inputs to the remote machine, - register the validity of the inputs, perform the remote build, and - copy the outputs back to the local machine. An exit code other than - <literal>0</literal> indicates that the hook has failed. An exit - code equal to 100 means that the remote build failed (as opposed to, - e.g., a network error).</para> - --> - - </listitem> - - -</varlistentry> - - <varlistentry xml:id="envar-remote"><term><envar>NIX_REMOTE</envar></term> <listitem><para>This variable should be set to diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index fcd799e177d0..9bfb00e306b6 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -8,3 +8,4 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process Environment=XDG_CACHE_HOME=/root/.cache +Environment=XDG_CONFIG_HOME=/root/.config diff --git a/nix.spec.in b/nix.spec.in index 390893d64dc2..3ba2dfc94b41 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -20,9 +20,11 @@ Requires: curl Requires: bzip2 Requires: gzip Requires: xz +Requires: readline BuildRequires: bzip2-devel BuildRequires: sqlite-devel BuildRequires: libcurl-devel +BuildRequires: readline-devel # Hack to make that shitty RPM scanning hack shut up. Provides: perl(Nix::SSH) diff --git a/release.nix b/release.nix index 534c218c1123..7adc87386f9b 100644 --- a/release.nix +++ b/release.nix @@ -299,7 +299,7 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" ] + [ "sqlite" "sqlite-devel" "bzip2-devel" "emacs" "libcurl-devel" "openssl-devel" "xz-devel" "readline-devel" ] ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; @@ -321,14 +321,14 @@ let src = jobs.tarball; diskImage = (diskImageFun vmTools.diskImageFuns) { extraPackages = - [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" ] + [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libreadline-dev" ] ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; postInstall = "make installcheck"; configureFlags = "--sysconfdir=/etc"; debRequires = - [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" ] + [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libreadline6" ] ++ extraDebPackages; debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>"; doInstallCheck = true; diff --git a/shell.nix b/shell.nix index 37a936fd2efb..bbce68564b95 100644 --- a/shell.nix +++ b/shell.nix @@ -35,7 +35,7 @@ with import <nixpkgs> {}; shellHook = '' export prefix=$(pwd)/inst - configureFlags+=" --prefix=prefix" + configureFlags+=" --prefix=$prefix" PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH PATH=$prefix/bin:$PATH ''; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index d7aee288670a..8876da6c063c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,6 +9,7 @@ #include <sys/time.h> #endif +#include "machines.hh" #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" @@ -22,131 +23,56 @@ using std::cin; static void handleAlarm(int sig) { } -class Machine { - const std::set<string> supportedFeatures; - const std::set<string> mandatoryFeatures; - -public: - const string hostName; - const std::vector<string> systemTypes; - const string sshKey; - const unsigned int maxJobs; - const unsigned int speedFactor; - bool enabled; - - bool allSupported(const std::set<string> & features) const { - return std::all_of(features.begin(), features.end(), - [&](const string & feature) { - return supportedFeatures.count(feature) || - mandatoryFeatures.count(feature); - }); - } - - bool mandatoryMet(const std::set<string> & features) const { - return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { - return features.count(feature); - }); - } - - Machine(decltype(hostName) hostName, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures) : - supportedFeatures(supportedFeatures), - mandatoryFeatures(mandatoryFeatures), - hostName(hostName), - systemTypes(systemTypes), - sshKey(sshKey), - maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), - enabled(true) - {}; -};; - -static std::vector<Machine> readConf() +std::string escapeUri(std::string uri) { - auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines"); - - auto machines = std::vector<Machine>{}; - auto lines = std::vector<string>{}; - try { - lines = tokenizeString<std::vector<string>>(readFile(conf), "\n"); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - for (auto line : lines) { - chomp(line); - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) { - continue; - } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 4) - throw FormatError("bad machines.conf file ‘%1%’", conf); - machines.emplace_back(tokens[0], - tokenizeString<std::vector<string>>(tokens[1], ","), - tokens[2], - stoull(tokens[3]), - sz >= 5 ? stoull(tokens[4]) : 1LL, - sz >= 6 ? - tokenizeString<std::set<string>>(tokens[5], ",") : - std::set<string>{}, - sz >= 7 ? - tokenizeString<std::set<string>>(tokens[6], ",") : - std::set<string>{}); - } - return machines; + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; } static string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) { - std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out); - fn_stream << "/"; - for (auto t : m.systemTypes) { - fn_stream << t << "-"; - } - fn_stream << m.hostName << "-" << slot; - return openLockFile(fn_stream.str(), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static char display_env[] = "DISPLAY="; -static char ssh_env[] = "SSH_ASKPASS="; - int main (int argc, char * * argv) { return handleExceptions(argv[0], [&]() { initNix(); /* Ensure we don't get any SSH passphrase or host key popups. */ - if (putenv(display_env) == -1 || - putenv(ssh_env) == -1) - throw SysError("setting SSH env vars"); + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); - if (argc != 4) + if (argc != 6) throw UsageError("called without required arguments"); auto store = openStore(); auto localSystem = argv[1]; - settings.maxSilentTime = stoull(string(argv[2])); - settings.buildTimeout = stoull(string(argv[3])); + settings.maxSilentTime = std::stoll(argv[2]); + settings.buildTimeout = std::stoll(argv[3]); + verbosity = (Verbosity) std::stoll(argv[4]); + settings.builders = argv[5]; - currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load"); + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on OS X. */ + currentLoad = settings.nixStateDir + "/current-load"; std::shared_ptr<Store> sshStore; AutoCloseFD bestSlotLock; - auto machines = readConf(); + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return; + } + string drvPath; - string hostName; + string storeUri; for (string line; getline(cin, line);) { auto tokens = tokenizeString<std::vector<string>>(line); auto sz = tokens.size(); @@ -173,6 +99,8 @@ int main (int argc, char * * argv) Machine * bestMachine = nullptr; unsigned long long bestLoad = 0; for (auto & m : machines) { + debug("considering building on ‘%s’", m.storeUri); + if (m.enabled && std::find(m.systemTypes.begin(), m.systemTypes.end(), neededSystem) != m.systemTypes.end() && @@ -233,16 +161,22 @@ int main (int argc, char * * argv) lock = -1; try { - sshStore = openStore("ssh-ng://" + bestMachine->hostName, - { {"ssh-key", bestMachine->sshKey }, - {"max-connections", "1" } }); - hostName = bestMachine->hostName; + + Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}}; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + } catch (std::exception & e) { printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...", - bestMachine->hostName, e.what()); + bestMachine->storeUri, e.what()); bestMachine->enabled = false; continue; } + goto connected; } } @@ -252,22 +186,29 @@ connected: string line; if (!getline(cin, line)) throw Error("hook caller didn't send inputs"); + auto inputs = tokenizeString<PathSet>(line); if (!getline(cin, line)) throw Error("hook caller didn't send outputs"); + auto outputs = tokenizeString<PathSet>(line); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true); + + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); if (!lockFile(uploadLock.get(), ltWrite, true)) printError("somebody is hogging the upload lock for ‘%s’, continuing..."); alarm(0); signal(SIGALRM, old); - copyPaths(store, ref<Store>(sshStore), inputs); + copyPaths(store, ref<Store>(sshStore), inputs, false, true); uploadLock = -1; - printError("building ‘%s’ on ‘%s’", drvPath, hostName); - sshStore->buildDerivation(drvPath, readDerivation(drvPath)); + BasicDerivation drv(readDerivation(drvPath)); + drv.inputSrcs = inputs; + + printError("building ‘%s’ on ‘%s’", drvPath, storeUri); + sshStore->buildDerivation(drvPath, drv); PathSet missing; for (auto & path : outputs) @@ -275,7 +216,7 @@ connected: if (!missing.empty()) { setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing); + copyPaths(ref<Store>(sshStore), store, missing, false, true); } return; diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 62d5a010c247..64368a43ff73 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -7,5 +7,3 @@ build-remote_INSTALL_DIR := $(libexecdir)/nix build-remote_LIBS = libmain libutil libformat libstore build-remote_SOURCES := $(d)/build-remote.cc - -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5b1ff0350cd1..40ca77258037 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -142,25 +142,34 @@ or { return OR_KW; } \{ { return '{'; } <INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { PUSH_STATE(STRING); return '"'; } +<INITIAL,INSIDE_DOLLAR_CURLY>\" { + PUSH_STATE(STRING); return '"'; + } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { - /* It is impossible to match strings ending with '$' with one - regex because trailing contexts are only valid at the end - of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); - return STR; - } + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext); + return STR; + } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -<STRING>\" { POP_STATE(); return '"'; } -<STRING>. return yytext[0]; /* just in case: shouldn't be reached */ +<STRING>\" { POP_STATE(); return '"'; } +<STRING>\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } <INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; } -<IND_STRING>\'\'\$ { +<IND_STRING>\'\'\$ | +<IND_STRING>\$ { yylval->e = new ExprIndStr("$"); return IND_STR; } @@ -178,7 +187,6 @@ or { return OR_KW; } yylval->e = new ExprIndStr("'"); return IND_STR; } -<IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ <INITIAL,INSIDE_DOLLAR_CURLY>{ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b536c6c00044..46c5aa21b2eb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -114,11 +114,6 @@ void BinaryCacheStore::init() } } -void BinaryCacheStore::notImpl() -{ - throw Error("operation not implemented for binary cache stores"); -} - std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) { std::promise<std::shared_ptr<std::string>> promise; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5c2d0acfdbb7..87d4aa43838e 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -27,8 +27,6 @@ protected: BinaryCacheStore(const Params & params); - [[noreturn]] void notImpl(); - public: virtual bool fileExists(const std::string & path) = 0; @@ -65,7 +63,7 @@ public: bool isValidPathUncached(const Path & path) override; PathSet queryAllValidPaths() override - { notImpl(); } + { unsupported(); } void queryPathInfoUncached(const Path & path, std::function<void(std::shared_ptr<ValidPathInfo>)> success, @@ -73,16 +71,16 @@ public: void queryReferrers(const Path & path, PathSet & referrers) override - { notImpl(); } + { unsupported(); } PathSet queryDerivationOutputs(const Path & path) override - { notImpl(); } + { unsupported(); } StringSet queryDerivationOutputNames(const Path & path) override - { notImpl(); } + { unsupported(); } Path queryPathFromHashPart(const string & hashPart) override - { notImpl(); } + { unsupported(); } bool wantMassQuery() override { return wantMassQuery_; } @@ -99,32 +97,29 @@ public: void narFromPath(const Path & path, Sink & sink) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { notImpl(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { notImpl(); } + { unsupported(); } void ensurePath(const Path & path) override - { notImpl(); } + { unsupported(); } void addTempRoot(const Path & path) override - { notImpl(); } + { unsupported(); } void addIndirectRoot(const Path & path) override - { notImpl(); } + { unsupported(); } Roots findRoots() override - { notImpl(); } + { unsupported(); } void collectGarbage(const GCOptions & options, GCResults & results) override - { notImpl(); } + { unsupported(); } ref<FSAccessor> getFSAccessor() override; void addSignatures(const Path & storePath, const StringSet & sigs) override - { notImpl(); } + { unsupported(); } std::shared_ptr<std::string> getBuildLog(const Path & path) override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9bf1ab5aa581..8c2602a701bd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -583,11 +583,7 @@ struct HookInstance HookInstance::HookInstance() { - debug("starting build hook"); - - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; - buildHook = canonPath(buildHook); + debug("starting build hook ‘%s’", settings.buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -614,15 +610,17 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(buildHook), + baseNameOf(settings.buildHook), settings.thisSystem, - (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.buildTimeout).str() + std::to_string(settings.maxSilentTime), + std::to_string(settings.buildTimeout), + std::to_string(verbosity), + settings.builders }; - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError(format("executing ‘%1%’") % buildHook); + throw SysError("executing ‘%s’", settings.buildHook); }); pid.setSeparatePG(true); @@ -1568,7 +1566,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; + if (!settings.useBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1601,8 +1599,15 @@ HookReply DerivationGoal::tryBuildHook() debug(format("hook reply is ‘%1%’") % reply); - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + settings.useBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; else if (reply != "accept") throw Error(format("bad hook reply ‘%1%’") % reply); @@ -1621,23 +1626,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - worker.store.computeFSClosure(drvPath, allInputs); - - string s; - for (auto & i : allInputs) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + remote system. */ + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - s = ""; - for (auto & i : missingPaths) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); hook->toHook.writeSide = -1; @@ -1876,6 +1870,7 @@ void DerivationGoal::startBuilder() dirsInChroot[i] = r; else { Path p = chrootRootDir + i; + debug("linking ‘%1%’ to ‘%2%’", p, r); if (link(r.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), @@ -3105,7 +3100,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide.get()) - printError(data); // FIXME? + printError(chomp(data)); } @@ -3379,7 +3374,7 @@ void SubstitutionGoal::tryToRun() if maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::min(1U, (unsigned int) settings.maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 953bf6aaaa0a..4bdbde989ab2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -43,6 +43,10 @@ Settings::Settings() lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt")); + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s != "") builderFiles = tokenizeString<Strings>(s, ":"); + #if __linux__ sandboxPaths = tokenizeString<StringSet>("/bin/sh=" BASH_PATH); #endif diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b4f44de2e65d..ac6f6a2cfa36 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -127,6 +127,16 @@ public: Setting<bool> useBuildHook{this, true, "remote-builds", "Whether to use build hooks (for distributed builds)."}; + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + "The path of the helper program that executes builds to remote machines."}; + + Setting<std::string> builders{this, "", "builders", + "A semicolon-separated list of build machines, in the format of nix.machines."}; + + Setting<Strings> builderFiles{this, + {nixConfDir + "/machines"}, "builder-files", + "A list of files specifying build machines."}; + Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index befc560bfcec..e09932e3d182 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "worker-protocol.hh" #include "ssh.hh" +#include "derivations.hh" namespace nix { @@ -16,11 +17,15 @@ struct LegacySSHStore : public Store const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; + // Hack for getting remote build log output. + const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + struct Connection { std::unique_ptr<SSHMaster::Connection> sshConn; FdSink to; FdSource from; + int remoteVersion; }; std::string host; @@ -42,7 +47,8 @@ struct LegacySSHStore : public Store sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - compress) + compress, + logFD) { } @@ -53,8 +59,6 @@ struct LegacySSHStore : public Store conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - int remoteVersion; - try { conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; conn->to.flush(); @@ -62,8 +66,8 @@ struct LegacySSHStore : public Store unsigned int magic = readInt(conn->from); if (magic != SERVE_MAGIC_2) throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host); - remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host); } catch (EndOfFile & e) { @@ -148,12 +152,6 @@ struct LegacySSHStore : public Store sink(*savedNAR.data); } - /* Unsupported methods. */ - [[noreturn]] void unsupported() - { - throw Error("operation not supported on SSH stores"); - } - PathSet queryAllValidPaths() override { unsupported(); } void queryReferrers(const Path & path, PathSet & referrers) override @@ -177,12 +175,36 @@ struct LegacySSHStore : public Store const PathSet & references, bool repair) override { unsupported(); } - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { unsupported(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { unsupported(); } + { + auto conn(connections->get()); + + conn->to + << cmdBuildDerivation + << drvPath + << drv + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to + << settings.buildRepeat + << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status) readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + + return status; + } void ensurePath(const Path & path) override { unsupported(); } @@ -205,9 +227,6 @@ struct LegacySSHStore : public Store void addSignatures(const Path & storePath, const StringSet & sigs) override { unsupported(); } - bool isTrusted() override - { return true; } - void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -243,6 +262,11 @@ struct LegacySSHStore : public Store return readStorePaths<PathSet>(*this, conn->from); } + + void connect() override + { + auto conn(connections->get()); + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bf247903c9db..bf28a1c70c62 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -31,7 +31,7 @@ struct LocalStoreAccessor : public FSAccessor auto realPath = toRealPath(path); struct stat st; - if (lstat(path.c_str(), &st)) { + if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; throw SysError(format("getting status of ‘%1%’") % path); } @@ -51,7 +51,7 @@ struct LocalStoreAccessor : public FSAccessor { auto realPath = toRealPath(path); - auto entries = nix::readDirectory(path); + auto entries = nix::readDirectory(realPath); StringSet res; for (auto & entry : entries) @@ -73,7 +73,8 @@ struct LocalStoreAccessor : public FSAccessor ref<FSAccessor> LocalFSStore::getFSAccessor() { - return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); + return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( + std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } void LocalFSStore::narFromPath(const Path & path, Sink & sink) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a98454ab38e..c8e61126c1b8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -915,6 +915,8 @@ void LocalStore::invalidatePath(State & state, const Path & path) void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { + assert(info.narHash); + Hash h = hashString(htSHA256, *nar); if (h != info.narHash) throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc new file mode 100644 index 000000000000..c1d9047537d3 --- /dev/null +++ b/src/libstore/machines.cc @@ -0,0 +1,91 @@ +#include "machines.hh" +#include "util.hh" +#include "globals.hh" + +#include <algorithm> + +namespace nix { + +Machine::Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) : + storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) +{} + +bool Machine::allSupported(const std::set<string> & features) const { + return std::all_of(features.begin(), features.end(), + [&](const string & feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); +} + +bool Machine::mandatoryMet(const std::set<string> & features) const { + return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string & feature) { + return features.count(feature); + }); +} + +void parseMachines(const std::string & s, Machines & machines) +{ + for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + chomp(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; + auto tokens = tokenizeString<std::vector<string>>(line); + auto sz = tokens.size(); + if (sz < 1) + throw FormatError("bad machine specification ‘%s’", line); + + auto isSet = [&](int n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; + + machines.emplace_back(tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() +{ + Machines machines; + + for (auto & file : settings.builderFiles.get()) { + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + } + } + + parseMachines(settings.builders, machines); + + return machines; +} + +} diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh new file mode 100644 index 000000000000..de92eb924e4a --- /dev/null +++ b/src/libstore/machines.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct Machine { + + const string storeUri; + const std::vector<string> systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set<string> supportedFeatures; + const std::set<string> mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set<string> & features) const; + + bool mandatoryMet(const std::set<string> & features) const; + + Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); +}; + +typedef std::vector<Machine> Machines; + +void parseMachines(const std::string & s, Machines & machines); + +Machines getMachines(); + +} diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cf234e35d373..d354812e3da4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -220,8 +220,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - if (st.st_size) - printInfo(format("‘%1%’ has maximum number of links") % linkPath); + debug("‘%s’ has reached maximum number of links", linkPath); return; } throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index af59d51106fc..be8819bbc004 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -100,7 +100,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() throw Error(format("socket path ‘%1%’ is too long") % socketPath); strcpy(addr.sun_path, socketPath.c_str()); - if (connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); conn->from.fd = conn->fd.get(); @@ -613,6 +613,12 @@ void RemoteStore::queryMissing(const PathSet & targets, } +void RemoteStore::connect() +{ + auto conn(connections->get()); +} + + RemoteStore::Connection::~Connection() { try { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 479cf3a7909d..ed430e4cabb6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -92,6 +92,8 @@ public: PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) override; + void connect() override; + protected: struct Connection diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index e54f3f4ba284..6edabaa3a1d9 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -31,6 +31,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string throw SysError("duping over stdin"); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) + throw SysError("duping over stderr"); Strings args = { "ssh", host.c_str(), "-x", "-a" }; addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index b4396467e54e..18dea227ad1f 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -13,6 +13,7 @@ private: const std::string keyFile; const bool useMaster; const bool compress; + const int logFD; struct State { @@ -27,11 +28,12 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress) + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1) : host(host) , keyFile(keyFile) , useMaster(useMaster) , compress(compress) + , logFD(logFD) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 835bbb90e0bb..b5a91e53672f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -523,6 +523,17 @@ const Store::Stats & Store::getStats() } +void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & path : paths) + if (isDerivation(path)) + unsupported(); + + if (queryValidPaths(paths).size() != paths.size()) + unsupported(); +} + + void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, bool repair, bool dontCheckSigs) { @@ -531,15 +542,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); - if (srcStore->isTrusted()) - dontCheckSigs = true; - if (!info->narHash && dontCheckSigs) { auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); info = info2; } + assert(info->narHash); + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + assert(info->narHash); + dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); } @@ -698,10 +716,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStore(const std::string & uri_) +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) { auto uri(uri_); - Store::Params params; + Store::Params params(extraParams); auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { @@ -711,11 +730,7 @@ ref<Store> openStore(const std::string & uri_) } uri = uri_.substr(0, q); } - return openStore(uri, params); -} -ref<Store> openStore(const std::string & uri, const Store::Params & params) -{ for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { @@ -724,7 +739,7 @@ ref<Store> openStore(const std::string & uri, const Store::Params & params) } } - throw Error(format("don't know how to open Nix store ‘%s’") % uri); + throw Error("don't know how to open Nix store ‘%s’", uri); } @@ -794,7 +809,8 @@ std::list<ref<Store>> getDefaultSubstituters() } -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute) +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute, bool dontCheckSigs) { PathSet valid = to->queryValidPaths(storePaths, substitute); @@ -822,7 +838,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool if (!to->isValidPath(storePath)) { Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); - copyStorePath(from, to, storePath); + copyStorePath(from, to, storePath, false, dontCheckSigs); logger->incProgress(copiedLabel); } else diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 067309c9e956..b06f5d86a93a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -18,6 +18,12 @@ namespace nix { +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) +MakeError(Unsupported, Error) + + struct BasicDerivation; struct Derivation; class FSAccessor; @@ -414,7 +420,7 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for @@ -564,10 +570,6 @@ public: const Stats & getStats(); - /* Whether this store paths from this store can be imported even - if they lack a signature. */ - virtual bool isTrusted() { return false; } - /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr<std::string> getBuildLog(const Path & path) @@ -580,10 +582,20 @@ public: state.lock()->pathInfoCache.clear(); } + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect() { }; + protected: Stats stats; + /* Unsupported methods. */ + [[noreturn]] void unsupported() + { + throw Unsupported("requested operation is not supported by store ‘%s’", getUri()); + } + }; @@ -656,23 +668,35 @@ void removeTempRoots(); /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: - * ‘direct’: The Nix store in /nix/store and database in + * ‘local’: The Nix store in /nix/store and database in /nix/var/nix/db, accessed directly. * ‘daemon’: The Nix store accessed via a Unix domain socket connection to nix-daemon. + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + * ‘file://<path>’: A binary cache stored in <path>. - If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on - whether the user has write access to the local Nix store/database. - set to true *unless* you're going to collect garbage. */ -ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); + * ‘https://<path>’: A binary cache accessed via HTTP. + + * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]<host>’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. -ref<Store> openStore(const std::string & uri, const Store::Params & params); + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), + const Store::Params & extraParams = Store::Params()); -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false); +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute = false, bool dontCheckSigs = false); enum StoreType { tDaemon, @@ -720,10 +744,4 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); - -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) - - } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 62c6433c741b..497afaa1fed4 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -215,6 +215,7 @@ template class BaseSetting<unsigned long>; template class BaseSetting<long long>; template class BaseSetting<unsigned long long>; template class BaseSetting<bool>; +template class BaseSetting<std::string>; void PathSetting::set(const std::string & str) { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9f4afd93c2fc..fa1bb5d97183 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -224,7 +224,7 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, unsigned int len) + const unsigned char * bytes, size_t len) { if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 026e493514ea..98c0aff1e722 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1078,9 +1078,9 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & suffix) +bool hasPrefix(const string & s, const string & prefix) { - return s.compare(0, suffix.size(), suffix) == 0; + return s.compare(0, prefix.size(), prefix) == 0; } diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index ed43bffbc8c8..dc324abcb3ba 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -58,6 +58,6 @@ int main(int argc, char ** argv) PathSet closure; from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, closure, useSubstitutes); + copyPaths(from, to, closure, useSubstitutes, true); }); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 07ad0b45b3e4..1b90fad165af 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -483,7 +483,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe }; try { - if (trusted + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted || name == settings.buildTimeout.name || name == settings.connectTimeout.name) settings.set(name, value); diff --git a/src/nix/command.hh b/src/nix/command.hh index dc7b2637d66a..cf0097d78926 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -78,7 +78,7 @@ struct InstallablesCommand : virtual Args, StoreCommand = import ...; bla = import ...; }’. */ Value * getSourceExpr(EvalState & state); - std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings installables); + std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss); PathSet buildInstallables(ref<Store> store, bool dryRun); @@ -86,6 +86,8 @@ struct InstallablesCommand : virtual Args, StoreCommand void prepare() override; + virtual bool useDefaultInstallables() { return true; } + private: Strings _installables; @@ -112,6 +114,8 @@ public: virtual void run(ref<Store> store, Paths storePaths) = 0; void run(ref<Store> store) override; + + bool useDefaultInstallables() override { return !all; } }; typedef std::map<std::string, ref<Command>> Commands; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 57580049f25f..4756fc44bba7 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -177,21 +177,21 @@ struct InstallableAttrPath : Installable std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); -std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings installables) +std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings ss) { std::vector<std::shared_ptr<Installable>> result; - if (installables.empty()) { + if (ss.empty() && useDefaultInstallables()) { if (file == "") file = "."; - installables = Strings{""}; + ss = Strings{""}; } - for (auto & installable : installables) { + for (auto & s : ss) { - if (installable.find("/") != std::string::npos) { + if (s.find("/") != std::string::npos) { - auto path = store->toStorePath(store->followLinksToStore(installable)); + auto path = store->toStorePath(store->followLinksToStore(s)); if (store->isStorePath(path)) { if (isDerivation(path)) @@ -201,14 +201,14 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables } } - else if (installable.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(*this, installable)); + else if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(*this, s)); - else if (installable == "" || std::regex_match(installable, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(*this, installable)); + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared<InstallableAttrPath>(*this, s)); else - throw UsageError("don't know what to do with argument ‘%s’", installable); + throw UsageError("don't know what to do with argument ‘%s’", s); } return result; diff --git a/src/nix/local.mk b/src/nix/local.mk index 21f190e476f4..e71cf16fabf6 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -6,6 +6,8 @@ nix_SOURCES := $(wildcard $(d)/*.cc) nix_LIBS = libexpr libmain libstore libutil libformat -nix_LDFLAGS = -lreadline +ifeq ($(HAVE_READLINE), 1) + nix_LDFLAGS += -lreadline +endif $(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 17203d3c299f..13488bf1dbd4 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,3 +1,5 @@ +#if HAVE_LIBREADLINE + #include <iostream> #include <cstdlib> @@ -726,3 +728,5 @@ struct CmdRepl : StoreCommand static RegisterCommand r1(make_ref<CmdRepl>()); } + +#endif diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 666cc6ef8041..8bff0fe79032 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -5,6 +5,7 @@ let input1 = mkDerivation { name = "build-hook-input-1"; builder = ./dependencies.builder1.sh; + requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { diff --git a/tests/build-hook.sh b/tests/build-hook.sh index ef77a3ae5285..2005c7cebdc4 100644 --- a/tests/build-hook.sh +++ b/tests/build-hook.sh @@ -1,8 +1,8 @@ source common.sh -export NIX_BUILD_HOOK="$(pwd)/build-hook.hook.sh" +clearStore -outPath=$(nix-build build-hook.nix --no-out-link) +outPath=$(nix-build build-hook.nix --no-out-link --option build-hook $(pwd)/build-hook.hook.sh) echo "output path is $outPath" diff --git a/tests/build-remote.sh b/tests/build-remote.sh new file mode 100644 index 000000000000..071011dcb71d --- /dev/null +++ b/tests/build-remote.sh @@ -0,0 +1,24 @@ +source common.sh + +clearStore + +if [[ $(uname) != Linux ]]; then exit; fi +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +chmod -R u+w $TEST_ROOT/store1 || true +rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 + +# FIXME: --option is not passed to build-remote, so have to create a config file. +export NIX_CONF_DIR=$TEST_ROOT/etc2 +mkdir -p $NIX_CONF_DIR +echo "build-sandbox-paths = /nix/store" > $NIX_CONF_DIR/nix.conf + +outPath=$(nix-build build-hook.nix --no-out-link -j0 --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo" --option build-sandbox-paths /nix/store) + +cat $outPath/foobar | grep FOOBAR + +# Ensure that input1 was built on store1 due to the required feature. +p=$(readlink -f $outPath/input-2) +(! nix path-info --store local?root=$TEST_ROOT/store0 --all | grep dependencies.builder1.sh) +nix path-info --store local?root=$TEST_ROOT/store1 --all | grep dependencies.builder1.sh diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/lang/eval-okay-ind-string.exp index 886219dcf652..9cf4bd2ee78a 100644 --- a/tests/lang/eval-okay-ind-string.exp +++ b/tests/lang/eval-okay-ind-string.exp @@ -1 +1 @@ -"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\n" +"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n" diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/lang/eval-okay-ind-string.nix index 1556aae9f54f..1669dc0648ea 100644 --- a/tests/lang/eval-okay-ind-string.nix +++ b/tests/lang/eval-okay-ind-string.nix @@ -117,4 +117,12 @@ let bar ''; -in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + # Regression test: accept $'. + s16 = '' + cut -d $'\t' -f 1 + ''; + + # Accept dollars at end of strings + s17 = ''ending dollar $'' + ''$'' + "\n"; + +in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh new file mode 100644 index 000000000000..740b2c357099 --- /dev/null +++ b/tests/linux-sandbox.sh @@ -0,0 +1,27 @@ +source common.sh + +clearStore + +if [[ $(uname) != Linux ]]; then exit; fi + +# Note: we need to bind-mount $SHELL into the chroot. Currently we +# only support the case where $SHELL is in the Nix store, because +# otherwise things get complicated (e.g. if it's in /bin, do we need +# /lib as well?). +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +export NIX_STORE_DIR=/my/store +export NIX_REMOTE="local?root=$TEST_ROOT/store0" + +outPath=$( nix-build dependencies.nix --no-out-link --option build-sandbox-paths /nix/store) + +[[ $outPath =~ /my/store/.*-dependencies ]] + +nix path-info -r $outPath | grep input-2 + +nix ls-store -R -l $outPath | grep foobar + +nix cat-store $outPath/foobar | grep FOOBAR diff --git a/tests/local.mk b/tests/local.mk index b3ce39cda806..108e3febdb0c 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -11,7 +11,9 @@ nix_tests = \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ - placeholders.sh nix-shell.sh + placeholders.sh nix-shell.sh \ + linux-sandbox.sh \ + build-remote.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 63aaa4d88f56..39bd090e43e7 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -43,7 +43,6 @@ in { config, pkgs, ... }: { nix.maxJobs = 0; # force remote building nix.distributedBuilds = true; - nix.envVars = pkgs.lib.mkAfter { NIX_BUILD_HOOK = "${nix}/libexec/nix/build-remote"; }; nix.buildMachines = [ { hostName = "slave1"; sshUser = "root"; |