diff options
75 files changed, 1340 insertions, 843 deletions
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 6b90083f0873..fb4d8cefc4d2 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -563,7 +563,8 @@ password <replaceable>my-password</replaceable> <para>If set to <literal>true</literal>, the Nix evaluator will not allow access to any files outside of the Nix search path (as set via the <envar>NIX_PATH</envar> environment variable or the - <option>-I</option> option). The default is + <option>-I</option> option), or to URIs outside of + <option>allowed-uri</option>. The default is <literal>false</literal>.</para> </listitem> @@ -571,6 +572,21 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term> + + <listitem> + + <para>A list of URI prefixes to which access is allowed in + restricted evaluation mode. For example, when set to + <literal>https://github.com/NixOS</literal>, builtin functions + such as <function>fetchGit</function> are allowed to access + <literal>https://github.com/NixOS/patchelf.git</literal>.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term> <listitem> diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index fe69dba837a1..2f0027d479cd 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -333,7 +333,20 @@ with (import ./definitions.nix); ...</programlisting> makes all attributes defined in the file <filename>definitions.nix</filename> available as if they were defined -locally in a <literal>rec</literal>-expression.</para> +locally in a <literal>let</literal>-expression.</para> + +<para>The bindings introduced by <literal>with</literal> do not shadow bindings +introduced by other means, e.g. + +<programlisting> +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...</programlisting> + +establishes the same scope as + +<programlisting> +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...</programlisting> + +</para> </simplesect> diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index a3323ced4c50..f1f75093461b 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -41,6 +41,11 @@ weakest binding).</para> argument <replaceable>e2</replaceable>.</entry> </row> <row> + <entry><literal>-</literal> <replaceable>e</replaceable></entry> + <entry>none</entry> + <entry>Arithmetic negation.</entry> + </row> + <row> <entry><replaceable>e</replaceable> <literal>?</literal> <replaceable>attrpath</replaceable></entry> <entry>none</entry> @@ -55,13 +60,24 @@ weakest binding).</para> <entry>List concatenation.</entry> </row> <row> - <entry><replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable></entry> + <entry> + <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>, + <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable> + </entry> <entry>left</entry> - <entry>String or path concatenation.</entry> + <entry>Arithmetic multiplication and division.</entry> </row> <row> - <entry><literal>!</literal> <replaceable>e</replaceable></entry> + <entry> + <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>, + <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable> + </entry> <entry>left</entry> + <entry>Arithmetic addition and subtraction. String or path concatenation (only by <literal>+</literal>).</entry> + </row> + <row> + <entry><literal>!</literal> <replaceable>e</replaceable></entry> + <entry>none</entry> <entry>Boolean negation.</entry> </row> <row> @@ -75,16 +91,22 @@ weakest binding).</para> attributes).</entry> </row> <row> - <entry><replaceable>e1</replaceable> <literal>==</literal> - <replaceable>e2</replaceable></entry> + <entry> + <replaceable>e1</replaceable> <literal><</literal> <replaceable>e2</replaceable>, + <replaceable>e1</replaceable> <literal>></literal> <replaceable>e2</replaceable>, + <replaceable>e1</replaceable> <literal><=</literal> <replaceable>e2</replaceable>, + <replaceable>e1</replaceable> <literal>>=</literal> <replaceable>e2</replaceable> + </entry> <entry>none</entry> - <entry>Equality.</entry> + <entry>Arithmetic comparison.</entry> </row> <row> - <entry><replaceable>e1</replaceable> <literal>!=</literal> - <replaceable>e2</replaceable></entry> + <entry> + <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>, + <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable> + </entry> <entry>none</entry> - <entry>Inequality.</entry> + <entry>Equality and inequality.</entry> </row> <row> <entry><replaceable>e1</replaceable> <literal>&&</literal> diff --git a/doc/manual/expressions/language-values.xml b/doc/manual/expressions/language-values.xml index 67da688a4fc5..bb2090c881fd 100644 --- a/doc/manual/expressions/language-values.xml +++ b/doc/manual/expressions/language-values.xml @@ -98,13 +98,17 @@ configureFlags = " <para>Since <literal>${</literal> and <literal>''</literal> have special meaning in indented strings, you need a way to quote them. - <literal>${</literal> can be escaped by prefixing it with + <literal>$</literal> can be escaped by prefixing it with <literal>''</literal> (that is, two single quotes), i.e., - <literal>''${</literal>. <literal>''</literal> can be escaped by + <literal>''$</literal>. <literal>''</literal> can be escaped by prefixing it with <literal>'</literal>, i.e., - <literal>'''</literal>. Finally, linefeed, carriage-return and - tab characters can be written as <literal>''\n</literal>, - <literal>''\r</literal>, <literal>''\t</literal>.</para> + <literal>'''</literal>. <literal>$</literal> removes any special meaning + from the following <literal>$</literal>. Linefeed, carriage-return and tab + characters can be written as <literal>''\n</literal>, + <literal>''\r</literal>, <literal>''\t</literal>, and <literal>''\</literal> + escapes any other character. + + </para> <para>Indented strings are primarily useful in that they allow multi-line string literals to follow the indentation of the diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index 7311e4885e74..49660c36e397 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -10,14 +10,14 @@ <listitem><para>GNU Make.</para></listitem> - <listitem><para>A version of GCC or Clang that supports C++11.</para></listitem> + <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem> <listitem><para><command>pkg-config</command> to locate dependencies. If your distribution does not provide it, you can get it from <link xlink:href="http://www.freedesktop.org/wiki/Software/pkg-config" />.</para></listitem> - + <listitem><para>The OpenSSL library to calculate cryptographic hashes. If your distribution does not provide it, you can get it from <link xlink:href="https://www.openssl.org"/>.</para></listitem> diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml index e15296965308..7c9a8b75ecee 100644 --- a/doc/manual/release-notes/rl-1.12.xml +++ b/doc/manual/release-notes/rl-1.12.xml @@ -52,8 +52,7 @@ information about store paths.</para></listitem> <listitem><para><command>nix run</command> starts a shell in - which the specified packages are available. (TODO: currently - broken in chroot mode)</para></listitem> + which the specified packages are available.</para></listitem> <listitem><para><command>nix log</command> shows the build log of a package or path. If the build log is not available locally, @@ -74,6 +73,17 @@ name, package name or description. It caches available packages to speed up searches.</para></listitem> + <listitem><para><command>nix why-depends</command> (d41c5eb13f4f3a37d80dbc6d3888644170c3b44a).</para></listitem> + + <listitem><para><command>nix show-derivation</command> (e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2).</para></listitem> + + <listitem><para><command>nix add-to-store</command> (970366266b8df712f5f9cedb45af183ef5a8357f).</para></listitem> + + <listitem><para>Progress indicator.</para></listitem> + + <listitem><para>All options are available as flags now + (b8283773bd64d7da6859ed520ee19867742a03ba).</para></listitem> + </itemizedlist> </listitem> @@ -84,10 +94,6 @@ </listitem> <listitem> - <para>Nix now supports floating-point numbers.</para> - </listitem> - - <listitem> <para>New build mode <command>nix-build --hash</command> that builds a derivation, computes the hash of the output, and moves the output to the store path corresponding to what a fixed-output @@ -172,18 +178,13 @@ </listitem> <listitem> - <para>“Nested” log output was removed. As a result, - <command>nix-log2xml</command> was also removed.</para> - </listitem> - - <listitem> <para>When <option>--no-build-output</option> is given, the last 10 lines of the build log will be shown if a build fails.</para> </listitem> <listitem> - <para><function>builtins.fetchgit</function>. + <para><function>builtins.fetchGit</function>. (38539b943a060d9cdfc24d6e5d997c0885b8aa2f)</para> </listitem> @@ -195,10 +196,6 @@ </listitem> <listitem> - <para>Removed OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para> - </listitem> - - <listitem> <para><literal><nix/fetchurl.nix></literal> now uses the content-addressable tarball cache at <uri>http://tarballs.nixos.org/</uri>, just like @@ -226,8 +223,7 @@ <para><function>builtins.fetchurl</function> and <function>builtins.fetchTarball</function> now support <varname>sha256</varname> and <varname>name</varname> - attributes. Using these functions is now allowed in restricted - mode if a hash is supplied.</para> + attributes.</para> </listitem> <listitem> @@ -252,21 +248,11 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev" </listitem> <listitem> - <para>Added builtin function - <varname>builtins.partition</varname>.</para> - </listitem> - - <listitem> <para>Support for HTTP/2. This makes binary cache lookups much more efficient. (90ad02bf626b885a5dd8967894e2eafc953bdf92)</para> </listitem> <listitem> - <para>Caching of failed builds has been - removed. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para> - </listitem> - - <listitem> <para>The <option>build-sandbox-paths</option> configuration option can now specify optional paths by appending a <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will @@ -373,7 +359,7 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev" </listitem> <listitem> - <para><command>nix-shell</command> now used + <para><command>nix-shell</command> now uses <varname>bashInteractive</varname> from Nixpkgs, rather than the <command>bash</command> command that happens to be in the caller’s <envar>PATH</envar>. This is especially important on macOS where @@ -383,28 +369,61 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev" </listitem> <listitem> - <para><function>builtins.split</function>. - (b8867a0239b1930a16f9ef3f7f3e864b01416dff))</para> + <para>New builtin functions: <function>builtins.split</function> + (b8867a0239b1930a16f9ef3f7f3e864b01416dff), + <function>builtins.partition</function>.</para> </listitem> <listitem> - <para><filename>nix-mode.el</filename> has been removed from Nix. It is now a separate repository - in <uri>https://github.com/NixOS/nix-mode</uri> and can be installed through the MELPA - package repository.</para> + <para>Automatic garbage collection.</para> </listitem> <listitem> - <para>Automatic garbage collection.</para> + <para><command>nix-store -q --roots</command> and + <command>nix-store --gc --print-roots</command> now show temporary + and in-memory roots.</para> </listitem> <listitem> - <para><command>nix why-depends</command></para> + <para>Builders can now communicate what build phase they are in by + writing messages to the file descriptor specified in + <envar>NIX_LOG_FD</envar>. (88e6bb76de5564b3217be9688677d1c89101b2a3) + </para> </listitem> +</itemizedlist> + +<para>Some features were removed:</para> + +<itemizedlist> + <listitem> - <para><command>nix-store -q --roots</command> and - <command>nix-store --gc --print-roots</command> now show temporary - and in-memory roots.</para> + <para>“Nested” log output. As a result, + <command>nix-log2xml</command> was also removed.</para> + </listitem> + + <listitem> + <para>OpenSSL-based signing. (f435f8247553656774dd1b2c88e9de5d59cab203)</para> + </listitem> + + <listitem> + <para>Caching of failed + builds. (8cffec84859cec8b610a2a22ab0c4d462a9351ff)</para> + </listitem> + + <listitem> + <para><filename>nix-mode.el</filename> has been removed from + Nix. It is now a separate repository in + <uri>https://github.com/NixOS/nix-mode</uri> and can be installed + through the MELPA package repository.</para> + </listitem> + + <listitem> + <para>In restricted evaluation mode + (<option>--restrict-eval</option>), builtin functions that + download from the network (such as <function>fetchGit</function>) + are permitted to fetch underneath the list of URI prefixes + specified in the option <option>allowed-uris</option>.</para> </listitem> </itemizedlist> diff --git a/nix.spec.in b/nix.spec.in index 241e28f3678b..4e7dd90ea68c 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -140,6 +140,7 @@ systemctl start nix-daemon.socket %{_mandir}/man5/*.5* %{_mandir}/man8/*.8* %config(noreplace) %{_sysconfdir}/profile.d/nix.sh +%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh /nix %files devel diff --git a/scripts/local.mk b/scripts/local.mk index 9524baf81b03..2a00558521b2 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -8,5 +8,6 @@ noinst-scripts += $(nix_noinst_scripts) profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644)) clean-files += $(nix_noinst_scripts) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in new file mode 100644 index 000000000000..43c7156062de --- /dev/null +++ b/scripts/nix-profile-daemon.sh.in @@ -0,0 +1,54 @@ +# Only execute this file once per shell. +if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi +__ETC_PROFILE_NIX_SOURCED=1 + +# Set up secure multi-user builds: non-root users build through the +# Nix daemon. +if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then + export NIX_REMOTE=daemon +fi + +export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" +export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set up the per-user profile. +mkdir -m 0755 -p $NIX_USER_PROFILE_DIR +if ! test -O "$NIX_USER_PROFILE_DIR"; then + echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2 +fi + +if test -w $HOME; then + if ! test -L $HOME/.nix-profile; then + if test "$USER" != root; then + ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile + else + # Root installs in the system-wide profile by default. + ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile + fi + fi + + # Subscribe the root user to the NixOS channel by default. + if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels + fi + + # Create the per-user garbage collector roots directory. + NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER + mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR + if ! test -O "$NIX_USER_GCROOTS_DIR"; then + echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2 + fi + + # Set up a default Nix expression from which to install stuff. + if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then + rm -f $HOME/.nix-defexpr + mkdir -p $HOME/.nix-defexpr + if [ "$USER" != root ]; then + ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root + fi + fi +fi + +export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" +export NIX_PATH="@localstatedir@/nix/profiles/per-user/root/channels" +export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH" diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 4b7a24d03e9a..6e05e165545d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -16,6 +16,7 @@ #include "serialise.hh" #include "store-api.hh" #include "derivations.hh" +#include "local-store.hh" using namespace nix; using std::cin; @@ -41,24 +42,33 @@ int main (int argc, char * * argv) return handleExceptions(argv[0], [&]() { initNix(); + logger = makeJSONLogger(*logger); + /* Ensure we don't get any SSH passphrase or host key popups. */ unsetenv("DISPLAY"); unsetenv("SSH_ASKPASS"); - if (argc != 6) + if (argc != 2) throw UsageError("called without required arguments"); - auto store = openStore(); + verbosity = (Verbosity) std::stoll(argv[1]); + + FdSource source(STDIN_FILENO); + + /* Read the parent's settings. */ + while (readInt(source)) { + auto name = readString(source); + auto value = readString(source); + settings.set(name, value); + } + + settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work - auto localSystem = argv[1]; - settings.maxSilentTime = std::stoll(argv[2]); - settings.buildTimeout = std::stoll(argv[3]); - verbosity = (Verbosity) std::stoll(argv[4]); - settings.builders = argv[5]; + auto store = openStore().cast<LocalStore>(); /* It would be more appropriate to use $XDG_RUNTIME_DIR, since that gets cleared on reboot, but it wouldn't work on macOS. */ - currentLoad = settings.nixStateDir + "/current-load"; + currentLoad = store->stateDir + "/current-load"; std::shared_ptr<Store> sshStore; AutoCloseFD bestSlotLock; @@ -73,18 +83,20 @@ int main (int argc, char * * argv) string drvPath; string storeUri; - for (string line; getline(cin, line);) { - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz != 3 && sz != 4) - throw Error("invalid build hook line '%1%'", line); - auto amWilling = tokens[0] == "1"; - auto neededSystem = tokens[1]; - drvPath = tokens[2]; - auto requiredFeatures = sz == 3 ? - std::set<string>{} : - tokenizeString<std::set<string>>(tokens[3], ","); - auto canBuildLocally = amWilling && (neededSystem == localSystem); + + while (true) { + + try { + auto s = readString(source); + if (s != "try") return; + } catch (EndOfFile &) { return; } + + auto amWilling = readInt(source); + auto neededSystem = readString(source); + source >> drvPath; + auto requiredFeatures = readStrings<std::set<std::string>>(source); + + auto canBuildLocally = amWilling && (neededSystem == settings.thisSystem); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -99,7 +111,7 @@ int main (int argc, char * * argv) Machine * bestMachine = nullptr; unsigned long long bestLoad = 0; for (auto & m : machines) { - debug("considering building on '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri); if (m.enabled && std::find(m.systemTypes.begin(), m.systemTypes.end(), @@ -162,9 +174,15 @@ int main (int argc, char * * argv) try { - Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}}; - if (bestMachine->sshKey != "") - storeParams["ssh-key"] = bestMachine->sshKey; + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + + Store::Params storeParams; + if (hasPrefix(storeUri, "ssh://")) { + storeParams["max-connections"] ="1"; + storeParams["log-fd"] = "4"; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + } sshStore = openStore(bestMachine->storeUri, storeParams); sshStore->connect(); @@ -182,32 +200,34 @@ int main (int argc, char * * argv) } connected: - std::cerr << "# accept\n"; - string line; - if (!getline(cin, line)) - throw Error("hook caller didn't send inputs"); + std::cerr << "# accept\n" << storeUri << "\n"; - auto inputs = tokenizeString<PathSet>(line); - if (!getline(cin, line)) - throw Error("hook caller didn't send outputs"); - - auto outputs = tokenizeString<PathSet>(line); + auto inputs = readStrings<PathSet>(source); + auto outputs = readStrings<PathSet>(source); AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); - auto old = signal(SIGALRM, handleAlarm); - alarm(15 * 60); - if (!lockFile(uploadLock.get(), ltWrite, true)) - printError("somebody is hogging the upload lock for '%s', continuing..."); - alarm(0); - signal(SIGALRM, old); - copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs); + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); + + auto old = signal(SIGALRM, handleAlarm); + alarm(15 * 60); + if (!lockFile(uploadLock.get(), ltWrite, true)) + printError("somebody is hogging the upload lock for '%s', continuing..."); + alarm(0); + signal(SIGALRM, old); + } + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); + copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs); + } + uploadLock = -1; - BasicDerivation drv(readDerivation(drvPath)); + BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); drv.inputSrcs = inputs; - printError("building '%s' on '%s'", drvPath, storeUri); auto result = sshStore->buildDerivation(drvPath, drv); if (!result.success()) @@ -218,6 +238,7 @@ connected: if (!store->isValidPath(path)) missing.insert(path); if (!missing.empty()) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs); } diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc new file mode 100644 index 000000000000..3e0c78f280f7 --- /dev/null +++ b/src/libexpr/common-eval-args.cc @@ -0,0 +1,57 @@ +#include "common-eval-args.hh" +#include "shared.hh" +#include "download.hh" +#include "util.hh" +#include "eval.hh" + +namespace nix { + +MixEvalArgs::MixEvalArgs() +{ + mkFlag() + .longName("arg") + .description("argument to be passed to Nix functions") + .labels({"name", "expr"}) + .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); + + mkFlag() + .longName("argstr") + .description("string-valued argument to be passed to Nix functions") + .labels({"name", "string"}) + .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); + + mkFlag() + .shortName('I') + .longName("include") + .description("add a path to the list of locations used to look up <...> file names") + .label("path") + .handler([&](std::string s) { searchPath.push_back(s); }); +} + +Bindings * MixEvalArgs::getAutoArgs(EvalState & state) +{ + Bindings * res = state.allocBindings(autoArgs.size()); + for (auto & i : autoArgs) { + Value * v = state.allocValue(); + if (i.second[0] == 'E') + state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); + else + mkString(*v, string(i.second, 1)); + res->push_back(Attr(state.symbols.create(i.first), v)); + } + res->sort(); + return res; +} + +Path lookupFileArg(EvalState & state, string s) +{ + if (isUri(s)) + return getDownloader()->downloadCached(state.store, s, true); + else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + return state.findFile(p); + } else + return absPath(s); +} + +} diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh new file mode 100644 index 000000000000..09fa406b2cdc --- /dev/null +++ b/src/libexpr/common-eval-args.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +class Store; +class EvalState; +struct Bindings; + +struct MixEvalArgs : virtual Args +{ + MixEvalArgs(); + + Bindings * getAutoArgs(EvalState & state); + + Strings searchPath; + +private: + + std::map<std::string, std::string> autoArgs; +}; + +Path lookupFileArg(EvalState & state, string s); + +} diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc deleted file mode 100644 index 6b31961d345b..000000000000 --- a/src/libexpr/common-opts.cc +++ /dev/null @@ -1,67 +0,0 @@ -#include "common-opts.hh" -#include "shared.hh" -#include "download.hh" -#include "util.hh" - - -namespace nix { - - -bool parseAutoArgs(Strings::iterator & i, - const Strings::iterator & argsEnd, std::map<string, string> & res) -{ - string arg = *i; - if (arg != "--arg" && arg != "--argstr") return false; - - UsageError error(format("'%1%' requires two arguments") % arg); - - if (++i == argsEnd) throw error; - string name = *i; - if (++i == argsEnd) throw error; - string value = *i; - - res[name] = (arg == "--arg" ? 'E' : 'S') + value; - - return true; -} - - -Bindings * evalAutoArgs(EvalState & state, std::map<string, string> & in) -{ - Bindings * res = state.allocBindings(in.size()); - for (auto & i : in) { - Value * v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); - else - mkString(*v, string(i.second, 1)); - res->push_back(Attr(state.symbols.create(i.first), v)); - } - res->sort(); - return res; -} - - -bool parseSearchPathArg(Strings::iterator & i, - const Strings::iterator & argsEnd, Strings & searchPath) -{ - if (*i != "-I") return false; - if (++i == argsEnd) throw UsageError("'-I' requires an argument"); - searchPath.push_back(*i); - return true; -} - - -Path lookupFileArg(EvalState & state, string s) -{ - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { - Path p = s.substr(1, s.size() - 2); - return state.findFile(p); - } else - return absPath(s); -} - - -} diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh deleted file mode 100644 index cb2732d6fe7e..000000000000 --- a/src/libexpr/common-opts.hh +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include "eval.hh" - -namespace nix { - -class Store; - -/* Some common option parsing between nix-env and nix-instantiate. */ -bool parseAutoArgs(Strings::iterator & i, - const Strings::iterator & argsEnd, std::map<string, string> & res); - -Bindings * evalAutoArgs(EvalState & state, std::map<string, string> & in); - -bool parseSearchPathArg(Strings::iterator & i, - const Strings::iterator & argsEnd, Strings & searchPath); - -Path lookupFileArg(EvalState & state, string s); - -} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 78f6b0010523..63de2d60a147 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -149,7 +149,7 @@ string showType(const Value & v) switch (v.type) { case tInt: return "an integer"; case tBool: return "a boolean"; - case tString: return "a string"; + case tString: return v.string.context ? "a string with context" : "a string"; case tPath: return "a path"; case tNull: return "null"; case tAttrs: return "a set"; @@ -355,6 +355,26 @@ Path EvalState::checkSourcePath(const Path & path_) } +void EvalState::checkURI(const std::string & uri) +{ + if (!restricted) return; + + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. Note: this allows 'http://' and + 'https://' as prefixes for any http/https URI. */ + for (auto & prefix : settings.allowedUris.get()) + if (uri == prefix || + (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + return; + + throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); +} + + void EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 04a36b14cefa..f0ab1435bff3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -110,6 +110,8 @@ public: Path checkSourcePath(const Path & path); + void checkURI(const std::string & uri); + /* Parse a Nix expression from the specified file. */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 28a0a6a87896..828356bbf447 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -90,7 +90,7 @@ FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> -URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ +URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:\/\/[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+|channel\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ %% diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 669312bb7cff..eee31522830f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -667,7 +667,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl try { if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git")) // FIXME: support specifying revision/branch - res = { true, exportGit(store, elem.second, "master") }; + res = { true, exportGit(store, elem.second, "master").storePath }; else res = { true, getDownloader()->downloadCached(store, elem.second, true) }; } catch (DownloadError & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fcd3f8efee3f..cd0dfbc03e94 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -713,7 +713,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); - drv.env["out"] = outPath; + if (!jsonObject) drv.env["out"] = outPath; drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); } @@ -724,7 +724,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * an empty value. This ensures that changes in the set of output names do get reflected in the hash. */ for (auto & i : outputs) { - drv.env[i] = ""; + if (!jsonObject) drv.env[i] = ""; drv.outputs[i] = DerivationOutput("", "", ""); } @@ -735,7 +735,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : drv.outputs) if (i.second.path == "") { Path outPath = state.store->makeOutputPath(i.first, h, drvName); - drv.env[i.first] = outPath; + if (!jsonObject) drv.env[i.first] = outPath; i.second.path = outPath; } } @@ -1907,11 +1907,11 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack) + const string & who, bool unpack, const std::string & defaultName) { string url; Hash expectedHash; - string name; + string name = defaultName; state.forceValue(*args[0]); @@ -1937,8 +1937,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, } else url = state.forceStringNoCtx(*args[0], pos); - if (state.restricted && !expectedHash) - throw Error(format("'%1%' is not allowed in restricted mode") % who); + state.checkURI(url); Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); mkString(v, res, PathSet({res})); @@ -1947,13 +1946,13 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) { - fetch(state, pos, args, v, "fetchurl", false); + fetch(state, pos, args, v, "fetchurl", false, ""); } static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) { - fetch(state, pos, args, v, "fetchTarball", true); + fetch(state, pos, args, v, "fetchTarball", true, "source"); } @@ -2008,7 +2007,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 4); + mkInt(v, 5); addConstant("__langVersion", v); // Miscellaneous diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc index e16c8235378d..4af5301247bc 100644 --- a/src/libexpr/primops/fetchgit.cc +++ b/src/libexpr/primops/fetchgit.cc @@ -1,3 +1,4 @@ +#include "fetchgit.hh" #include "primops.hh" #include "eval-inline.hh" #include "download.hh" @@ -8,25 +9,22 @@ #include <regex> +#include <nlohmann/json.hpp> + +using namespace std::string_literals; + namespace nix { -Path exportGit(ref<Store> store, const std::string & uri, - const std::string & ref, const std::string & rev) +GitInfo exportGit(ref<Store> store, const std::string & uri, + const std::string & ref, const std::string & rev, + const std::string & name) { - if (!isUri(uri)) - throw EvalError(format("'%s' is not a valid URI") % uri); - if (rev != "") { std::regex revRegex("^[0-9a-fA-F]{40}$"); if (!std::regex_match(rev, revRegex)) throw Error("invalid Git revision '%s'", rev); } - // FIXME: too restrictive, but better safe than sorry. - std::regex refRegex("^[0-9a-zA-Z][0-9a-zA-Z.-]+$"); - if (!std::regex_match(ref, refRegex)) - throw Error("invalid Git ref '%s'", ref); - Path cacheDir = getCacheDir() + "/nix/git"; if (!pathExists(cacheDir)) { @@ -34,8 +32,6 @@ Path exportGit(ref<Store> store, const std::string & uri, runProgram("git", true, { "init", "--bare", cacheDir }); } - //Activity act(*logger, lvlInfo, format("fetching Git repository '%s'") % uri); - std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, ref)).to_string(Base32, false); Path localRefFile = cacheDir + "/refs/heads/" + localRef; @@ -47,7 +43,11 @@ Path exportGit(ref<Store> store, const std::string & uri, if (stat(localRefFile.c_str(), &st) != 0 || st.st_mtime < now - settings.tarballTtl) { - runProgram("git", true, { "-C", cacheDir, "fetch", "--force", uri, ref + ":" + localRef }); + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, ref + ":" + localRef }); struct timeval times[2]; times[0].tv_sec = now; @@ -59,46 +59,65 @@ Path exportGit(ref<Store> store, const std::string & uri, } // FIXME: check whether rev is an ancestor of ref. - std::string commitHash = - rev != "" ? rev : chomp(readFile(localRefFile)); + GitInfo gitInfo; + gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - printTalkative("using revision %s of repo '%s'", uri, commitHash); + printTalkative("using revision %s of repo '%s'", uri, gitInfo.rev); - Path storeLink = cacheDir + "/" + commitHash + ".link"; + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); - if (pathExists(storeLink)) { - auto storePath = readLink(storeLink); - store->addTempRoot(storePath); - if (store->isValidPath(storePath)) { - return storePath; + try { + // FIXME: doesn't handle empty lines + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == gitInfo.rev); + + gitInfo.storePath = json["storePath"]; + + if (store->isValidPath(gitInfo.storePath)) { + gitInfo.revCount = json["revCount"]; + return gitInfo; } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; } // FIXME: should pipe this, or find some better way to extract a // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", commitHash }); + auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); runProgram("tar", true, { "x", "-C", tmpDir }, tar); - auto storePath = store->addToStore("git-export", tmpDir); + gitInfo.storePath = store->addToStore(name, tmpDir); + + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); + + nlohmann::json json; + json["storePath"] = gitInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["rev"] = gitInfo.rev; + json["revCount"] = gitInfo.revCount; - replaceSymlink(storePath, storeLink); + writeFile(storeLink, json.dump()); - return storePath; + return gitInfo; } -static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { - // FIXME: cut&paste from fetch(). - if (state.restricted) throw Error("'fetchgit' is not allowed in restricted mode"); - std::string url; std::string ref = "master"; std::string rev; + std::string name = "source"; + PathSet context; state.forceValue(*args[0]); @@ -107,31 +126,41 @@ static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Va state.forceAttrs(*args[0], pos); for (auto & attr : *args[0]->attrs) { - string name(attr.name); - if (name == "url") { - PathSet context; + string n(attr.name); + if (n == "url") url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - if (hasPrefix(url, "/")) url = "file://" + url; - } - else if (name == "ref") + else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (name == "rev") + else if (n == "rev") rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%s' to 'fetchgit', at %s", attr.name, *attr.pos); + throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); } if (url.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + url = state.coerceToString(pos, *args[0], context, false, false); + + if (hasPrefix(url, "/")) url = "file://" + url; + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); - Path storePath = exportGit(state.store, url, ref, rev); + auto gitInfo = exportGit(state.store, url, ref, rev, name); - mkString(v, storePath, PathSet({storePath})); + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + v.attrs->sort(); } -static RegisterPrimOp r("__fetchgit", 1, prim_fetchgit); +static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); } diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh index ff228f3b3c6a..056b6fcbe78d 100644 --- a/src/libexpr/primops/fetchgit.hh +++ b/src/libexpr/primops/fetchgit.hh @@ -2,13 +2,22 @@ #include <string> -#include "ref.hh" +#include "util.hh" namespace nix { class Store; -Path exportGit(ref<Store> store, const std::string & uri, - const std::string & ref, const std::string & rev = ""); +struct GitInfo +{ + Path storePath; + std::string rev; + std::string shortRev; + uint64_t revCount = 0; +}; + +GitInfo exportGit(ref<Store> store, const std::string & uri, + const std::string & ref, const std::string & rev = "", + const std::string & name = ""); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3fa42c2aafa9..ea27aaa35e03 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -6,28 +6,30 @@ namespace nix { MixCommonArgs::MixCommonArgs(const string & programName) : programName(programName) { - mkFlag('v', "verbose", "increase verbosity level", []() { - verbosity = (Verbosity) (verbosity + 1); - }); + mkFlag() + .longName("verbose") + .shortName('v') + .description("increase verbosity level") + .handler([]() { verbosity = (Verbosity) (verbosity + 1); }); - mkFlag(0, "quiet", "decrease verbosity level", []() { - verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; - }); + mkFlag() + .longName("quiet") + .description("decrease verbosity level") + .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); - mkFlag(0, "debug", "enable debug output", []() { - verbosity = lvlDebug; - }); + mkFlag() + .longName("debug") + .description("enable debug output") + .handler([]() { verbosity = lvlDebug; }); mkFlag() .longName("option") .labels({"name", "value"}) .description("set a Nix configuration option (overriding nix.conf)") .arity(2) - .handler([](Strings ss) { - auto name = ss.front(); ss.pop_front(); - auto value = ss.front(); + .handler([](std::vector<std::string> ss) { try { - settings.set(name, value); + settings.set(ss[0], ss[1]); } catch (UsageError & e) { warn(e.what()); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6393d80bbf00..0f599f388585 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,4 +1,3 @@ -#include "common-args.hh" #include "globals.hh" #include "shared.hh" #include "store-api.hh" @@ -149,74 +148,84 @@ void initNix() } -struct LegacyArgs : public MixCommonArgs +LegacyArgs::LegacyArgs(const std::string & programName, + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) + : MixCommonArgs(programName), parseArg(parseArg) { - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; - - LegacyArgs(const std::string & programName, - std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) - : MixCommonArgs(programName), parseArg(parseArg) - { - mkFlag('Q', "no-build-output", "do not show build output", - &settings.verboseBuild, false); - - mkFlag('K', "keep-failed", "keep temporary directories of failed builds", - &(bool&) settings.keepFailed); - - mkFlag('k', "keep-going", "keep going after a build fails", - &(bool&) settings.keepGoing); + mkFlag() + .longName("no-build-output") + .shortName('Q') + .description("do not show build output") + .set(&settings.verboseBuild, false); + + mkFlag() + .longName("keep-failed") + .shortName('K') + .description("keep temporary directories of failed builds") + .set(&(bool&) settings.keepFailed, true); + + mkFlag() + .longName("keep-going") + .shortName('k') + .description("keep going after a build fails") + .set(&(bool&) settings.keepGoing, true); + + mkFlag() + .longName("fallback") + .description("build from source if substitution fails") + .set(&(bool&) settings.tryFallback, true); + + mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { + settings.set("max-jobs", s); + }); - mkFlag(0, "fallback", "build from source if substitution fails", []() { - settings.tryFallback = true; + auto intSettingAlias = [&](char shortName, const std::string & longName, + const std::string & description, const std::string & dest) { + mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); }); + }; - mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { - settings.set("max-jobs", s); - }); + intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); + intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); - auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); - }); - }; + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", + &settings.showTrace); - mkFlag(0, "readonly-mode", "do not write to the Nix store", - &settings.readOnlyMode); + mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + &gcWarning, false); - mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", - &(bool&) settings.useBuildHook, false); + mkFlag() + .longName("store") + .label("store-uri") + .description("URI of the Nix store to use") + .dest(&(std::string&) settings.storeUri); +} - mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", - &settings.showTrace); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", - &gcWarning, false); - } +bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; +} - bool processFlag(Strings::iterator & pos, Strings::iterator end) override - { - if (MixCommonArgs::processFlag(pos, end)) return true; - bool res = parseArg(pos, end); - if (res) ++pos; - return res; - } - bool processArgs(const Strings & args, bool finish) override - { - if (args.empty()) return true; - assert(args.size() == 1); - Strings ss(args); - auto pos = ss.begin(); - if (!parseArg(pos, ss.end())) - throw UsageError(format("unexpected argument '%1%'") % args.front()); - return true; - } -}; +bool LegacyArgs::processArgs(const Strings & args, bool finish) +{ + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError(format("unexpected argument '%1%'") % args.front()); + return true; +} void parseCmdLine(int argc, char * * argv, diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 2a1e42dd9774..9219dbed8325 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -2,6 +2,7 @@ #include "util.hh" #include "args.hh" +#include "common-args.hh" #include <signal.h> @@ -69,6 +70,19 @@ template<class N> N getIntArg(const string & opt, } +struct LegacyArgs : public MixCommonArgs +{ + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; + + LegacyArgs(const std::string & programName, + std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg); + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + + /* Show the manual page for the specified program. */ void showManPage(const string & name); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 556fa3d59355..67607ab3d43a 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -17,66 +17,6 @@ namespace nix { -/* Given requests for a path /nix/store/<x>/<y>, this accessor will - first download the NAR for /nix/store/<x> from the binary cache, - build a NAR accessor for that NAR, and use that to access <y>. */ -struct BinaryCacheStoreAccessor : public FSAccessor -{ - ref<BinaryCacheStore> store; - - std::map<Path, ref<FSAccessor>> nars; - - BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) - : store(store) - { - } - - std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) - { - auto path = canonPath(path_); - - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); - - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); - - auto i = nars.find(storePath); - if (i != nars.end()) return {i->second, restPath}; - - StringSink sink; - store->narFromPath(storePath, sink); - - auto accessor = makeNarAccessor(sink.s); - nars.emplace(storePath, accessor); - return {accessor, restPath}; - } - - Stat stat(const Path & path) override - { - auto res = fetch(path); - return res.first->stat(res.second); - } - - StringSet readDirectory(const Path & path) override - { - auto res = fetch(path); - return res.first->readDirectory(res.second); - } - - std::string readFile(const Path & path) override - { - auto res = fetch(path); - return res.first->readFile(res.second); - } - - std::string readLink(const Path & path) override - { - auto res = fetch(path); - return res.first->readLink(res.second); - } -}; - BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) { @@ -161,7 +101,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); - auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); + auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -174,8 +114,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str auto narAccessor = makeNarAccessor(nar); - if (accessor_) + if (accessor_) { accessor_->nars.emplace(info.path, narAccessor); + accessor_->addToCache(info.path, *nar); + } std::function<void(const Path &, JSONPlaceholder &)> recurse; @@ -220,8 +162,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } else { - if (accessor_) + if (accessor_) { accessor_->nars.emplace(info.path, makeNarAccessor(nar)); + accessor_->addToCache(info.path, *nar); + } } /* Compress the NAR. */ @@ -379,7 +323,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, ref<FSAccessor> BinaryCacheStore::getFSAccessor() { - return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); } std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index f9c1c2cbe8a8..d3b0e0bd9332 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -18,6 +18,7 @@ public: const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; private: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9069d9b06e08..061682377257 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -18,6 +18,7 @@ #include <thread> #include <future> #include <chrono> +#include <regex> #include <limits.h> #include <sys/time.h> @@ -274,6 +275,10 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + Worker(LocalStore & store); ~Worker(); @@ -606,6 +611,10 @@ struct HookInstance /* The process ID of the hook. */ Pid pid; + FdSink sink; + + std::map<ActivityId, Activity> activities; + HookInstance(); ~HookInstance(); @@ -642,11 +651,7 @@ HookInstance::HookInstance() Strings args = { baseNameOf(settings.buildHook), - settings.thisSystem, - std::to_string(settings.maxSilentTime), - std::to_string(settings.buildTimeout), std::to_string(verbosity), - settings.builders }; execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); @@ -657,6 +662,11 @@ HookInstance::HookInstance() pid.setSeparatePG(true); fromHook.writeSide = -1; toHook.readSide = -1; + + sink = FdSink(toHook.writeSide.get()); + for (auto & setting : settings.getSettings()) + sink << 1 << setting.first << setting.second; + sink << 0; } @@ -762,6 +772,8 @@ private: std::string currentLogLine; size_t currentLogLinePos = 0; // to handle carriage return + std::string currentHookLine; + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -843,6 +855,9 @@ private: std::map<ActivityId, Activity> builderActivities; + /* The remote machine on which we're building. */ + std::string machineName; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -899,9 +914,6 @@ private: /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); - /* Handle the exportReferencesGraph attribute. */ - void doExportReferencesGraph(); - /* Run the builder's process. */ void runChild(); @@ -1390,8 +1402,15 @@ void DerivationGoal::tryToBuild() bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); auto started = [&]() { - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, - fmt("building '%s'", drvPath), Logger::Fields{drvPath}); + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", drvPath, curRound, nrRounds); + fmt("building '%s'", drvPath); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, + Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); }; @@ -1619,7 +1638,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || !useDerivation) return rpDecline; + if (!worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1633,21 +1652,29 @@ HookReply DerivationGoal::tryBuildHook() for (auto & i : features) checkStoreName(i); /* !!! abuse */ /* Send the request to the hook. */ - writeLine(worker.hook->toHook.writeSide.get(), (format("%1% %2% %3% %4%") - % (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0") - % drv->platform % drvPath % concatStringsSep(",", features)).str()); + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << drvPath + << features; + worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ string reply; while (true) { string s = readLine(worker.hook->fromHook.readSide.get()); - if (string(s, 0, 2) == "# ") { + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { reply = string(s, 2); break; } - s += "\n"; - writeToStderr(s); + else { + s += "\n"; + writeToStderr(s); + } } debug(format("hook reply is '%1%'") % reply); @@ -1655,7 +1682,7 @@ HookReply DerivationGoal::tryBuildHook() if (reply == "decline") return rpDecline; else if (reply == "decline-permanently") { - settings.useBuildHook = false; + worker.tryBuildHook = false; worker.hook = 0; return rpDecline; } @@ -1674,18 +1701,19 @@ HookReply DerivationGoal::tryBuildHook() throw; } - printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - hook = std::move(worker.hook); + machineName = readLine(hook->fromHook.readSide.get()); + /* Tell the hook all the inputs that have to be copied to the remote system. */ - writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); + hook->sink << inputPaths; /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); + hook->sink << missingPaths; + hook->sink = FdSink(); hook->toHook.writeSide = -1; /* Create the log file and pipe. */ @@ -1714,16 +1742,44 @@ int childEntry(void * arg) } -void DerivationGoal::startBuilder() +PathSet exportReferences(Store & store, PathSet storePaths) { - auto f = format( - buildMode == bmRepair ? "repairing path(s) %1%" : - buildMode == bmCheck ? "checking path(s) %1%" : - nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : - "building path(s) %1%"); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - printInfo(f % showPaths(missingPaths) % curRound % nrRounds); + PathSet paths; + + for (auto storePath : storePaths) { + /* Check that the store path is valid. */ + if (!store.isInStore(storePath)) + throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") + % storePath); + storePath = store.toStorePath(storePath); + if (!store.isValidPath(storePath)) + throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") + % storePath); + + store.computeFSClosure(storePath, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths2(paths); + + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = store.derivationFromPath(j); + for (auto & k : drv.outputs) + store.computeFSClosure(k.second.path, paths); + } + } + + return paths; +} + + +void DerivationGoal::startBuilder() +{ /* Right platform? */ if (!drv->canBuildLocally()) { throw Error( @@ -1797,7 +1853,29 @@ void DerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - doExportReferencesGraph(); + if (!drv->env.count("__json")) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + Path storePath = *i++; + + /* Write closure info to <fileName>. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences(worker.store, {storePath}), false, false)); + } + } if (useChroot) { @@ -2266,88 +2344,127 @@ void DerivationGoal::initEnv() } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + void DerivationGoal::writeStructuredAttrs() { - auto json = drv->env.find("__json"); - if (json == drv->env.end()) return; + auto jsonAttr = drv->env.find("__json"); + if (jsonAttr == drv->env.end()) return; - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites)); -} + try { + auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); -void DerivationGoal::chownToBuilder(const Path & path) -{ - if (!buildUser) return; - if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); -} + auto json = nlohmann::json::parse(jsonStr); + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(worker.store, storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + writeFile(tmpDir + "/.attrs.json", json.dump()); -void DerivationGoal::doExportReferencesGraph() -{ - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); - Strings ss = tokenizeString<Strings>(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - /* Check that the store path is valid. */ - Path storePath = *i++; - if (!worker.store.isInStore(storePath)) - throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") - % storePath); - storePath = worker.store.toStorePath(storePath); - if (!worker.store.isValidPath(storePath)) - throw BuildError(format("'exportReferencesGraph' contains an invalid path '%1%'") - % storePath); + auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { + if (value.is_string()) + return shellEscape(value); - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths, paths2; - worker.store.computeFSClosure(storePath, paths); - paths2 = paths; - - for (auto & j : paths2) { - if (isDerivation(j)) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); } - } - if (!drv->env.count("__json")) { + if (value.is_null()) + return std::string("''"); - /* Write closure info to <fileName>. */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration(paths, false, false)); + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - } else { + return {}; + }; - /* Write a more comprehensive JSON serialisation to - <fileName>. */ - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - worker.store.pathInfoToJSON(jsonRoot, paths, false, true); + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); } - writeFile(tmpDir + "/" + fileName, str.str()); + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } } + + writeFile(tmpDir + "/.attrs.sh", jsonSh); + + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); } } +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % path); +} + + void setupSeccomp() { #if __linux__ @@ -2402,64 +2519,6 @@ void setupSeccomp() } -struct BuilderLogger : Logger -{ - Logger & prevLogger; - - BuilderLogger(Logger & prevLogger) : prevLogger(prevLogger) { } - - void addFields(nlohmann::json & json, const Fields & fields) - { - if (fields.empty()) return; - auto & arr = json["fields"] = nlohmann::json::array(); - for (auto & f : fields) - if (f.type == Logger::Field::tInt) - arr.push_back(f.i); - else if (f.type == Logger::Field::tString) - arr.push_back(f.s); - else - abort(); - } - - void log(Verbosity lvl, const FormatOrString & fs) override - { - prevLogger.log(lvl, fs); - } - - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, - const std::string & s, const Fields & fields, ActivityId parent) override - { - nlohmann::json json; - json["action"] = "start"; - json["id"] = act; - json["level"] = lvl; - json["type"] = type; - json["text"] = s; - addFields(json, fields); - // FIXME: handle parent - log(lvlError, "@nix " + json.dump()); - } - - void stopActivity(ActivityId act) override - { - nlohmann::json json; - json["action"] = "stop"; - json["id"] = act; - log(lvlError, "@nix " + json.dump()); - } - - void result(ActivityId act, ResultType type, const Fields & fields) override - { - nlohmann::json json; - json["action"] = "result"; - json["id"] = act; - json["type"] = type; - addFields(json, fields); - log(lvlError, "@nix " + json.dump()); - } -}; - - void DerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite @@ -2868,7 +2927,7 @@ void DerivationGoal::runChild() /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { - logger = new BuilderLogger(*logger); + logger = makeJSONLogger(*logger); if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv, netrcData); else @@ -3309,8 +3368,14 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) if (logSink) (*logSink)(data); } - if (hook && fd == hook->fromHook.readSide.get()) - printError(chomp(data)); + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } } @@ -3321,56 +3386,10 @@ void DerivationGoal::handleEOF(int fd) } -static Logger::Fields getFields(nlohmann::json & json) -{ - Logger::Fields fields; - for (auto & f : json) { - if (f.type() == nlohmann::json::value_t::number_unsigned) - fields.emplace_back(Logger::Field(f.get<uint64_t>())); - else if (f.type() == nlohmann::json::value_t::string) - fields.emplace_back(Logger::Field(f.get<std::string>())); - else throw Error("unsupported JSON type %d", (int) f.type()); - } - return fields; -} - - void DerivationGoal::flushLine() { - if (hasPrefix(currentLogLine, "@nix ")) { - - try { - auto json = nlohmann::json::parse(std::string(currentLogLine, 5)); - - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (type == actDownload) - builderActivities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act->id)); - } - - else if (action == "stop") - builderActivities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = builderActivities.find((ActivityId) json["id"]); - if (i != builderActivities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act->result(resSetPhase, phase); - } - - } catch (std::exception & e) { - printError("bad log message from builder: %s", e.what()); - } - } + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; else { if (settings.verboseBuild && @@ -3683,8 +3702,6 @@ void SubstitutionGoal::tryToRun() return; } - printInfo(format("fetching path '%1%'...") % storePath); - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3992,7 +4009,7 @@ void Worker::run(const Goals & _topGoals) else { if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( "unable to start any build; either increase '--max-jobs' " - "or enable distributed builds"); + "or enable remote builds"); assert(!awake.empty()); } } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 608b8fd399b4..579a5e8c1b59 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -23,6 +23,8 @@ #include <cmath> #include <random> +using namespace std::string_literals; + namespace nix { double getTime() @@ -604,7 +606,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Path cacheDir = getCacheDir() + "/nix/tarballs"; createDirs(cacheDir); - string urlHash = hashString(htSHA256, url).to_string(Base32, false); + string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); Path dataFile = cacheDir + "/" + urlHash + ".info"; Path fileLink = cacheDir + "/" + urlHash + "-file"; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7da4bce87753..4fa02f92085a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -53,7 +53,12 @@ Settings::Settings() /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); - if (s != "") builderFiles = tokenizeString<Strings>(s, ":"); + if (s != "") { + Strings ss; + for (auto & p : tokenizeString<Strings>(s, ":")) + ss.push_back("@" + p); + builders = concatStringsSep(" ", ss); + } #if defined(__linux__) && defined(SANDBOX_SHELL) sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); @@ -111,17 +116,17 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s args.mkFlag() .longName(name) .description("Enable sandboxing.") - .handler([=](Strings ss) { value = smEnabled; }) + .handler([=](std::vector<std::string> ss) { value = smEnabled; }) .category(category); args.mkFlag() .longName("no-" + name) .description("Disable sandboxing.") - .handler([=](Strings ss) { value = smDisabled; }) + .handler([=](std::vector<std::string> ss) { value = smDisabled; }) .category(category); args.mkFlag() .longName("relaxed-" + name) .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](Strings ss) { value = smRelaxed; }) + .handler([=](std::vector<std::string> ss) { value = smRelaxed; }) .category(category); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 264e82a16e20..a4aa842d70fd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "config.hh" +#include "util.hh" #include <map> #include <limits> @@ -84,6 +85,9 @@ public: /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; + Setting<std::string> storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", + "The default Nix store to use."}; + Setting<bool> keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; @@ -128,19 +132,12 @@ public: "The maximum duration in seconds that a builder can run. " "0 means infinity.", {"build-timeout"}}; - Setting<bool> useBuildHook{this, true, "remote-builds", - "Whether to use build hooks (for distributed builds)."}; - PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", "The path of the helper program that executes builds to remote machines."}; - Setting<std::string> builders{this, "", "builders", + Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders", "A semicolon-separated list of build machines, in the format of nix.machines."}; - Setting<Strings> builderFiles{this, - {nixConfDir + "/machines"}, "builder-files", - "A list of files specifying build machines."}; - Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; @@ -228,7 +225,7 @@ public: Setting<bool> restrictEval{this, false, "restrict-eval", "Whether to restrict file system access to paths in $NIX_PATH, " - "and to disallow fetching files from the network."}; + "and network access to the URI prefixes listed in 'allowed-uris'."}; Setting<size_t> buildRepeat{this, 0, "repeat", "The number of times to repeat a build in order to verify determinism.", @@ -274,7 +271,7 @@ public: "Number of parallel HTTP connections.", {"binary-caches-parallel-connections"}}; - Setting<bool> enableHttp2{this, true, "enable-http2", + Setting<bool> enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", @@ -356,6 +353,8 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<Strings> allowedUris{this, {}, "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; }; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 076c3cab3e90..edd03d147832 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -17,7 +17,11 @@ Machine::Machine(decltype(storeUri) storeUri, storeUri( // Backwards compatibility: if the URI is a hostname, // prepend ssh://. - storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto") + storeUri.find("://") != std::string::npos + || hasPrefix(storeUri, "local") + || hasPrefix(storeUri, "remote") + || hasPrefix(storeUri, "auto") + || hasPrefix(storeUri, "/") ? storeUri : "ssh://" + storeUri), systemTypes(systemTypes), @@ -47,9 +51,22 @@ bool Machine::mandatoryMet(const std::set<string> & features) const { void parseMachines(const std::string & s, Machines & machines) { for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { - chomp(line); + trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue; + + if (line[0] == '@') { + auto file = trim(std::string(line, 1)); + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + debug("cannot find machines file '%s'", file); + } + continue; + } + auto tokens = tokenizeString<std::vector<string>>(line); auto sz = tokens.size(); if (sz < 1) @@ -74,15 +91,6 @@ Machines getMachines() { Machines machines; - for (auto & file : settings.builderFiles.get()) { - try { - parseMachines(readFile(file), machines); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - } - parseMachines(settings.builders, machines); return machines; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 098151f8c0f6..ba9620a175bb 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -3,10 +3,29 @@ namespace nix { - -RemoteFSAccessor::RemoteFSAccessor(ref<Store> store) +RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) : store(store) + , cacheDir(cacheDir) { + if (cacheDir != "") + createDirs(cacheDir); +} + +Path RemoteFSAccessor::makeCacheFile(const Path & storePath) +{ + assert(cacheDir != ""); + return fmt("%s/%s.nar", cacheDir, storePathToHash(storePath)); +} + +void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar) +{ + try { + if (cacheDir == "") return; + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(storePath), nar); + } catch (...) { + ignoreException(); + } } std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) @@ -23,7 +42,16 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) if (i != nars.end()) return {i->second, restPath}; StringSink sink; - store->narFromPath(storePath, sink); + + try { + if (cacheDir != "") + *sink.s = nix::readFile(makeCacheFile(storePath)); + } catch (SysError &) { } + + if (sink.s->empty()) { + store->narFromPath(storePath, sink); + addToCache(storePath, *sink.s); + } auto accessor = makeNarAccessor(sink.s); nars.emplace(storePath, accessor); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 28f36c8296e1..2a3fc01eff58 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -12,10 +12,20 @@ class RemoteFSAccessor : public FSAccessor std::map<Path, ref<FSAccessor>> nars; + Path cacheDir; + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_); + + friend class BinaryCacheStore; + + Path makeCacheFile(const Path & storePath); + + void addToCache(const Path & storePath, const std::string & nar); + public: - RemoteFSAccessor(ref<Store> store); + RemoteFSAccessor(ref<Store> store, + const /* FIXME: use std::optional */ Path & cacheDir = ""); Stat stat(const Path & path) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b9076c0474d6..77b41b6bf8a8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -166,7 +166,7 @@ void RemoteStore::setOptions(Connection & conn) << verbosity << settings.maxBuildJobs << settings.maxSilentTime - << settings.useBuildHook + << true << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type << 0 /* obsolete print build trace */ diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fa6ade75002a..c57e42fec00d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -565,8 +565,16 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - Activity act(*logger, lvlInfo, actCopyPath, fmt("copying path '%s'", storePath), - {storePath, srcStore->getUri(), dstStore->getUri()}); + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); PushActivity pact(act.id); auto info = srcStore->queryPathInfo(storePath); @@ -619,6 +627,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + if (missing.empty()) return; + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); std::atomic<size_t> nrDone{0}; @@ -833,7 +843,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir) { if (uri == "daemon") { return tDaemon; - } else if (uri == "local") { + } else if (uri == "local" || hasPrefix(uri, "/")) { return tLocal; } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) @@ -855,8 +865,12 @@ static RegisterStoreImplementation regStore([]( switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { case tDaemon: return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); - case tLocal: - return std::shared_ptr<Store>(std::make_shared<LocalStore>(params)); + case tLocal: { + Store::Params params2 = params; + if (hasPrefix(uri, "/")) + params2["root"] = uri; + return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); + } default: return nullptr; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5f3d8c7b9529..d1e1b5d6f452 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -716,7 +716,7 @@ void removeTempRoots(); You can pass parameters to the store implementation by appending ‘?key=value&key=value&...’ to the URI. */ -ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), +ref<Store> openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); @@ -727,7 +727,8 @@ enum StoreType { }; -StoreType getStoreType(const std::string & uri = getEnv("NIX_REMOTE"), const std::string & stateDir = settings.nixStateDir); +StoreType getStoreType(const std::string & uri = settings.storeUri.get(), + const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options like diff --git a/src/libutil/args.cc b/src/libutil/args.cc index d17a1e7a9abb..7af2a1bf731a 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -100,7 +100,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; - Strings args; + std::vector<std::string> args; for (size_t n = 0 ; n < flag.arity; ++n) { if (pos == end) { if (flag.arity == ArityAny) break; @@ -109,7 +109,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) } args.push_back(*pos++); } - flag.handler(args); + flag.handler(std::move(args)); return true; }; @@ -144,7 +144,9 @@ bool Args::processArgs(const Strings & args, bool finish) if ((exp.arity == 0 && finish) || (exp.arity > 0 && args.size() == exp.arity)) { - exp.handler(args); + std::vector<std::string> ss; + for (auto & s : args) ss.push_back(s); + exp.handler(std::move(ss)); expectedArgs.pop_front(); res = true; } @@ -155,13 +157,17 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } -void Args::mkHashTypeFlag(const std::string & name, HashType * ht) +Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) { - mkFlag1(0, name, "TYPE", "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", [=](std::string s) { + arity(1); + label("type"); + description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); + handler([ht](std::string s) { *ht = parseHashType(s); if (*ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); + throw UsageError("unknown hash type '%1%'", s); }); + return *this; } Strings argvToStrings(int argc, char * * argv) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 37e31825ab37..ad5fcca39418 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -37,7 +37,7 @@ protected: std::string description; Strings labels; size_t arity = 0; - std::function<void(Strings)> handler; + std::function<void(std::vector<std::string>)> handler; std::string category; }; @@ -54,7 +54,7 @@ protected: std::string label; size_t arity; // 0 = any bool optional; - std::function<void(Strings)> handler; + std::function<void(std::vector<std::string>)> handler; }; std::list<ExpectedArg> expectedArgs; @@ -76,24 +76,35 @@ public: FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }; FlagMaker & shortName(char s) { flag->shortName = s; return *this; }; FlagMaker & description(const std::string & s) { flag->description = s; return *this; }; - FlagMaker & labels(const Strings & ls) { flag->labels = ls; return *this; }; + FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }; + FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }; FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }; - FlagMaker & handler(std::function<void(Strings)> handler) { flag->handler = handler; return *this; }; + FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }; + FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }; + FlagMaker & handler(std::function<void(std::string)> handler) { + flag->arity = 1; + flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); }; + return *this; + }; FlagMaker & category(const std::string & s) { flag->category = s; return *this; }; template<class T> - FlagMaker & dest(T * dest) { + FlagMaker & dest(T * dest) + { flag->arity = 1; - flag->handler = [=](Strings ss) { *dest = ss.front(); }; + flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; return *this; }; template<class T> - FlagMaker & set(T * dest, const T & val) { + FlagMaker & set(T * dest, const T & val) + { flag->arity = 0; - flag->handler = [=](Strings ss) { *dest = val; }; + flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; return *this; }; + + FlagMaker & mkHashTypeFlag(HashType * ht); }; FlagMaker mkFlag(); @@ -101,16 +112,6 @@ public: /* Helper functions for constructing flags / positional arguments. */ - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function<void()> fun) - { - mkFlag() - .shortName(shortName) - .longName(longName) - .description(description) - .handler(std::bind(fun)); - } - void mkFlag1(char shortName, const std::string & longName, const std::string & label, const std::string & description, std::function<void(std::string)> fun) @@ -121,7 +122,7 @@ public: .labels({label}) .description(description) .arity(1) - .handler([=](Strings ss) { fun(ss.front()); }); + .handler([=](std::vector<std::string> ss) { fun(ss[0]); }); } void mkFlag(char shortName, const std::string & name, @@ -130,17 +131,6 @@ public: mkFlag(shortName, name, description, dest, true); } - void mkFlag(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - string * dest) - { - mkFlag1(shortName, longName, label, description, [=](std::string s) { - *dest = s; - }); - } - - void mkHashTypeFlag(const std::string & name, HashType * ht); - template<class T> void mkFlag(char shortName, const std::string & longName, const std::string & description, T * dest, const T & value) @@ -149,7 +139,7 @@ public: .shortName(shortName) .longName(longName) .description(description) - .handler([=](Strings ss) { *dest = value; }); + .handler([=](std::vector<std::string> ss) { *dest = value; }); } template<class I> @@ -171,10 +161,10 @@ public: .labels({"N"}) .description(description) .arity(1) - .handler([=](Strings ss) { + .handler([=](std::vector<std::string> ss) { I n; - if (!string2Int(ss.front(), n)) - throw UsageError(format("flag '--%1%' requires a integer argument") % longName); + if (!string2Int(ss[0], n)) + throw UsageError("flag '--%s' requires a integer argument", longName); fun(n); }); } @@ -182,16 +172,16 @@ public: /* Expect a string argument. */ void expectArg(const std::string & label, string * dest, bool optional = false) { - expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](Strings ss) { - *dest = ss.front(); + expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) { + *dest = ss[0]; }}); } /* Expect 0 or more arguments. */ - void expectArgs(const std::string & label, Strings * dest) + void expectArgs(const std::string & label, std::vector<std::string> * dest) { - expectedArgs.push_back(ExpectedArg{label, 0, false, [=](Strings ss) { - *dest = ss; + expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) { + *dest = std::move(ss); }}); } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 27157a83178a..14c4cca031bb 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -152,7 +152,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .longName(name) .description(description) .arity(1) - .handler([=](Strings ss) { set(*ss.begin()); }) + .handler([=](std::vector<std::string> ss) { set(ss[0]); }) .category(category); } @@ -201,12 +201,12 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & args.mkFlag() .longName(name) .description(description) - .handler([=](Strings ss) { value = true; }) + .handler([=](std::vector<std::string> ss) { value = true; }) .category(category); args.mkFlag() .longName("no-" + name) .description(description) - .handler([=](Strings ss) { value = false; }) + .handler([=](std::vector<std::string> ss) { value = false; }) .category(category); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index e38a460537ea..011155871122 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -2,6 +2,7 @@ #include "util.hh" #include <atomic> +#include <nlohmann/json.hpp> namespace nix { @@ -90,4 +91,133 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, logger.startActivity(id, lvl, type, s, fields, parent); } +struct JSONLogger : Logger +{ + Logger & prevLogger; + + JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + + void addFields(nlohmann::json & json, const Fields & fields) + { + if (fields.empty()) return; + auto & arr = json["fields"] = nlohmann::json::array(); + for (auto & f : fields) + if (f.type == Logger::Field::tInt) + arr.push_back(f.i); + else if (f.type == Logger::Field::tString) + arr.push_back(f.s); + else + abort(); + } + + void write(const nlohmann::json & json) + { + prevLogger.log(lvlError, "@nix " + json.dump()); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + nlohmann::json json; + json["action"] = "msg"; + json["level"] = lvl; + json["msg"] = fs.s; + write(json); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + nlohmann::json json; + json["action"] = "start"; + json["id"] = act; + json["level"] = lvl; + json["type"] = type; + json["text"] = s; + addFields(json, fields); + // FIXME: handle parent + write(json); + } + + void stopActivity(ActivityId act) override + { + nlohmann::json json; + json["action"] = "stop"; + json["id"] = act; + write(json); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + addFields(json, fields); + write(json); + } +}; + +Logger * makeJSONLogger(Logger & prevLogger) +{ + return new JSONLogger(prevLogger); +} + +static Logger::Fields getFields(nlohmann::json & json) +{ + Logger::Fields fields; + for (auto & f : json) { + if (f.type() == nlohmann::json::value_t::number_unsigned) + fields.emplace_back(Logger::Field(f.get<uint64_t>())); + else if (f.type() == nlohmann::json::value_t::string) + fields.emplace_back(Logger::Field(f.get<std::string>())); + else throw Error("unsupported JSON type %d", (int) f.type()); + } + return fields; +} + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) +{ + if (!hasPrefix(msg, "@nix ")) return false; + + try { + auto json = nlohmann::json::parse(std::string(msg, 5)); + + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actDownload) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + } catch (std::exception & e) { + printError("bad log message from builder: %s", e.what()); + } + + return true; +} + } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 21898c03a276..677aa4daec4d 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -130,6 +130,12 @@ extern Logger * logger; Logger * makeDefaultLogger(); +Logger * makeJSONLogger(Logger & prevLogger); + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map<ActivityId, Activity> & activities, + bool trusted); + extern Verbosity verbosity; /* suppress msgs > this */ /* Print a message if the current log level is at least the specified diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 70b193941638..2ea5b6354ee9 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -92,7 +92,17 @@ struct FdSink : BufferedSink FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink&&) = default; + + FdSink& operator=(FdSink && s) + { + flush(); + fd = s.fd; + s.fd = -1; + warn = s.warn; + written = s.written; + return *this; + } + ~FdSink(); void write(const unsigned char * data, size_t len) override; @@ -112,6 +122,16 @@ struct FdSource : BufferedSource FdSource() : fd(-1) { } FdSource(int fd) : fd(fd) { } + FdSource(FdSource&&) = default; + + FdSource& operator=(FdSource && s) + { + fd = s.fd; + s.fd = -1; + read = s.read; + return *this; + } + size_t readUnbuffered(unsigned char * data, size_t len) override; bool good() override; private: diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3c98a61f9e50..9346d5dc4cf8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1142,6 +1142,16 @@ std::string toLower(const std::string & s) } +std::string shellEscape(const std::string & s) +{ + std::string r = "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; + return r; +} + + void ignoreException() { try { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6a66576e96ce..fccf5d854800 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -352,10 +352,8 @@ bool hasSuffix(const string & s, const string & suffix); std::string toLower(const std::string & s); -/* Escape a string that contains octal-encoded escape codes such as - used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to - "foo bar"). */ -string decodeOctalEscaped(const string & s); +/* Escape a string as a shell word. */ +std::string shellEscape(const std::string & s); /* Exception handling in destructors: print an error message, then diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a3d3c8007be6..21b0a18dd887 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -14,7 +14,7 @@ #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "attr-path.hh" using namespace nix; @@ -80,8 +80,6 @@ void mainWrapped(int argc, char * * argv) auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings attrPaths; Strings left; - Strings searchPath; - std::map<string, string> autoArgs_; RepairFlag repair = NoRepair; Path gcRoot; BuildMode buildMode = bmNormal; @@ -129,7 +127,12 @@ void mainWrapped(int argc, char * * argv) } catch (SysError &) { } } - parseCmdLine(myName, args, [&](Strings::iterator & arg, const Strings::iterator & end) { + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") { deletePath(tmpDir); showManPage(myName); @@ -153,12 +156,6 @@ void mainWrapped(int argc, char * * argv) else if (*arg == "--out-link" || *arg == "-o") outLink = getArg(*arg, arg, end); - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - - else if (parseSearchPathArg(arg, end, searchPath)) - ; - else if (*arg == "--add-root") gcRoot = getArg(*arg, arg, end); @@ -170,6 +167,9 @@ void mainWrapped(int argc, char * * argv) buildMode = bmRepair; } + else if (*arg == "--hash") + buildMode = bmHash; + else if (*arg == "--run-env") // obsolete runEnv = true; @@ -199,10 +199,6 @@ void mainWrapped(int argc, char * * argv) interactive = false; auto execArgs = ""; - auto shellEscape = [](const string & s) { - return "'" + std::regex_replace(s, std::regex("'"), "'\\''") + "'"; - }; - // Überhack to support Perl. Perl examines the shebang and // executes it unless it contains the string "perl" or "indir", // or (undocumented) argv[0] does not contain "perl". Exploit @@ -237,15 +233,17 @@ void mainWrapped(int argc, char * * argv) return true; }); + myArgs.parseCmdline(args); + if (packages && fromArgs) throw UsageError("'-p' and '-E' are mutually exclusive"); auto store = openStore(); - EvalState state(searchPath, store); + EvalState state(myArgs.searchPath, store); state.repair = repair; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + Bindings & autoArgs = *myArgs.getAutoArgs(state); if (packages) { std::ostringstream joined; @@ -278,7 +276,7 @@ void mainWrapped(int argc, char * * argv) /* If we're in a #! script, interpret filenames relative to the script. */ exprs.push_back(state.parseExprFromFile(resolveExprPath(lookupFileArg(state, - inShebang && !packages ? absPath(i, dirOf(script)) : i)))); + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); } /* Evaluate them into derivations. */ diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index dbf301a91533..5629cc64b96e 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -509,7 +509,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, verbosity = (Verbosity) readInt(from); settings.maxBuildJobs.assign(readInt(from)); settings.maxSilentTime = readInt(from); - settings.useBuildHook = readInt(from) != 0; + readInt(from); // obsolete useBuildHook settings.verboseBuild = lvlError == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 94fbc09f6642..016caf6d2346 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,5 +1,5 @@ #include "attr-path.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "derivations.hh" #include "eval.hh" #include "get-drvs.hh" @@ -1309,8 +1309,7 @@ int main(int argc, char * * argv) initNix(); initGC(); - Strings opFlags, opArgs, searchPath; - std::map<string, string> autoArgs_; + Strings opFlags, opArgs; Operation op = 0; RepairFlag repair = NoRepair; string file; @@ -1326,7 +1325,12 @@ int main(int argc, char * * argv) globals.removeAll = false; globals.prebuiltOnly = false; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") @@ -1335,10 +1339,6 @@ int main(int argc, char * * argv) op = opVersion; else if (*arg == "--install" || *arg == "-i") op = opInstall; - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - else if (parseSearchPathArg(arg, end, searchPath)) - ; else if (*arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = getArg(*arg, arg, end); else if (*arg == "--uninstall" || *arg == "-e") @@ -1391,17 +1391,19 @@ int main(int argc, char * * argv) return true; }); + myArgs.parseCmdline(argvToStrings(argc, argv)); + if (!op) throw UsageError("no operation specified"); auto store = openStore(); - globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath, store)); + globals.state = std::shared_ptr<EvalState>(new EvalState(myArgs.searchPath, store)); globals.state->repair = repair; if (file != "") globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); - globals.instSource.autoArgs = evalAutoArgs(*globals.state, autoArgs_); + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); if (globals.profile == "") globals.profile = getEnv("NIX_PROFILE", ""); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 2498df0f0d72..55ac007e8682 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,7 +8,7 @@ #include "value-to-json.hh" #include "util.hh" #include "store-api.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include <map> #include <iostream> @@ -89,7 +89,7 @@ int main(int argc, char * * argv) initNix(); initGC(); - Strings files, searchPath; + Strings files; bool readStdin = false; bool fromArgs = false; bool findFile = false; @@ -100,10 +100,14 @@ int main(int argc, char * * argv) bool strict = false; Strings attrPaths; bool wantsReadWrite = false; - std::map<string, string> autoArgs_; RepairFlag repair = NoRepair; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-instantiate"); else if (*arg == "--version") @@ -122,10 +126,6 @@ int main(int argc, char * * argv) findFile = true; else if (*arg == "--attr" || *arg == "-A") attrPaths.push_back(getArg(*arg, arg, end)); - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - else if (parseSearchPathArg(arg, end, searchPath)) - ; else if (*arg == "--add-root") gcRoot = getArg(*arg, arg, end); else if (*arg == "--indirect") @@ -149,15 +149,17 @@ int main(int argc, char * * argv) return true; }); + myArgs.parseCmdline(argvToStrings(argc, argv)); + if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; auto store = openStore(); - EvalState state(searchPath, store); + EvalState state(myArgs.searchPath, store); state.repair = repair; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + Bindings & autoArgs = *myArgs.getAutoArgs(state); if (attrPaths.empty()) attrPaths = {""}; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 7e62a033b458..fef3eaa45538 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -4,7 +4,7 @@ #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "attr-path.hh" #include <iostream> @@ -48,15 +48,18 @@ int main(int argc, char * * argv) HashType ht = htSHA256; std::vector<string> args; - Strings searchPath; bool printPath = getEnv("PRINT_PATH") != ""; bool fromExpr = false; string attrPath; - std::map<string, string> autoArgs_; bool unpack = false; string name; - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-prefetch-url"); else if (*arg == "--version") @@ -77,10 +80,6 @@ int main(int argc, char * * argv) unpack = true; else if (*arg == "--name") name = getArg(*arg, arg, end); - else if (parseAutoArgs(arg, end, autoArgs_)) - ; - else if (parseSearchPathArg(arg, end, searchPath)) - ; else if (*arg != "" && arg->at(0) == '-') return false; else @@ -88,13 +87,15 @@ int main(int argc, char * * argv) return true; }); + myArgs.parseCmdline(argvToStrings(argc, argv)); + if (args.size() > 2) throw UsageError("too many arguments"); auto store = openStore(); - EvalState state(searchPath, store); + EvalState state(myArgs.searchPath, store); - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + Bindings & autoArgs = *myArgs.getAutoArgs(state); /* If -A is given, get the URI from the specified Nix expression. */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 85bcbc22e9db..f6f276dd1798 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -440,15 +440,6 @@ static void opQuery(Strings opFlags, Strings opArgs) } -static string shellEscape(const string & s) -{ - string r; - for (auto & i : s) - if (i == '\'') r += "'\\''"; else r += i; - return r; -} - - static void opPrintEnv(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); @@ -460,7 +451,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Print each environment variable in the derivation in a format that can be sourced by the shell. */ for (auto & i : drv.env) - cout << format("export %1%; %1%='%2%'\n") % i.first % shellEscape(i.second); + cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ diff --git a/src/nix/command.cc b/src/nix/command.cc index 0f6bb294b38c..1e6f0d2bb75d 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -24,11 +24,11 @@ void Command::printHelp(const string & programName, std::ostream & out) MultiCommand::MultiCommand(const Commands & _commands) : commands(_commands) { - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](Strings ss) { + expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { assert(!command); - auto i = commands.find(ss.front()); + auto i = commands.find(ss[0]); if (i == commands.end()) - throw UsageError(format("'%1%' is not a recognised command") % ss.front()); + throw UsageError("'%s' is not a recognised command", ss[0]); command = i->second; }}); } @@ -78,9 +78,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) StoreCommand::StoreCommand() { - storeUri = getEnv("NIX_REMOTE"); - - mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); } ref<Store> StoreCommand::getStore() @@ -92,7 +89,7 @@ ref<Store> StoreCommand::getStore() ref<Store> StoreCommand::createStore() { - return openStore(storeUri); + return openStore(); } void StoreCommand::run() diff --git a/src/nix/command.hh b/src/nix/command.hh index bf897f620db6..daa3b3fa7030 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,10 +1,12 @@ #pragma once #include "args.hh" +#include "common-eval-args.hh" namespace nix { struct Value; +struct Bindings; class EvalState; /* A command is an argument parser that can be executed by calling its @@ -33,7 +35,6 @@ class Store; /* A command that require a Nix store. */ struct StoreCommand : virtual Command { - std::string storeUri; StoreCommand(); void run() override; ref<Store> getStore(); @@ -69,14 +70,11 @@ struct Installable } }; -struct SourceExprCommand : virtual Args, StoreCommand +struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { Path file; - SourceExprCommand() - { - mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); - } + SourceExprCommand(); /* Return a value representing the Nix expression from which we are installing. This is either the file specified by ‘--file’, @@ -112,7 +110,7 @@ struct InstallablesCommand : virtual Args, SourceExprCommand private: - Strings _installables; + std::vector<std::string> _installables; }; struct InstallableCommand : virtual Args, SourceExprCommand diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 071ac3890aa9..2ddea9e70a6a 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -19,8 +19,16 @@ struct CmdCopy : StorePathsCommand CmdCopy() : StorePathsCommand(true) { - mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri); - mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri); + mkFlag() + .longName("from") + .labels({"store-uri"}) + .description("URI of the source Nix store") + .dest(&srcUri); + mkFlag() + .longName("to") + .labels({"store-uri"}) + .description("URI of the destination Nix store") + .dest(&dstUri); mkFlag() .longName("no-check-sigs") diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 923dabb103d3..64062fb97955 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -12,14 +12,16 @@ struct CmdHash : Command Base base = Base16; bool truncate = false; HashType ht = htSHA512; - Strings paths; + std::vector<std::string> paths; CmdHash(Mode mode) : mode(mode) { mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); mkFlag(0, "base16", "print hash in base-16", &base, Base16); - mkHashTypeFlag("type", &ht); + mkFlag() + .longName("type") + .mkHashTypeFlag(&ht); expectArgs("paths", &paths); } @@ -53,11 +55,13 @@ struct CmdToBase : Command { Base base; HashType ht = htSHA512; - Strings args; + std::vector<std::string> args; CmdToBase(Base base) : base(base) { - mkHashTypeFlag("type", &ht); + mkFlag() + .longName("type") + .mkHashTypeFlag(&ht); expectArgs("strings", &args); } @@ -95,7 +99,7 @@ static int compatNixHash(int argc, char * * argv) bool base32 = false; bool truncate = false; enum { opHash, opTo32, opTo16 } op = opHash; - Strings ss; + std::vector<std::string> ss; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") diff --git a/src/nix/installables.cc b/src/nix/installables.cc index c83d6316d3f3..ae93c4ef649e 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,6 +1,6 @@ #include "command.hh" #include "attr-path.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" @@ -12,6 +12,16 @@ namespace nix { +SourceExprCommand::SourceExprCommand() +{ + mkFlag() + .shortName('f') + .longName("file") + .label("file") + .description("evaluate FILE rather than the default") + .dest(&file); +} + Value * SourceExprCommand::getSourceExpr(EvalState & state) { if (vSourceExpr) return vSourceExpr; @@ -66,7 +76,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) ref<EvalState> SourceExprCommand::getEvalState() { if (!evalState) - evalState = std::make_shared<EvalState>(Strings{}, getStore()); + evalState = std::make_shared<EvalState>(searchPath, getStore()); return ref<EvalState>(evalState); } @@ -120,9 +130,7 @@ struct InstallableValue : Installable auto v = toValue(*state); - // FIXME - std::map<string, string> autoArgs_; - Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_)); + Bindings & autoArgs = *cmd.getAutoArgs(*state); DrvInfos drvs; getDerivations(*state, *v, "", autoArgs, drvs, false); @@ -187,9 +195,7 @@ struct InstallableAttrPath : InstallableValue { auto source = cmd.getSourceExpr(state); - // FIXME - std::map<string, string> autoArgs_; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + Bindings & autoArgs = *cmd.getAutoArgs(state); Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); state.forceValue(*v); @@ -203,14 +209,14 @@ std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); static std::vector<std::shared_ptr<Installable>> parseInstallables( - SourceExprCommand & cmd, ref<Store> store, Strings ss, bool useDefaultInstallables) + SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables) { std::vector<std::shared_ptr<Installable>> result; if (ss.empty() && useDefaultInstallables) { if (cmd.file == "") cmd.file = "."; - ss = Strings{""}; + ss = {""}; } for (auto & s : ss) { diff --git a/src/nix/main.cc b/src/nix/main.cc index ec9b58b20fe8..060402cd08d5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -20,19 +20,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { - mkFlag('h', "help", "show usage information", [&]() { showHelpAndExit(); }); - - mkFlag(0, "help-config", "show configuration options", [=]() { - std::cout << "The following configuration options are available:\n\n"; - Table2 tbl; - for (const auto & s : settings._getSettings()) - if (!s.second.isAlias) - tbl.emplace_back(s.first, s.second.setting->description); - printTable(std::cout, tbl); - throw Exit(); - }); - - mkFlag(0, "version", "show version information", std::bind(printVersion, programName)); + mkFlag() + .longName("help") + .shortName('h') + .description("show usage information") + .handler([&]() { showHelpAndExit(); }); + + mkFlag() + .longName("help-config") + .description("show configuration options") + .handler([&]() { + std::cout << "The following configuration options are available:\n\n"; + Table2 tbl; + for (const auto & s : settings._getSettings()) + if (!s.second.isAlias) + tbl.emplace_back(s.first, s.second.setting->description); + printTable(std::cout, tbl); + throw Exit(); + }); + + mkFlag() + .longName("version") + .description("show version information") + .handler([&]() { printVersion(programName); }); std::string cat = "config"; settings.convertToArgs(*this, cat); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 76138b2cce28..fb9955190b40 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -156,6 +156,13 @@ public: if (hasSuffix(name, ".drv")) name.resize(name.size() - 4); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); + auto machineName = getS(fields, 1); + if (machineName != "") + i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); + auto curRound = getI(fields, 2); + auto nrRounds = getI(fields, 3); + if (nrRounds != 1) + i->s += fmt(" (round %d/%d)", curRound, nrRounds); } if (type == actSubstitute) { diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 781b4463e54a..28a8ebc8c499 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -7,7 +7,7 @@ #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" -#include "common-opts.hh" +#include "common-eval-args.hh" #include "get-drvs.hh" #include "derivations.hh" #include "affinity.hh" @@ -44,7 +44,7 @@ struct NixRepl NixRepl(const Strings & searchPath, nix::ref<Store> store); ~NixRepl(); - void mainLoop(const Strings & files); + void mainLoop(const std::vector<std::string> & files); StringSet completePrefix(string prefix); bool getLine(string & input, const std::string &prompt); Path getDerivationPath(Value & v); @@ -131,7 +131,7 @@ static void completionCallback(const char * s, linenoiseCompletions *lc) } -void NixRepl::mainLoop(const Strings & files) +void NixRepl::mainLoop(const std::vector<std::string> & files) { string error = ANSI_RED "error:" ANSI_NORMAL " "; std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; @@ -664,9 +664,9 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m return str; } -struct CmdRepl : StoreCommand +struct CmdRepl : StoreCommand, MixEvalArgs { - Strings files; + std::vector<std::string> files; CmdRepl() { @@ -682,8 +682,7 @@ struct CmdRepl : StoreCommand void run(ref<Store> store) override { - // FIXME: pass searchPath - NixRepl repl({}, openStore()); + NixRepl repl(searchPath, openStore()); repl.mainLoop(files); } }; diff --git a/src/nix/run.cc b/src/nix/run.cc index 2f93ca351502..6657a86314bf 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -20,7 +20,7 @@ extern char * * environ; struct CmdRun : InstallablesCommand { - Strings command = { "bash" }; + std::vector<std::string> command = { "bash" }; StringSet keep, unset; bool ignoreEnvironment = false; @@ -32,7 +32,7 @@ struct CmdRun : InstallablesCommand .description("command and arguments to be executed; defaults to 'bash'") .arity(ArityAny) .labels({"command", "args"}) - .handler([&](Strings ss) { + .handler([&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; }); @@ -49,7 +49,7 @@ struct CmdRun : InstallablesCommand .description("keep specified environment variable") .arity(1) .labels({"name"}) - .handler([&](Strings ss) { keep.insert(ss.front()); }); + .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); }); mkFlag() .longName("unset") @@ -57,7 +57,7 @@ struct CmdRun : InstallablesCommand .description("unset specified environment variable") .arity(1) .labels({"name"}) - .handler([&](Strings ss) { unset.insert(ss.front()); }); + .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); }); } std::string name() override @@ -126,7 +126,8 @@ struct CmdRun : InstallablesCommand setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); std::string cmd = *command.begin(); - Strings args = command; + Strings args; + for (auto & arg : command) args.push_back(arg); stopProgressBar(); diff --git a/src/nix/search.cc b/src/nix/search.cc index 9476b79fbc1b..f458367dcb55 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -38,12 +38,12 @@ struct CmdSearch : SourceExprCommand, MixJSON .longName("update-cache") .shortName('u') .description("update the package search cache") - .handler([&](Strings ss) { writeCache = true; useCache = false; }); + .handler([&]() { writeCache = true; useCache = false; }); mkFlag() .longName("no-cache") .description("do not use or update the package search cache") - .handler([&](Strings ss) { writeCache = false; useCache = false; }); + .handler([&]() { writeCache = false; useCache = false; }); } std::string name() override diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 992ff742835e..b1825c412c2d 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -19,7 +19,7 @@ struct CmdCopySigs : StorePathsCommand .labels({"store-uri"}) .description("use signatures from specified store") .arity(1) - .handler([&](Strings ss) { substituterUris.push_back(ss.front()); }); + .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); }); } std::string name() override @@ -101,7 +101,12 @@ struct CmdSignPaths : StorePathsCommand CmdSignPaths() { - mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile); + mkFlag() + .shortName('k') + .longName("key-file") + .label("file") + .description("file containing the secret signing key") + .dest(&secretKeyFile); } std::string name() override diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 4913d990097d..6540208a8a2c 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -25,7 +25,7 @@ struct CmdVerify : StorePathsCommand .labels({"store-uri"}) .description("use signatures from specified store") .arity(1) - .handler([&](Strings ss) { substituterUris.push_back(ss.front()); }); + .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); }); mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); } diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 30d7cc6ba88c..2a044d2edc56 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -100,7 +100,7 @@ clearStore rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) nix-build --option binary-caches "file://$cacheDir" --option signed-binary-caches '' dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "fetching path" $TEST_ROOT/log +grep -q "copying path" $TEST_ROOT/log if [ -n "$HAVE_SODIUM" ]; then diff --git a/tests/build-hook.hook.sh b/tests/build-hook.hook.sh deleted file mode 100755 index c7472eab7600..000000000000 --- a/tests/build-hook.hook.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/sh - -#set -x - -while read x y drv rest; do - - echo "HOOK for $drv" >&2 - - outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv` - - echo "output path is $outPath" >&2 - - if `echo $outPath | grep -q input-1`; then - echo "# accept" >&2 - read inputs - read outputs - mkdir $outPath - echo "BAR" > $outPath/foo - else - echo "# decline" >&2 - fi - -done diff --git a/tests/build-hook.sh b/tests/build-hook.sh deleted file mode 100644 index 2005c7cebdc4..000000000000 --- a/tests/build-hook.sh +++ /dev/null @@ -1,10 +0,0 @@ -source common.sh - -clearStore - -outPath=$(nix-build build-hook.nix --no-out-link --option build-hook $(pwd)/build-hook.hook.sh) - -echo "output path is $outPath" - -text=$(cat "$outPath"/foobar) -if test "$text" != "BARBAR"; then exit 1; fi diff --git a/tests/build-remote.sh b/tests/build-remote.sh index e27ce7e25a8e..cf3bb4633183 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -9,20 +9,15 @@ chmod -R u+w $TEST_ROOT/store0 || true chmod -R u+w $TEST_ROOT/store1 || true rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 -# FIXME: --option is not passed to build-remote, so have to create a config file. -export NIX_CONF_DIR=$TEST_ROOT/etc2 -mkdir -p $NIX_CONF_DIR -echo " -sandbox-paths = /nix/store -sandbox-build-dir = /build-tmp -" > $NIX_CONF_DIR/nix.conf +nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ + --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ + --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" -outPath=$(nix-build build-hook.nix --no-out-link -j0 \ - --option builders "local?root=$TEST_ROOT/store0; local?root=$TEST_ROOT/store1 - - 1 1 foo") +outPath=$TEST_ROOT/result cat $outPath/foobar | grep FOOBAR # Ensure that input1 was built on store1 due to the required feature. p=$(readlink -f $outPath/input-2) -(! nix path-info --store local?root=$TEST_ROOT/store0 --all | grep dependencies.builder1.sh) -nix path-info --store local?root=$TEST_ROOT/store1 --all | grep dependencies.builder1.sh +(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh) +nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index a8d56707e27d..04d605e34438 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -26,7 +26,6 @@ mkdir -p $TEST_HOME export PATH=@bindir@:$PATH coreutils=@coreutils@ -export NIX_BUILD_HOOK= export dot=@dot@ export xmllint="@xmllint@" export SHELL="@bash@" diff --git a/tests/config.nix b/tests/config.nix index 76388fdd5b95..6ba91065b83d 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -13,7 +13,7 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder.sh" "eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; PATH = path; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/lang/lib.nix b/tests/lang/lib.nix index 028a538314b7..1c63b2f31d67 100644 --- a/tests/lang/lib.nix +++ b/tests/lang/lib.nix @@ -49,7 +49,7 @@ rec { if comp (head list2) (head list1) then [(head list2)] ++ mergeLists comp list1 (tail list2) else [(head list1)] ++ mergeLists comp (tail list1) list2; - id = x: x; + id = x:x; # sic const = x: y: x; diff --git a/tests/lang/parse-okay-url.nix b/tests/lang/parse-okay-url.nix index fce3b13ee64b..fb74d66f0923 100644 --- a/tests/lang/parse-okay-url.nix +++ b/tests/lang/parse-okay-url.nix @@ -1,7 +1,8 @@ -[ x:x +[ https://svn.cs.uu.nl:12443/repos/trace/trunk http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2 http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz + channel:nixos-17.09 ] diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 7f4e83b560be..0691c30be1ce 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -14,7 +14,7 @@ chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 export NIX_STORE_DIR=/my/store -export NIX_REMOTE="local?root=$TEST_ROOT/store0" +export NIX_REMOTE=$TEST_ROOT/store0 outPath=$(nix-build dependencies.nix --no-out-link --option sandbox-paths /nix/store) diff --git a/tests/local.mk b/tests/local.mk index 7d99a0fc7675..6160b04c2598 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,7 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - build-hook.sh gc.sh gc-concurrent.sh \ + gc.sh gc-concurrent.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ @@ -14,7 +14,8 @@ nix_tests = \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ build-remote.sh \ - nar-index.sh + nar-index.sh \ + structured-attrs.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/restricted.sh b/tests/restricted.sh index 19096a9f8dd2..a297847cc8e3 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -16,3 +16,15 @@ nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/b (! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. +p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)") +cmp $p restricted.sh + +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval) + +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") + +nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" + +(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) +(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) +(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval) diff --git a/tests/structured-attrs.nix b/tests/structured-attrs.nix new file mode 100644 index 000000000000..6c77a43913a7 --- /dev/null +++ b/tests/structured-attrs.nix @@ -0,0 +1,66 @@ +with import ./config.nix; + +let + + dep = mkDerivation { + name = "dep"; + buildCommand = '' + mkdir $out; echo bla > $out/bla + ''; + }; + +in + +mkDerivation { + name = "structured"; + + __structuredAttrs = true; + + buildCommand = '' + set -x + + [[ $int = 123456789 ]] + [[ -z $float ]] + [[ -n $boolTrue ]] + [[ -z $boolFalse ]] + [[ -n ''${hardening[format]} ]] + [[ -z ''${hardening[fortify]} ]] + [[ ''${#buildInputs[@]} = 7 ]] + [[ ''${buildInputs[2]} = c ]] + [[ -v nothing ]] + [[ -z $nothing ]] + + mkdir ''${outputs[out]} + echo bar > $dest + + json=$(cat .attrs.json) + [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] + [[ $json =~ '"narSize":288' ]] + [[ $json =~ '"closureSize":288' ]] + [[ $json =~ '"references":[]' ]] + ''; + + buildInputs = [ "a" "b" "c" 123 "'" "\"" null ]; + + hardening.format = true; + hardening.fortify = false; + + outer.inner = [ 1 2 3 ]; + + int = 123456789; + + float = 123.456; + + boolTrue = true; + boolFalse = false; + + nothing = null; + + dest = "${placeholder "out"}/foo"; + + "foo bar" = "BAD"; + "1foobar" = "BAD"; + "foo$" = "BAD"; + + exportReferencesGraph.refs = [ dep ]; +} diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh new file mode 100644 index 000000000000..9ba2672b6833 --- /dev/null +++ b/tests/structured-attrs.sh @@ -0,0 +1,7 @@ +source common.sh + +clearStore + +outPath=$(nix-build structured-attrs.nix --no-out-link) + +[[ $(cat $outPath/foo) = bar ]] |