diff options
68 files changed, 1100 insertions, 1593 deletions
diff --git a/.gitignore b/.gitignore index 70ee11f9bfae..a175e8dfe291 100644 --- a/.gitignore +++ b/.gitignore @@ -68,9 +68,6 @@ Makefile.config # /src/nix-instantiate/ /src/nix-instantiate/nix-instantiate -# /src/nix-log2xml/ -/src/nix-log2xml/nix-log2xml - # /src/nix-store/ /src/nix-store/nix-store diff --git a/Makefile b/Makefile index 943b639183aa..52312343d429 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,6 @@ makefiles = \ src/nix-daemon/local.mk \ src/nix-collect-garbage/local.mk \ src/download-via-ssh/local.mk \ - src/nix-log2xml/local.mk \ src/nix-prefetch-url/local.mk \ perl/local.mk \ scripts/local.mk \ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 598b15827883..4c8f3d9d3809 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -406,16 +406,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>binary-cache-secret-key-file</literal></term> - - <listitem><para>Path of the file containing the secret key to be - used for signing binary caches. This file can be generated using - <command>nix-store - --generate-binary-cache-key</command>.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>binary-caches-parallel-connections</literal></term> <listitem><para>The maximum number of parallel HTTP connections diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index d65f4009ee6e..5b7936393951 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -31,7 +31,6 @@ <arg><option>-K</option></arg> <arg><option>--fallback</option></arg> <arg><option>--readonly-mode</option></arg> -<arg><option>--log-type</option> <replaceable>type</replaceable></arg> <arg><option>--show-trace</option></arg> <arg> <option>-I</option> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index c7e8ae1ed05f..bc26a90616e4 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -201,61 +201,6 @@ </varlistentry> -<varlistentry xml:id="opt-log-type"><term><option>--log-type</option> -<replaceable>type</replaceable></term> - - <listitem> - - <para>This option determines how the output written to standard - error is formatted. Nix’s diagnostic messages are typically - <emphasis>nested</emphasis>. For instance, when tracing Nix - expression evaluation (<command>nix-env -vvvvv</command>, messages - from subexpressions are nested inside their parent expressions. Nix - builder output is also often nested. For instance, the Nix Packages - generic builder nests the various build tasks (unpack, configure, - compile, etc.), and the GNU Make in <literal>stdenv-linux</literal> - has been patched to provide nesting for recursive Make - invocations.</para> - - <para><replaceable>type</replaceable> can be one of the - following: - - <variablelist> - - <varlistentry><term><literal>pretty</literal></term> - - <listitem><para>Pretty-print the output, indicating different - nesting levels using spaces. This is the - default.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>escapes</literal></term> - - <listitem><para>Indicate nesting using escape codes that can be - interpreted by the <command>nix-log2xml</command> tool in the - Nix source distribution. The resulting XML file can be fed into - the <command>log2html.xsl</command> stylesheet to create an HTML - file that can be browsed interactively, using JavaScript to - expand and collapse parts of the output.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>flat</literal></term> - - <listitem><para>Remove all nesting.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - </listitem> - -</varlistentry> - - <varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term> <listitem><para>This option is accepted by diff --git a/misc/emacs/nix-mode.el b/misc/emacs/nix-mode.el index 10035e96ef30..e129e9efe1d4 100644 --- a/misc/emacs/nix-mode.el +++ b/misc/emacs/nix-mode.el @@ -8,6 +8,20 @@ ;;; Code: +(defun nix-syntax-match-antiquote (limit) + (let ((pos (next-single-char-property-change (point) 'nix-syntax-antiquote + nil limit))) + (when (and pos (> pos (point))) + (goto-char pos) + (let ((char (char-after pos))) + (pcase char + (`?$ + (forward-char 2)) + (`?} + (forward-char 1))) + (set-match-data (list pos (point))) + t)))) + (defconst nix-font-lock-keywords '("\\_<if\\_>" "\\_<then\\_>" "\\_<else\\_>" "\\_<assert\\_>" "\\_<with\\_>" "\\_<let\\_>" "\\_<in\\_>" "\\_<rec\\_>" "\\_<inherit\\_>" "\\_<or\\_>" @@ -26,7 +40,8 @@ ("<[a-zA-Z0-9._\\+-]+\\(/[a-zA-Z0-9._\\+-]+\\)*>" . font-lock-constant-face) ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+" - . font-lock-constant-face)) + . font-lock-constant-face) + (nix-syntax-match-antiquote 0 font-lock-preprocessor-face t)) "Font lock keywords for nix.") (defvar nix-mode-syntax-table @@ -38,6 +53,67 @@ table) "Syntax table for Nix mode.") +(defun nix-syntax-propertize-escaped-antiquote () + "Set syntax properies for escaped antiquote marks." + nil) + +(defun nix-syntax-propertize-multiline-string () + "Set syntax properies for multiline string delimiters." + (let* ((start (match-beginning 0)) + (end (match-end 0)) + (context (save-excursion (save-match-data (syntax-ppss start)))) + (string-type (nth 3 context))) + (pcase string-type + (`t + ;; inside a multiline string + ;; ending multi-line string delimiter + (put-text-property (1- end) end + 'syntax-table (string-to-syntax "|"))) + (`nil + ;; beginning multi-line string delimiter + (put-text-property start (1+ start) + 'syntax-table (string-to-syntax "|")))))) + +(defun nix-syntax-propertize-antiquote () + "Set syntax properties for antiquote marks." + (let* ((start (match-beginning 0))) + (put-text-property start (1+ start) + 'syntax-table (string-to-syntax "|")) + (put-text-property start (+ start 2) + 'nix-syntax-antiquote t))) + +(defun nix-syntax-propertize-close-brace () + "Set syntax properties for close braces. +If a close brace `}' ends an antiquote, the next character begins a string." + (let* ((start (match-beginning 0)) + (end (match-end 0)) + (context (save-excursion (save-match-data (syntax-ppss start)))) + (open (nth 1 context))) + (when open ;; a corresponding open-brace was found + (let* ((antiquote (get-text-property open 'nix-syntax-antiquote))) + (when antiquote + (put-text-property (+ start 1) (+ start 2) + 'syntax-table (string-to-syntax "|")) + (put-text-property start (1+ start) + 'nix-syntax-antiquote t)))))) + +(defun nix-syntax-propertize (start end) + "Special syntax properties for Nix." + ;; search for multi-line string delimiters + (goto-char start) + (remove-text-properties start end '(syntax-table nil nix-syntax-antiquote nil)) + (funcall + (syntax-propertize-rules + ("''\\${" + (0 (ignore (nix-syntax-propertize-escaped-antiquote)))) + ("''" + (0 (ignore (nix-syntax-propertize-multiline-string)))) + ("\\${" + (0 (ignore (nix-syntax-propertize-antiquote)))) + ("}" + (0 (ignore (nix-syntax-propertize-close-brace))))) + start end)) + (defun nix-indent-line () "Indent current line in a Nix expression." (interactive) @@ -69,7 +145,13 @@ The hook `nix-mode-hook' is run when Nix mode is started. (set-syntax-table nix-mode-syntax-table) ;; Font lock support. - (setq font-lock-defaults '(nix-font-lock-keywords nil nil nil nil)) + (setq-local font-lock-defaults '(nix-font-lock-keywords nil nil nil nil)) + + ;; Special syntax properties for Nix + (setq-local syntax-propertize-function 'nix-syntax-propertize) + + ;; Look at text properties when parsing + (setq-local parse-sexp-lookup-properties t) ;; Automatic indentation [C-j]. (set (make-local-variable 'indent-line-function) 'nix-indent-line) diff --git a/release.nix b/release.nix index 9d9923afc582..6fab352b2687 100644 --- a/release.nix +++ b/release.nix @@ -181,17 +181,21 @@ let rpm_fedora21x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora21x86_64) [ "libsodium-devel" ]; - deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ]; - deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ]; - - deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) []; - deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) []; - deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386) []; - deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64) []; - deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) []; - deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) []; - deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ]; - deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ]; + deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium18" ]; + deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; + + deb_ubuntu1310i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1310i386) [] []; + deb_ubuntu1310x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1310x86_64) [] []; + deb_ubuntu1404i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1404i386) [] []; + deb_ubuntu1404x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1404x86_64) [] []; + deb_ubuntu1410i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1410i386) [] []; + deb_ubuntu1410x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1410x86_64) [] []; + deb_ubuntu1504i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1504i386) [ "libsodium-dev" ] [ "libsodium13" ]; + deb_ubuntu1504x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1504x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; + deb_ubuntu1510i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1510i386) [ "libsodium-dev" ] [ "libsodium13"]; + deb_ubuntu1510x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1510x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; + deb_ubuntu1604i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1604i386) [ "libsodium-dev" ] [ "libsodium18" ]; + deb_ubuntu1604x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1604x86_64) [ "libsodium-dev" ] [ "libsodium18" ]; # System tests. @@ -303,7 +307,7 @@ let makeDeb_x86_64 = makeDeb "x86_64-linux"; makeDeb = - system: diskImageFun: extraPackages: + system: diskImageFun: extraPackages: extraDebPackages: with import <nixpkgs> { inherit system; }; @@ -316,10 +320,11 @@ let ++ extraPackages; }; memSize = 1024; meta.schedulingPriority = 50; + postInstall = "make installcheck"; configureFlags = "--sysconfdir=/etc"; debRequires = [ "curl" "libdbd-sqlite3-perl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libwww-curl-perl" "libssl1.0.0" "liblzma5" ] - ++ lib.optionals (lib.elem "libsodium-dev" extraPackages) [ "libsodium13" ] ; + ++ extraDebPackages; debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>"; doInstallCheck = true; }; diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index ee214b93053b..bd8b44025785 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -53,7 +53,7 @@ sub all { $_ || return 0 for @_; 1 } # Initialisation. my $loadIncreased = 0; -my ($localSystem, $maxSilentTime, $printBuildTrace, $buildTimeout) = @ARGV; +my ($localSystem, $maxSilentTime, $buildTimeout) = @ARGV; my $currentLoad = $ENV{"NIX_CURRENT_LOAD"} // "/run/nix/current-load"; my $conf = $ENV{"NIX_REMOTE_SYSTEMS"} // "@sysconfdir@/nix/machines"; @@ -223,9 +223,6 @@ my @inputs = split /\s/, readline(STDIN); my @outputs = split /\s/, readline(STDIN); -print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace; - - my $maybeSign = ""; $maybeSign = "--sign" if -e "$Nix::Config::confDir/signing-key.sec"; @@ -259,13 +256,11 @@ close UPLOADLOCK; # Perform the build. print STDERR "building ‘$drvPath’ on ‘$hostName’\n"; -print STDERR "@ build-remote-start $drvPath $hostName\n" if $printBuildTrace; writeInt(6, $to) or die; # == cmdBuildPaths writeStrings([$drvPath], $to); writeInt($maxSilentTime, $to); writeInt($buildTimeout, $to); my $res = readInt($from); -print STDERR "@ build-remote-done $drvPath $hostName\n" if $printBuildTrace; if ($res != 0) { my $msg = decode("utf-8", readString($from)); print STDERR "error: $msg on ‘$hostName’\n"; diff --git a/scripts/local.mk b/scripts/local.mk index f6542a4cba2f..13b13a86bc6c 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -7,18 +7,13 @@ nix_bin_scripts := \ bin-scripts += $(nix_bin_scripts) -nix_substituters := \ - $(d)/copy-from-other-stores.pl \ - $(d)/download-from-binary-cache.pl - nix_noinst_scripts := \ $(d)/build-remote.pl \ $(d)/find-runtime-roots.pl \ $(d)/resolve-system-dependencies.pl \ $(d)/nix-http-export.cgi \ $(d)/nix-profile.sh \ - $(d)/nix-reduce-build \ - $(nix_substituters) + $(d)/nix-reduce-build noinst-scripts += $(nix_noinst_scripts) @@ -28,7 +23,6 @@ $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/find-runtime-roots.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) -$(foreach prog, $(nix_substituters), $(eval $(call install-program-in, $(prog), $(libexecdir)/nix/substituters))) $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index b93e5ab1390a..78a69c94e561 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -110,13 +110,6 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--log-type") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - push @instArgs, ($arg, $ARGV[$n]); - push @buildArgs, ($arg, $ARGV[$n]); - } - elsif ($arg eq "--option") { die "$0: ‘$arg’ requires two arguments\n" unless $n + 2 < scalar @ARGV; push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]); @@ -124,7 +117,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--log-type" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { + elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; push @buildArgs, ($arg, $ARGV[$n]); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7ad9a4e46d83..5a6428ca6b6f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -641,7 +641,7 @@ void EvalState::evalFile(const Path & path, Value & v) return; } - startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path2); + Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2); Expr * e = parseExprFromFile(checkSourcePath(path2)); try { eval(e, v); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 4889fe206a31..b06c539de0fb 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -290,7 +290,7 @@ static void getDerivations(EvalState & state, Value & vIn, attrs.insert(std::pair<string, Symbol>(i.name, i.name)); for (auto & i : attrs) { - startNest(nest, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); + Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); string pathPrefix2 = addToPath(pathPrefix, i.first); Value & v2(*v.attrs->find(i.second)->value); if (combineChannels) @@ -310,8 +310,7 @@ static void getDerivations(EvalState & state, Value & vIn, else if (v.isList()) { for (unsigned int n = 0; n < v.listSize(); ++n) { - startNest(nest, lvlDebug, - format("evaluating list element")); + Activity act(*logger, lvlDebug, "evaluating list element"); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 20ae1a696097..776e5cb39b81 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -520,9 +520,10 @@ formal #include <fcntl.h> #include <unistd.h> -#include <eval.hh> -#include <download.hh> -#include <store-api.hh> +#include "eval.hh" +#include "download.hh" +#include "store-api.hh" +#include "primops/fetchgit.hh" namespace nix { @@ -657,7 +658,11 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, makeDownloader()->downloadCached(store, elem.second, true) }; + if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git")) + // FIXME: support specifying revision/branch + res = { true, exportGit(store, elem.second, "master") }; + else + res = { true, makeDownloader()->downloadCached(store, elem.second, true) }; } catch (DownloadError & e) { printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 51680ad62ee2..c2852629a015 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -124,7 +124,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path); Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); @@ -284,7 +284,7 @@ typedef list<Value *> ValueList; static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { - startNest(nest, lvlDebug, "finding dependencies"); + Activity act(*logger, lvlDebug, "finding dependencies"); state.forceAttrs(*args[0], pos); @@ -457,7 +457,7 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation. */ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) { - startNest(nest, lvlVomit, "evaluating derivation"); + Activity act(*logger, lvlVomit, "evaluating derivation"); state.forceAttrs(*args[0], pos); @@ -494,7 +494,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : *args[0]->attrs) { if (i.name == state.sIgnoreNulls) continue; string key = i.name; - startNest(nest, lvlVomit, format("processing attribute ‘%1%’") % key); + Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); try { diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc new file mode 100644 index 000000000000..bd440c8c62ad --- /dev/null +++ b/src/libexpr/primops/fetchgit.cc @@ -0,0 +1,82 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "store-api.hh" + +namespace nix { + +Path exportGit(ref<Store> store, const std::string & uri, const std::string & rev) +{ + if (!isUri(uri)) + throw EvalError(format("‘%s’ is not a valid URI") % uri); + + Path cacheDir = getCacheDir() + "/nix/git"; + + if (!pathExists(cacheDir)) { + createDirs(cacheDir); + runProgram("git", true, { "init", "--bare", cacheDir }); + } + + Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); + + std::string localRef = "pid-" + std::to_string(getpid()); + Path localRefFile = cacheDir + "/refs/heads/" + localRef; + + runProgram("git", true, { "-C", cacheDir, "fetch", uri, rev + ":" + localRef }); + + std::string commitHash = chomp(readFile(localRefFile)); + + unlink(localRefFile.c_str()); + + debug(format("got revision ‘%s’") % commitHash); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, { "-C", cacheDir, "archive", commitHash }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("tar", true, { "x", "-C", tmpDir }, tar); + + return store->addToStore("git-export", tmpDir); +} + +static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + // FIXME: cut&paste from fetch(). + if (state.restricted) throw Error("‘fetchgit’ is not allowed in restricted mode"); + + std::string url; + std::string rev = "master"; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string name(attr.name); + if (name == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (name == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError(format("unsupported argument ‘%1%’ to ‘fetchgit’, at %3%") % attr.name % attr.pos); + } + + if (url.empty()) + throw EvalError(format("‘url’ argument required, at %1%") % pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + Path storePath = exportGit(state.store, url, rev); + + mkString(v, storePath, PathSet({storePath})); +} + +static RegisterPrimOp r("__fetchgit", 1, prim_fetchgit); + +} diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh new file mode 100644 index 000000000000..6ffb21a96daa --- /dev/null +++ b/src/libexpr/primops/fetchgit.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <string> + +#include "ref.hh" + +namespace nix { + +class Store; + +Path exportGit(ref<Store> store, + const std::string & uri, const std::string & rev); + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9219f380c74f..98693d78a7f4 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -18,15 +18,6 @@ MixCommonArgs::MixCommonArgs(const string & programName) verbosity = lvlDebug; }); - mkFlag1(0, "log-type", "type", "set logging format ('pretty', 'flat', 'systemd')", - [](std::string s) { - if (s == "pretty") logType = ltPretty; - else if (s == "escapes") logType = ltEscapes; - else if (s == "flat") logType = ltFlat; - else if (s == "systemd") logType = ltSystemd; - else throw UsageError("unknown log type"); - }); - mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2, [](Strings ss) { auto name = ss.front(); ss.pop_front(); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index ed997052be20..0b6311516ad4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -111,8 +111,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - if (getEnv("IN_SYSTEMD") == "1") - logType = ltSystemd; + logger = makeDefaultLogger(); /* Initialise OpenSSL locking. */ opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); @@ -172,10 +171,7 @@ struct LegacyArgs : public MixCommonArgs : MixCommonArgs(programName), parseArg(parseArg) { mkFlag('Q', "no-build-output", "do not show build output", - &settings.buildVerbosity, lvlVomit); - - mkFlag(0, "print-build-trace", "emit special build trace message", - &settings.printBuildTrace); + &settings.verboseBuild, false); mkFlag('K', "keep-failed", "keep temporary directories of failed builds", &settings.keepFailed); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 93863b95fe7f..411d10130a31 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -15,14 +15,13 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile) + const StoreParams & params) : localStore(localStore) + , compression(get(params, "compression", "xz")) { - if (secretKeyFile != "") { + auto secretKeyFile = get(params, "secret-key", ""); + if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); - publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys); - publicKeys->emplace(secretKey->name, secretKey->toPublicKey()); - } StringSink sink; sink << narVersionMagic1; @@ -47,8 +46,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath) return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToCache(const ValidPathInfo & info, - const string & nar) +void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref<std::string> nar) { /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -64,40 +62,43 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, auto narInfoFile = narInfoFileFor(info.path); if (fileExists(narInfoFile)) return; - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref<NarInfo>(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); /* Compress the NAR. */ - narInfo->compression = "xz"; + narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - string narXz = compressXZ(nar); + auto narCompressed = compress(compression, nar); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, narXz); - narInfo->fileSize = narXz.size(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ - narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + ""); if (!fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, narXz); + upsertFile(narInfo->url, *narCompressed); } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); - stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -139,12 +140,12 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) /* Decompress the NAR. FIXME: would be nice to have the remote side do this. */ - if (info->compression == "none") - ; - else if (info->compression == "xz") - nar = decompressXZ(*nar); - else - throw Error(format("unknown NAR compression type ‘%1%’") % nar); + try { + nar = decompress(info->compression, ref<std::string>(nar)); + } catch (UnknownCompressionMethod &) { + throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’") + % storePath % info->compression); + } stats.narReadBytes += nar->size(); @@ -213,11 +214,6 @@ std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Pat stats.narInfoRead++; - if (publicKeys) { - if (!narInfo->checkSignatures(*publicKeys)) - throw Error(format("no good signature on NAR info file ‘%1%’") % narInfoFile); - } - return std::shared_ptr<NarInfo>(narInfo); } @@ -268,7 +264,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); if (repair || !isValidPath(info.path)) - addToCache(info, *sink.s); + addToCache(info, sink.s); return info.path; } @@ -283,7 +279,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToCache(info, *sink.s); + addToCache(info, sink.s); } return info.path; @@ -313,7 +309,7 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) StringSink sink; dumpPath(storePath, sink); - addToCache(*info, *sink.s); + addToCache(*info, sink.s); } } @@ -411,7 +407,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> a bool haveSignature = readInt(source) == 1; assert(!haveSignature); - addToCache(info, *tee.data); + addToCache(info, tee.data); auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); if (accessor_) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 4e4346a43438..46a38a1e0fc3 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -16,13 +16,15 @@ class BinaryCacheStore : public Store private: std::unique_ptr<SecretKey> secretKey; - std::unique_ptr<PublicKeys> publicKeys; std::shared_ptr<Store> localStore; + std::string compression; + protected: - BinaryCacheStore(std::shared_ptr<Store> localStore, const Path & secretKeyFile); + BinaryCacheStore(std::shared_ptr<Store> localStore, + const StoreParams & params); [[noreturn]] void notImpl(); @@ -44,7 +46,7 @@ private: std::string narInfoFileFor(const Path & storePath); - void addToCache(const ValidPathInfo & info, const string & nar); + void addToCache(const ValidPathInfo & info, ref<std::string> nar); public: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ae8078069d07..65df2eea59a0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -8,11 +8,14 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "finally.hh" #include <algorithm> #include <iostream> #include <map> #include <sstream> +#include <thread> +#include <future> #include <limits.h> #include <time.h> @@ -199,8 +202,6 @@ struct Child time_t timeStarted; }; -typedef map<pid_t, Child> Children; - /* The worker class. */ class Worker @@ -220,7 +221,7 @@ private: WeakGoals wantingToBuild; /* Child processes currently running. */ - Children children; + std::list<Child> children; /* Number of build slots occupied. This includes local builds and substitutions but not remote builds via the build hook. */ @@ -278,14 +279,14 @@ public: /* Registers a running child process. `inBuildSlot' means that the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, pid_t pid, - const set<int> & fds, bool inBuildSlot, bool respectTimeouts); + void childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts); /* Unregisters a running child process. `wakeSleepers' should be false if there is no sense in waking up goals that are sleeping because they can't run yet (e.g., there is no free build slot, or the hook would still say `postpone'). */ - void childTerminated(pid_t pid, bool wakeSleepers = true); + void childTerminated(GoalPtr goal, bool wakeSleepers = true); /* Put `goal' to sleep until a build slot becomes available (which might be right away). */ @@ -633,7 +634,6 @@ HookInstance::HookInstance() baseNameOf(buildHook), settings.thisSystem, (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.printBuildTrace).str(), (format("%1%") % settings.buildTimeout).str() }; @@ -748,6 +748,12 @@ private: /* Number of bytes received from the builder's stdout/stderr. */ unsigned long logSize; + /* The most recent log lines. */ + std::list<std::string> logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -873,6 +879,7 @@ private: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; + void flushLine(); /* Return the set of (in)valid paths. */ PathSet checkPathValidity(bool returnValid, bool checkHash); @@ -936,7 +943,7 @@ DerivationGoal::~DerivationGoal() void DerivationGoal::killChild() { if (pid != -1) { - worker.childTerminated(pid); + worker.childTerminated(shared_from_this()); if (buildUser.enabled()) { /* If we're using a build user, then there is a tricky @@ -960,8 +967,6 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut() { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); killChild(); done(BuildResult::TimedOut); } @@ -1362,9 +1367,6 @@ void DerivationGoal::tryToBuild() printMsg(lvlError, e.msg()); outputLocks.unlock(); buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 0 % e.msg()); worker.permanentFailure = true; done(BuildResult::InputRejected, e.msg()); return; @@ -1402,22 +1404,14 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe --- just don't do that :-) */ - int status; - pid_t savedPid; - if (hook) { - savedPid = hook->pid; - status = hook->pid.wait(true); - } else { - /* !!! this could block! security problem! solution: kill the - child */ - savedPid = pid; - status = pid.wait(true); - } + /* !!! this could block! security problem! solution: kill the + child */ + int status = hook ? hook->pid.wait(true) : pid.wait(true); debug(format("builder process for ‘%1%’ finished") % drvPath); /* So the child is gone now. */ - worker.childTerminated(savedPid); + worker.childTerminated(shared_from_this()); /* Close the read side of the logger pipe. */ if (hook) { @@ -1468,11 +1462,19 @@ void DerivationGoal::buildDone() if (pathExists(chrootRootDir + i)) rename((chrootRootDir + i).c_str(), i.c_str()); + std::string msg = (format("builder for ‘%1%’ %2%") + % drvPath % statusToString(status)).str(); + + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto & line : logTail) + msg += "\n " + line; + } + if (diskFull) - printMsg(lvlError, "note: build failure may have been caused by lack of free disk space"); + msg += "\nnote: build failure may have been caused by lack of free disk space"; - throw BuildError(format("builder for ‘%1%’ %2%") - % drvPath % statusToString(status)); + throw BuildError(msg); } /* Compute the FS closure of the outputs and register them as @@ -1517,23 +1519,13 @@ void DerivationGoal::buildDone() BuildResult::Status st = BuildResult::MiscFailure; - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) st = BuildResult::TimedOut; - } else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") - % drvPath % status % e.msg()); } else { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 1 % e.msg()); - st = dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : @@ -1548,9 +1540,6 @@ void DerivationGoal::buildDone() /* Release the build user, if applicable. */ buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); - done(BuildResult::Built); } @@ -1625,11 +1614,7 @@ HookReply DerivationGoal::tryBuildHook() set<int> fds; fds.insert(hook->fromHook.readSide); fds.insert(hook->builderOut.readSide); - worker.childStarted(shared_from_this(), hook->pid, fds, false, false); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv->platform % logFile); + worker.childStarted(shared_from_this(), fds, false, false); return rpAccept; } @@ -1657,12 +1642,10 @@ void DerivationGoal::startBuilder() nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : "building path(s) %1%"); f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); + printMsg(lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ if (!drv->canBuildLocally()) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform); throw Error( format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’") % drv->platform % settings.thisSystem % drvPath); @@ -2165,7 +2148,7 @@ void DerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); builderOut.writeSide.close(); - worker.childStarted(shared_from_this(), pid, + worker.childStarted(shared_from_this(), singleton<set<int> >(builderOut.readSide), true, true); /* Check if setting up the build environment failed. */ @@ -2177,11 +2160,6 @@ void DerivationGoal::startBuilder() } printMsg(lvlDebug, msg); } - - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv->platform % logFile); - } } @@ -2192,8 +2170,6 @@ void DerivationGoal::runChild() try { /* child */ - logType = ltFlat; - commonChildInit(builderOut); #if __linux__ @@ -2535,7 +2511,6 @@ void DerivationGoal::runChild() /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { - logType = ltFlat; if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv); else @@ -2667,8 +2642,7 @@ void DerivationGoal::registerOutputs() rewritten = true; } - startNest(nest, lvlTalkative, - format("scanning for references inside ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified @@ -2954,8 +2928,18 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) done(BuildResult::LogLimitExceeded); return; } - if (verbosity >= settings.buildVerbosity) - writeToStderr(filterANSIEscapes(data, true)); + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + if (bzLogFile) { int err; BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); @@ -2965,16 +2949,30 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide) - writeToStderr(data); + printMsg(lvlError, data); // FIXME? } void DerivationGoal::handleEOF(int fd) { + if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); } +void DerivationGoal::flushLine() +{ + if (settings.verboseBuild) + printMsg(lvlInfo, filterANSIEscapes(currentLogLine, true)); + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); + } + currentLogLine = ""; + currentLogLinePos = 0; +} + + PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { PathSet result; @@ -3027,28 +3025,24 @@ private: Path storePath; /* The remaining substituters. */ - Paths subs; + std::list<ref<Store>> subs; /* The current substituter. */ - Path sub; + std::shared_ptr<Store> sub; - /* Whether any substituter can realise this path */ + /* Whether any substituter can realise this path. */ bool hasSubstitute; /* Path info returned by the substituter's query info operation. */ - SubstitutablePathInfo info; + std::shared_ptr<const ValidPathInfo> info; /* Pipe for the substituter's standard output. */ Pipe outPipe; - /* Pipe for the substituter's standard error. */ - Pipe logPipe; + /* The substituter thread. */ + std::thread thr; - /* The process ID of the builder. */ - Pid pid; - - /* Lock on the store path. */ - std::shared_ptr<PathLocks> outputLock; + std::promise<void> promise; /* Whether to try to repair a valid path. */ bool repair; @@ -3064,7 +3058,7 @@ public: SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); ~SubstitutionGoal(); - void timedOut(); + void timedOut() { abort(); }; string key() { @@ -3105,20 +3099,14 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool SubstitutionGoal::~SubstitutionGoal() { - if (pid != -1) worker.childTerminated(pid); -} - - -void SubstitutionGoal::timedOut() -{ - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); - if (pid != -1) { - pid_t savedPid = pid; - pid.kill(); - worker.childTerminated(savedPid); + try { + if (thr.joinable()) { + thr.join(); + worker.childTerminated(shared_from_this()); + } + } catch (...) { + ignoreException(); } - amDone(ecFailed); } @@ -3143,7 +3131,7 @@ void SubstitutionGoal::init() if (settings.readOnlyMode) throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath); - subs = settings.substituters; + subs = getDefaultSubstituters(); tryNext(); } @@ -3168,17 +3156,19 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - SubstitutablePathInfos infos; - PathSet dummy(singleton<PathSet>(storePath)); - worker.store.querySubstitutablePathInfos(sub, dummy, infos); - SubstitutablePathInfos::iterator k = infos.find(storePath); - if (k == infos.end()) { tryNext(); return; } - info = k->second; + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath &) { + tryNext(); + return; + } + hasSubstitute = true; /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info.references) + for (auto & i : info->references) if (i != storePath) /* ignore self-references */ addWaitee(worker.makeSubstitutionGoal(i)); @@ -3199,7 +3189,7 @@ void SubstitutionGoal::referencesValid() return; } - for (auto & i : info.references) + for (auto & i : info->references) if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); @@ -3221,75 +3211,32 @@ void SubstitutionGoal::tryToRun() return; } - /* Maybe a derivation goal has already locked this path - (exceedingly unlikely, since it should have used a substitute - first, but let's be defensive). */ - outputLock.reset(); // make sure this goal's lock is gone - if (pathIsLockedByMe(storePath)) { - debug(format("restarting substitution of ‘%1%’ because it's locked by another goal") - % storePath); - worker.waitForAnyGoal(shared_from_this()); - return; /* restart in the tryToRun() state when another goal finishes */ - } - - /* Acquire a lock on the output path. */ - outputLock = std::make_shared<PathLocks>(); - if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Check again whether the path is invalid. */ - if (!repair && worker.store.isValidPath(storePath)) { - debug(format("store path ‘%1%’ has become valid") % storePath); - outputLock->setDeletion(true); - amDone(ecSuccess); - return; - } - printMsg(lvlInfo, format("fetching path ‘%1%’...") % storePath); outPipe.create(); - logPipe.create(); - destPath = repair ? storePath + ".tmp" : storePath; + promise = std::promise<void>(); - /* Remove the (stale) output path if it exists. */ - deletePath(destPath); + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide.close(); }); - worker.store.setSubstituterEnv(); - - /* Fill in the arguments. */ - Strings args; - args.push_back(baseNameOf(sub)); - args.push_back("--substitute"); - args.push_back(storePath); - args.push_back(destPath); - - /* Fork the substitute program. */ - pid = startProcess([&]() { - - commonChildInit(logPipe); - - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); + StringSink sink; + sub->exportPaths({storePath}, false, sink); - execv(sub.c_str(), stringsToCharPtrs(args).data()); + StringSource source(*sink.s); + worker.store.importPaths(false, source, 0); - throw SysError(format("executing ‘%1%’") % sub); + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } }); - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - outPipe.writeSide.close(); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton<set<int> >(logPipe.readSide), true, true); + worker.childStarted(shared_from_this(), {outPipe.readSide}, true, false); state = &SubstitutionGoal::finished; - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); } @@ -3297,110 +3244,40 @@ void SubstitutionGoal::finished() { trace("substitute finished"); - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - pid_t savedPid = pid; - int status = pid.wait(true); + thr.join(); + worker.childTerminated(shared_from_this()); - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); - - /* Get the hash info from stdout. */ - string dummy = readLine(outPipe.readSide); - string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; - outPipe.readSide.close(); - - /* Check the exit status and the build result. */ - HashResult hash; try { - - if (!statusOk(status)) - throw SubstError(format("fetching path ‘%1%’ %2%") - % storePath % statusToString(status)); - - if (!pathExists(destPath)) - throw SubstError(format("substitute did not produce path ‘%1%’") % destPath); - - hash = hashPath(htSHA256, destPath); - - /* Verify the expected hash we got from the substituer. */ - if (expectedHashStr != "") { - size_t n = expectedHashStr.find(':'); - if (n == string::npos) - throw Error(format("bad hash from substituter: %1%") % expectedHashStr); - HashType hashType = parseHashType(string(expectedHashStr, 0, n)); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm in ‘%1%’") % expectedHashStr); - Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); - Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) - throw SubstError(format("hash mismatch in downloaded path ‘%1%’: expected %2%, got %3%") - % storePath % printHash(expectedHash) % printHash(actualHash)); - } - - } catch (SubstError & e) { - + promise.get_future().get(); + } catch (Error & e) { printMsg(lvlInfo, e.msg()); - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") - % storePath % status % e.msg()); - } - /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } - if (repair) replaceValidPath(storePath, destPath); - - canonicalisePathMetaData(storePath, -1); - - worker.store.optimisePath(storePath); // FIXME: combine with hashPath() - - ValidPathInfo info2; - info2.path = storePath; - info2.narHash = hash.first; - info2.narSize = hash.second; - info2.references = info.references; - info2.deriver = info.deriver; - worker.store.registerValidPath(info2); - - outputLock->setDeletion(true); - outputLock.reset(); - worker.markContentsGood(storePath); printMsg(lvlChatty, format("substitution of path ‘%1%’ succeeded") % storePath); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); - amDone(ecSuccess); } void SubstitutionGoal::handleChildOutput(int fd, const string & data) { - assert(fd == logPipe.readSide); - if (verbosity >= settings.buildVerbosity) writeToStderr(data); - /* Don't write substitution output to a log file for now. We - probably should, though. */ } void SubstitutionGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + if (fd == outPipe.readSide) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -3516,9 +3393,8 @@ unsigned Worker::getNrLocalBuilds() } -void Worker::childStarted(GoalPtr goal, - pid_t pid, const set<int> & fds, bool inBuildSlot, - bool respectTimeouts) +void Worker::childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; @@ -3526,30 +3402,29 @@ void Worker::childStarted(GoalPtr goal, child.timeStarted = child.lastOutput = time(0); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; - children[pid] = child; + children.emplace_back(child); if (inBuildSlot) nrLocalBuilds++; } -void Worker::childTerminated(pid_t pid, bool wakeSleepers) +void Worker::childTerminated(GoalPtr goal, bool wakeSleepers) { - assert(pid != -1); /* common mistake */ - - Children::iterator i = children.find(pid); + auto i = std::find_if(children.begin(), children.end(), + [&](const Child & child) { return child.goal.lock() == goal; }); assert(i != children.end()); - if (i->second.inBuildSlot) { + if (i->inBuildSlot) { assert(nrLocalBuilds > 0); nrLocalBuilds--; } - children.erase(pid); + children.erase(i); if (wakeSleepers) { /* Wake up goals waiting for a build slot. */ - for (auto & i : wantingToBuild) { - GoalPtr goal = i.lock(); + for (auto & j : wantingToBuild) { + GoalPtr goal = j.lock(); if (goal) wakeUp(goal); } @@ -3586,7 +3461,7 @@ void Worker::run(const Goals & _topGoals) { for (auto & i : _topGoals) topGoals.insert(i); - startNest(nest, lvlDebug, format("entered goal loop")); + Activity act(*logger, lvlDebug, "entered goal loop"); while (1) { @@ -3651,11 +3526,11 @@ void Worker::waitForInput() assert(sizeof(time_t) >= sizeof(long)); time_t nearest = LONG_MAX; // nearest deadline for (auto & i : children) { - if (!i.second.respectTimeouts) continue; + if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i.second.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i.second.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); } if (nearest != LONG_MAX) { timeout.tv_sec = std::max((time_t) 1, nearest - before); @@ -3673,7 +3548,6 @@ void Worker::waitForInput() timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); } else lastWokenUp = 0; - using namespace std; /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ @@ -3681,7 +3555,7 @@ void Worker::waitForInput() FD_ZERO(&fds); int fdMax = 0; for (auto & i : children) { - for (auto & j : i.second.fds) { + for (auto & j : i.fds) { FD_SET(j, &fds); if (j >= fdMax) fdMax = j + 1; } @@ -3695,22 +3569,16 @@ void Worker::waitForInput() time_t after = time(0); /* Process all available file descriptors. */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); - /* Since goals may be canceled from inside the loop below (causing - them go be erased from the `children' map), we have to be - careful that we don't keep iterators alive across calls to - timedOut(). */ - set<pid_t> pids; - for (auto & i : children) pids.insert(i.first); - - for (auto & i : pids) { checkInterrupt(); - Children::iterator j = children.find(i); - if (j == children.end()) continue; // child destroyed - GoalPtr goal = j->second.goal.lock(); + + GoalPtr goal = j->goal.lock(); assert(goal); - set<int> fds2(j->second.fds); + set<int> fds2(j->fds); for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { unsigned char buffer[4096]; @@ -3722,12 +3590,12 @@ void Worker::waitForInput() } else if (rd == 0) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); - j->second.fds.erase(k); + j->fds.erase(k); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); string data((char *) buffer, rd); - j->second.lastOutput = after; + j->lastOutput = after; goal->handleChildOutput(k, data); } } @@ -3735,8 +3603,8 @@ void Worker::waitForInput() if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && - j->second.respectTimeouts && - after - j->second.lastOutput >= (time_t) settings.maxSilentTime) + j->respectTimeouts && + after - j->lastOutput >= (time_t) settings.maxSilentTime) { printMsg(lvlError, format("%1% timed out after %2% seconds of silence") @@ -3746,8 +3614,8 @@ void Worker::waitForInput() else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && - j->second.respectTimeouts && - after - j->second.timeStarted >= (time_t) settings.buildTimeout) + j->respectTimeouts && + after - j->timeStarted >= (time_t) settings.buildTimeout) { printMsg(lvlError, format("%1% timed out after %2% seconds") @@ -3804,8 +3672,6 @@ void Worker::markContentsGood(const Path & path) void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - startNest(nest, lvlDebug, format("building %1%") % showPaths(drvPaths)); - Worker worker(*this); Goals goals; @@ -3835,8 +3701,6 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - startNest(nest, lvlDebug, format("building %1%") % showPaths({drvPath})); - Worker worker(*this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index 50417b644a02..a4785d6905bb 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -31,7 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv) auto unpack = drv.env.find("unpack"); if (unpack != drv.env.end() && unpack->second == "1") { if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data.data = decompressXZ(*data.data); + data.data = decompress("xz", ref<std::string>(data.data)); StringSource source(*data.data); restorePath(storePath, source); } else diff --git a/src/libstore/download.cc b/src/libstore/download.cc index eed630517d11..6e39330e40d9 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -313,7 +313,7 @@ bool isUri(const string & s) size_t pos = s.find("://"); if (pos == string::npos) return false; string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel"; + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git"; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 1536dcb590e0..8fc582f4c20d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -305,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = settings.nixStore + "/" + baseNameOf(path); - if (isValidPath(storePath)) + if (isStorePath(storePath) && isValidPath(storePath)) roots[path] = storePath; } @@ -514,7 +514,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) if (path == linksDir || path == state.trashDir) return; - startNest(nest, lvlDebug, format("considering whether to delete ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); if (!isValidPath(path)) { /* A lock file belonging to a path that we're building right diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b6078253319f..c12178e4028a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -28,7 +28,6 @@ Settings::Settings() keepFailed = false; keepGoing = false; tryFallback = false; - buildVerbosity = lvlError; maxBuildJobs = 1; buildCores = 1; #ifdef _SC_NPROCESSORS_ONLN @@ -40,7 +39,6 @@ Settings::Settings() maxSilentTime = 0; buildTimeout = 0; useBuildHook = true; - printBuildTrace = false; reservedSize = 8 * 1024 * 1024; fsyncMetadata = true; useSQLiteWAL = true; @@ -186,19 +184,6 @@ void Settings::update() _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); _get(useCaseHack, "use-case-hack"); _get(preBuildHook, "pre-build-hook"); - - string subs = getEnv("NIX_SUBSTITUTERS", "default"); - if (subs == "default") { - substituters.clear(); -#if 0 - if (getEnv("NIX_OTHER_STORES") != "") - substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); -#endif - substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); - if (useSshSubstituter && !sshSubstituterHosts.empty()) - substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); - } else - substituters = tokenizeString<Strings>(subs, ":"); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 572fa7188c14..65f763ace3c7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <map> #include <sys/types.h> @@ -77,8 +78,12 @@ struct Settings { instead. */ bool tryFallback; - /* Verbosity level for build output. */ - Verbosity buildVerbosity; + /* Whether to show build log output in real time. */ + bool verboseBuild = true; + + /* If verboseBuild is false, the number of lines of the tail of + the log to show if a build fails. */ + size_t logLines = 10; /* Maximum number of parallel build jobs. 0 means unlimited. */ unsigned int maxBuildJobs; @@ -105,31 +110,10 @@ struct Settings { means infinity. */ time_t buildTimeout; - /* The substituters. There are programs that can somehow realise - a store path without building, e.g., by downloading it or - copying it from a CD. */ - Paths substituters; - /* Whether to use build hooks (for distributed builds). Sometimes users want to disable this from the command-line. */ bool useBuildHook; - /* Whether buildDerivations() should print out lines on stderr in - a fixed format to allow its progress to be monitored. Each - line starts with a "@". The following are defined: - - @ build-started <drvpath> <outpath> <system> <logfile> - @ build-failed <drvpath> <outpath> <exitcode> <error text> - @ build-succeeded <drvpath> <outpath> - @ substituter-started <outpath> <substituter> - @ substituter-failed <outpath> <exitcode> <error text> - @ substituter-succeeded <outpath> - - Best combined with --no-build-output, otherwise stderr might - conceivably contain lines in this format printed by the - builders. */ - bool printBuildTrace; - /* Amount of reserved space for the garbage collector (/nix/var/nix/db/reserved). */ off_t reservedSize; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 771eb42ee561..92d94aeeacd5 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -16,8 +16,8 @@ private: public: HttpBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & _cacheUri) - : BinaryCacheStore(localStore, secretKeyFile) + const StoreParams & params, const Path & _cacheUri) + : BinaryCacheStore(localStore, params) , cacheUri(_cacheUri) , downloaders( std::numeric_limits<size_t>::max(), @@ -85,12 +85,14 @@ protected: }; -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://") return 0; auto store = std::make_shared<HttpBinaryCacheStore>(std::shared_ptr<Store>(0), - settings.get("binary-cache-secret-key-file", string("")), - uri); + params, uri); store->init(); return store; }); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 7968c98b9558..2c2944938761 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -12,10 +12,19 @@ private: public: LocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir); + const StoreParams & params, const Path & binaryCacheDir) + : BinaryCacheStore(localStore, params) + , binaryCacheDir(binaryCacheDir) + { + } void init() override; + std::string getUri() override + { + return "file://" + binaryCacheDir; + } + protected: bool fileExists(const std::string & path) override; @@ -24,14 +33,21 @@ protected: std::shared_ptr<std::string> getFile(const std::string & path) override; -}; + PathSet queryAllValidPaths() override + { + PathSet paths; -LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir) - : BinaryCacheStore(localStore, secretKeyFile) - , binaryCacheDir(binaryCacheDir) -{ -} + for (auto & entry : readDirectory(binaryCacheDir)) { + if (entry.name.size() != 40 || + !hasSuffix(entry.name, ".narinfo")) + continue; + paths.insert(settings.nixStore + "/" + entry.name.substr(0, entry.name.size() - 8)); + } + + return paths; + } + +}; void LocalBinaryCacheStore::init() { @@ -69,20 +85,15 @@ std::shared_ptr<std::string> LocalBinaryCacheStore::getFile(const std::string & } } -ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir) +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> { - auto store = make_ref<LocalBinaryCacheStore>( - localStore, secretKeyFile, binaryCacheDir); + if (std::string(uri, 0, 7) != "file://") return 0; + auto store = std::make_shared<LocalBinaryCacheStore>( + std::shared_ptr<Store>(0), params, std::string(uri, 7)); store->init(); return store; -} - -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { - if (std::string(uri, 0, 7) != "file://") return 0; - return openLocalBinaryCacheStore(std::shared_ptr<Store>(0), - settings.get("binary-cache-secret-key-file", string("")), - std::string(uri, 7)); }); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index baf3a528b705..01a11f11f65d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,12 +5,11 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" -#include "affinity.hh" +#include "nar-info.hh" #include <iostream> #include <algorithm> #include <cstring> -#include <atomic> #include <sys/types.h> #include <sys/stat.h> @@ -220,19 +219,6 @@ LocalStore::~LocalStore() auto state(_state.lock()); try { - for (auto & i : state->runningSubstituters) { - if (i.second.disabled) continue; - i.second.to.close(); - i.second.from.close(); - i.second.error.close(); - if (i.second.pid != -1) - i.second.pid.wait(true); - } - } catch (...) { - ignoreException(); - } - - try { if (state->fdTempRoots != -1) { state->fdTempRoots.close(); unlink(state->fnTempRoots.c_str()); @@ -243,6 +229,12 @@ LocalStore::~LocalStore() } +std::string LocalStore::getUri() +{ + return "local"; +} + + int LocalStore::getSchema() { int curSchema = 0; @@ -792,205 +784,42 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) } -void LocalStore::setSubstituterEnv() -{ - static std::atomic_flag done; - - if (done.test_and_set()) return; - - /* Pass configuration options (including those overridden with - --option) to substituters. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); -} - - -void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) -{ - if (run.disabled || run.pid != -1) return; - - debug(format("starting substituter program ‘%1%’") % substituter); - - Pipe toPipe, fromPipe, errorPipe; - - toPipe.create(); - fromPipe.create(); - errorPipe.create(); - - setSubstituterEnv(); - - run.pid = startProcess([&]() { - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing ‘%1%’") % substituter); - }); - - run.program = baseNameOf(substituter); - run.to = toPipe.writeSide.borrow(); - run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); - run.error = errorPipe.readSide.borrow(); - - toPipe.readSide.close(); - fromPipe.writeSide.close(); - errorPipe.writeSide.close(); - - /* The substituter may exit right away if it's disabled in any way - (e.g. copy-from-other-stores.pl will exit if no other stores - are configured). */ - try { - getLineFromSubstituter(run); - } catch (EndOfFile & e) { - run.to.close(); - run.from.close(); - run.error.close(); - run.disabled = true; - if (run.pid.wait(true) != 0) throw; - } -} - - -/* Read a line from the substituter's stdout, while also processing - its stderr. */ -string LocalStore::getLineFromSubstituter(RunningSubstituter & run) -{ - string res, err; - - /* We might have stdout data left over from the last time. */ - if (run.fromBuf.hasData()) goto haveData; - - while (1) { - checkInterrupt(); - - fd_set fds; - FD_ZERO(&fds); - FD_SET(run.from, &fds); - FD_SET(run.error, &fds); - - /* Wait for data to appear on the substituter's stdout or - stderr. */ - if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { - if (errno == EINTR) continue; - throw SysError("waiting for input from the substituter"); - } - - /* Completely drain stderr before dealing with stdout. */ - if (FD_ISSET(run.error, &fds)) { - char buf[4096]; - ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from substituter's stderr"); - } - if (n == 0) throw EndOfFile(format("substituter ‘%1%’ died unexpectedly") % run.program); - err.append(buf, n); - string::size_type p; - while ((p = err.find('\n')) != string::npos) { - printMsg(lvlError, run.program + ": " + string(err, 0, p)); - err = string(err, p + 1); - } - } - - /* Read from stdout until we get a newline or the buffer is empty. */ - else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { - haveData: - do { - unsigned char c; - run.fromBuf(&c, 1); - if (c == '\n') { - if (!err.empty()) printMsg(lvlError, run.program + ": " + err); - return res; - } - res += c; - } while (run.fromBuf.hasData()); - } - } -} - - -template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) -{ - string s = getLineFromSubstituter(run); - T res; - if (!string2Int(s, res)) throw Error("integer expected from stream"); - return res; -} - - PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { - auto state(_state.lock()); - PathSet res; - for (auto & i : settings.substituters) { - if (res.size() == paths.size()) break; - RunningSubstituter & run(state->runningSubstituters[i]); - startSubstituter(i, run); - if (run.disabled) continue; - string s = "have "; - for (auto & j : paths) - if (res.find(j) == res.end()) { s += j; s += " "; } - writeLine(run.to, s); - while (true) { - /* FIXME: we only read stderr when an error occurs, so - substituters should only write (short) messages to - stderr when they fail. I.e. they shouldn't write debug - output. */ - Path path = getLineFromSubstituter(run); - if (path == "") break; - res.insert(path); + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (res.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + if (sub->isValidPath(path)) + res.insert(path); } } - return res; } -void LocalStore::querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos) -{ - auto state(_state.lock()); - - RunningSubstituter & run(state->runningSubstituters[substituter]); - startSubstituter(substituter, run); - if (run.disabled) return; - - string s = "info "; - for (auto & i : paths) - if (infos.find(i) == infos.end()) { s += i; s += " "; } - writeLine(run.to, s); - - while (true) { - Path path = getLineFromSubstituter(run); - if (path == "") break; - if (paths.find(path) == paths.end()) - throw Error(format("got unexpected path ‘%1%’ from substituter") % path); - paths.erase(path); - SubstitutablePathInfo & info(infos[path]); - info.deriver = getLineFromSubstituter(run); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLineFromSubstituter<int>(run); - while (nrRefs--) { - Path p = getLineFromSubstituter(run); - assertStorePath(p); - info.references.insert(p); - } - info.downloadSize = getIntLineFromSubstituter<long long>(run); - info.narSize = getIntLineFromSubstituter<long long>(run); - } -} - - void LocalStore::querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) { - PathSet todo = paths; - for (auto & i : settings.substituters) { - if (todo.empty()) break; - querySubstitutablePathInfos(i, todo, infos); + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath) { + } + } } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index daf394c92805..6f2341decfbd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -40,17 +40,6 @@ struct OptimiseStats }; -struct RunningSubstituter -{ - Path program; - Pid pid; - AutoCloseFD to, from, error; - FdSource fromBuf; - bool disabled; - RunningSubstituter() : disabled(false) { }; -}; - - class LocalStore : public LocalFSStore { private: @@ -80,10 +69,6 @@ private: /* The file to which we write our temporary roots. */ Path fnTempRoots; AutoCloseFD fdTempRoots; - - typedef std::map<Path, RunningSubstituter> RunningSubstituters; - RunningSubstituters runningSubstituters; - }; Sync<State, std::recursive_mutex> _state; @@ -102,6 +87,8 @@ public: /* Implementations of abstract store API methods. */ + std::string getUri() override; + bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet & paths) override; @@ -122,9 +109,6 @@ public: PathSet querySubstitutablePaths(const PathSet & paths) override; - void querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos); - void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; @@ -192,8 +176,6 @@ public: a substituter (if available). */ void repairPath(const Path & path); - void setSubstituterEnv(); - void addSignatures(const Path & storePath, const StringSet & sigs) override; static bool haveWriteAccess(); @@ -246,13 +228,6 @@ private: void removeUnusedLinks(const GCState & state); - void startSubstituter(const Path & substituter, - RunningSubstituter & runningSubstituter); - - string getLineFromSubstituter(RunningSubstituter & run); - - template<class T> T getIntLineFromSubstituter(RunningSubstituter & run); - Path createTempDirInStore(); Path importPath(bool requireSignature, Source & source); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 15fa91b5cea6..22b0f235e0b2 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat -libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -laws-cpp-sdk-s3 -laws-cpp-sdk-core +libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -laws-cpp-sdk-s3 -laws-cpp-sdk-core -pthread ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 23cbe7e26b47..ad7fe0e8bebf 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -228,7 +228,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) for (auto & i : paths) { addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - startNest(nest, lvlChatty, format("hashing files in ‘%1%’") % i); + Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); optimisePath_(stats, i, inodeHash); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1edf3662e8b8..5a254a6104f4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -49,6 +49,12 @@ RemoteStore::RemoteStore(size_t maxConnections) } +std::string RemoteStore::getUri() +{ + return "daemon"; +} + + ref<RemoteStore::Connection> RemoteStore::openConnection() { auto conn = make_ref<Connection>(); @@ -120,9 +126,9 @@ void RemoteStore::setOptions(ref<Connection> conn) if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 2) conn->to << settings.useBuildHook; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 4) - conn->to << settings.buildVerbosity - << logType - << settings.printBuildTrace; + conn->to << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 6) conn->to << settings.buildCores; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 10) @@ -561,10 +567,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) writeString(buf, source->read(buf, len), to); to.flush(); } - else { - string s = readString(from); - writeToStderr(s); - } + else + printMsg(lvlError, chomp(readString(from))); } if (msg == STDERR_ERROR) { string error = readString(from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4ea8e8a5be4d..8e45a7449e2e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -26,6 +26,8 @@ public: /* Implementations of abstract store API methods. */ + std::string getUri() override; + bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet & paths) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 6fadf7be28c9..cffcb1bf214f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -43,8 +43,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore Stats stats; S3BinaryCacheStoreImpl(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const std::string & bucketName) - : S3BinaryCacheStore(localStore, secretKeyFile) + const StoreParams & params, const std::string & bucketName) + : S3BinaryCacheStore(localStore, params) , bucketName(bucketName) , config(makeConfig()) , client(make_ref<Aws::S3::S3Client>(*config)) @@ -227,8 +227,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); - if (!hasSuffix(key, ".narinfo")) continue; - paths.insert(settings.nixStore + "/" + std::string(key, 0, key.size() - 8)); + if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; + paths.insert(settings.nixStore + "/" + key.substr(0, key.size() - 8)); } marker = res.GetNextMarker(); @@ -239,11 +239,13 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore }; -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ if (std::string(uri, 0, 5) != "s3://") return 0; auto store = std::make_shared<S3BinaryCacheStoreImpl>(std::shared_ptr<Store>(0), - settings.get("binary-cache-secret-key-file", string("")), - std::string(uri, 5)); + params, std::string(uri, 5)); store->init(); return store; }); diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 0425f6bb95d9..2751a9d01cdb 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -11,8 +11,8 @@ class S3BinaryCacheStore : public BinaryCacheStore protected: S3BinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile) - : BinaryCacheStore(localStore, secretKeyFile) + const StoreParams & params) + : BinaryCacheStore(localStore, params) { } public: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 14932f9b0be7..463e132e0299 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -461,10 +461,22 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStoreAt(const std::string & uri) +ref<Store> openStoreAt(const std::string & uri_) { + auto uri(uri_); + StoreParams params; + auto q = uri.find('?'); + if (q != std::string::npos) { + for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { + auto e = s.find('='); + if (e != std::string::npos) + params[s.substr(0, e)] = s.substr(e + 1); + } + uri = uri_.substr(0, q); + } + for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri); + auto store = fun(uri, params); if (store) return ref<Store>(store); } @@ -478,7 +490,10 @@ ref<Store> openStore() } -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ enum { mDaemon, mLocal, mAuto } mode; if (uri == "daemon") mode = mDaemon; @@ -501,4 +516,39 @@ static RegisterStoreImplementation regStore([](const std::string & uri) -> std:: }); +std::list<ref<Store>> getDefaultSubstituters() +{ + struct State { + bool done = false; + std::list<ref<Store>> stores; + }; + static Sync<State> state_; + + auto state(state_.lock()); + + if (state->done) return state->stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (done.count(uri)) return; + done.insert(uri); + state->stores.push_back(openStoreAt(uri)); + }; + + for (auto uri : settings.get("substituters", Strings())) + addStore(uri); + + for (auto uri : settings.get("binary-caches", Strings())) + addStore(uri); + + for (auto uri : settings.get("extra-binary-caches", Strings())) + addStore(uri); + + state->done = true; + + return state->stores; +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e0cc32296a9b..29685c9d1676 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -192,7 +192,7 @@ public: virtual ~Store() { } - virtual std::string getUri(); + virtual std::string getUri() = 0; /* Check whether a path is valid. */ bool isValidPath(const Path & path); @@ -529,12 +529,17 @@ ref<Store> openStoreAt(const std::string & uri); ref<Store> openStore(); -ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir); +/* Return the default substituter stores, defined by the + ‘substituters’ option and various legacy options like + ‘binary-caches’. */ +std::list<ref<Store>> getDefaultSubstituters(); /* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>(const std::string & uri)> OpenStore; +typedef std::map<std::string, std::string> StoreParams; + +typedef std::function<std::shared_ptr<Store>( + const std::string & uri, const StoreParams & params)> OpenStore; struct RegisterStoreImplementation { diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index beede13211fa..4d15d2acdd4e 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,90 +1,88 @@ #include "compression.hh" #include "util.hh" +#include "finally.hh" #include <lzma.h> +#include <bzlib.h> #include <cstdio> +#include <cstring> namespace nix { -/* RAII wrapper around lzma_stream. */ -struct LzmaStream +static ref<std::string> compressXZ(const std::string & in) { lzma_stream strm; - LzmaStream() : strm(LZMA_STREAM_INIT) { }; - ~LzmaStream() { lzma_end(&strm); }; - lzma_stream & operator()() { return strm; } -}; - -std::string compressXZ(const std::string & in) -{ - LzmaStream strm; // FIXME: apply the x86 BCJ filter? lzma_ret ret = lzma_easy_encoder( - &strm(), 6, LZMA_CHECK_CRC64); + &strm, 6, LZMA_CHECK_CRC64); if (ret != LZMA_OK) throw Error("unable to initialise lzma encoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; - string res; - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) return res; if (ret != LZMA_OK) - throw Error("error while decompressing xz file"); + throw Error("error while compressing xz file"); } } -ref<std::string> decompressXZ(const std::string & in) +static ref<std::string> decompressXZ(const std::string & in) { - LzmaStream strm; + lzma_stream strm; lzma_ret ret = lzma_stream_decoder( - &strm(), UINT64_MAX, LZMA_CONCATENATED); + &strm, UINT64_MAX, LZMA_CONCATENATED); if (ret != LZMA_OK) throw Error("unable to initialise lzma decoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; ref<std::string> res = make_ref<std::string>(); - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) @@ -95,4 +93,108 @@ ref<std::string> decompressXZ(const std::string & in) } } +static ref<std::string> compressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 encoder"); + + Finally free([&]() { BZ2_bzCompressEnd(&strm); }); + + int action = BZ_RUN; + char outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + if (strm.avail_in == 0) + action = BZ_FINISH; + + int ret = BZ2_bzCompress(&strm, action); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK && ret != BZ_FINISH_OK) + Error("error while compressing bzip2 file"); + } + + return res; +} + +static ref<std::string> decompressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 decoder"); + + Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + + char outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK) + throw Error("error while decompressing bzip2 file"); + } +} + +ref<std::string> compress(const std::string & method, ref<std::string> in) +{ + if (method == "none") + return in; + else if (method == "xz") + return compressXZ(*in); + else if (method == "bzip2") + return compressBzip2(*in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +ref<std::string> decompress(const std::string & method, ref<std::string> in) +{ + if (method == "none") + return in; + else if (method == "xz") + return decompressXZ(*in); + else if (method == "bzip2") + return decompressBzip2(*in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 79a796db7756..33c465df8455 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,13 +1,16 @@ #pragma once #include "ref.hh" +#include "types.hh" #include <string> namespace nix { -std::string compressXZ(const std::string & in); +ref<std::string> compress(const std::string & method, ref<std::string> in); -ref<std::string> decompressXZ(const std::string & in); +ref<std::string> decompress(const std::string & method, ref<std::string> in); + +MakeError(UnknownCompressionMethod, Error); } diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh new file mode 100644 index 000000000000..47c64deaecea --- /dev/null +++ b/src/libutil/finally.hh @@ -0,0 +1,12 @@ +#pragma once + +/* A trivial class to run a function at the end of a scope. */ +class Finally +{ +private: + std::function<void()> fun; + +public: + Finally(std::function<void()> fun) : fun(fun) { } + ~Finally() { fun(); } +}; diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 2e5d2672e5f0..98cad00d6d95 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = -llzma -pthread $(OPENSSL_LIBS) +libutil_LDFLAGS = -llzma -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc new file mode 100644 index 000000000000..15bb1e175da6 --- /dev/null +++ b/src/libutil/logging.cc @@ -0,0 +1,79 @@ +#include "logging.hh" +#include "util.hh" + +namespace nix { + +Logger * logger = 0; + +class SimpleLogger : public Logger +{ +public: + + bool systemd, tty; + + SimpleLogger() + { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: c = '3'; break; + case lvlInfo: c = '5'; break; + case lvlTalkative: case lvlChatty: c = '6'; break; + default: c = '7'; + } + prefix = std::string("<") + c + ">"; + } + + writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; + +Verbosity verbosity = lvlInfo; + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % fs.s); + haveWarned = true; + } +} + +void writeToStderr(const string & s) +{ + try { + writeFull(STDERR_FILENO, s); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + +Logger * makeDefaultLogger() +{ + return new SimpleLogger(); +} + +} diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh new file mode 100644 index 000000000000..277dff280053 --- /dev/null +++ b/src/libutil/logging.hh @@ -0,0 +1,82 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +class Activity; + +class Logger +{ + friend class Activity; + +public: + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void setExpected(const std::string & label, uint64_t value = 1) { } + virtual void setProgress(const std::string & label, uint64_t value = 1) { } + virtual void incExpected(const std::string & label, uint64_t value = 1) { } + virtual void incProgress(const std::string & label, uint64_t value = 1) { } + +private: + + virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; + + virtual void stopActivity(Activity & activity) = 0; + +}; + +class Activity +{ +public: + Logger & logger; + + Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) + : logger(logger) + { + logger.startActivity(*this, lvl, fs); + } + + ~Activity() + { + logger.stopActivity(*this); + } +}; + +extern Logger * logger; + +Logger * makeDefaultLogger(); + +extern Verbosity verbosity; /* suppress msgs > this */ + +#define printMsg(level, f) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 33aaf5fc9c4d..bd192b8506b2 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -89,14 +89,4 @@ typedef list<Path> Paths; typedef set<Path> PathSet; -typedef enum { - lvlError = 0, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - - } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8ffa6973ddc2..67558cc0b33c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -356,8 +356,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - startNest(nest, lvlDebug, - format("recursively deleting path ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -456,113 +455,6 @@ void replaceSymlink(const Path & target, const Path & link) } -LogType logType = ltPretty; -Verbosity verbosity = lvlInfo; - -static int nestingLevel = 0; - - -Nest::Nest() -{ - nest = false; -} - - -Nest::~Nest() -{ - close(); -} - - -static string escVerbosity(Verbosity level) -{ - return std::to_string((int) level); -} - - -void Nest::open(Verbosity level, const FormatOrString & fs) -{ - if (level <= verbosity) { - if (logType == ltEscapes) - std::cerr << "\033[" << escVerbosity(level) << "p" - << fs.s << "\n"; - else - printMsg_(level, fs); - nest = true; - nestingLevel++; - } -} - - -void Nest::close() -{ - if (nest) { - nestingLevel--; - if (logType == ltEscapes) - std::cerr << "\033[q"; - nest = false; - } -} - - -void printMsg_(Verbosity level, const FormatOrString & fs) -{ - checkInterrupt(); - if (level > verbosity) return; - - string prefix; - if (logType == ltPretty) - for (int i = 0; i < nestingLevel; i++) - prefix += "| "; - else if (logType == ltEscapes && level != lvlInfo) - prefix = "\033[" + escVerbosity(level) + "s"; - else if (logType == ltSystemd) { - char c; - switch (level) { - case lvlError: c = '3'; break; - case lvlInfo: c = '5'; break; - case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; - } - prefix = string("<") + c + ">"; - } - - string s = (format("%1%%2%\n") % prefix % fs.s).str(); - if (!isatty(STDERR_FILENO)) s = filterANSIEscapes(s); - writeToStderr(s); -} - - -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - printMsg(lvlError, format("warning: %1%") % fs.s); - haveWarned = true; - } -} - - -void writeToStderr(const string & s) -{ - try { - if (_writeToStderr) - _writeToStderr((const unsigned char *) s.data(), s.size()); - else - writeFull(STDERR_FILENO, s); - } catch (SysError & e) { - /* Ignore failing writes to stderr if we're in an exception - handler, otherwise throw an exception. We need to ignore - write errors in exception handlers to ensure that cleanup - code runs to completion if the other side of stderr has - been closed unexpectedly. */ - if (!std::uncaught_exception()) throw; - } -} - - -std::function<void(const unsigned char * buf, size_t count)> _writeToStderr; - - void readFull(int fd, unsigned char * buf, size_t count) { while (count) { @@ -953,7 +845,8 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { auto wrapper = [&]() { - if (!options.allowVfork) _writeToStderr = 0; + if (!options.allowVfork) + logger = makeDefaultLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -1189,6 +1082,12 @@ bool statusOk(int status) } +bool hasPrefix(const string & s, const string & suffix) +{ + return s.compare(0, suffix.size(), suffix) == 0; +} + + bool hasSuffix(const string & s, const string & suffix) { return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index dabfafa7fb06..f3f0f92a0aaa 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <sys/types.h> #include <sys/stat.h> @@ -125,54 +126,6 @@ T singleton(const A & a) } -/* Messages. */ - - -typedef enum { - ltPretty, /* nice, nested output */ - ltEscapes, /* nesting indicated using escape codes (for log2xml) */ - ltFlat, /* no nesting */ - ltSystemd, /* use systemd severity prefixes */ -} LogType; - -extern LogType logType; -extern Verbosity verbosity; /* suppress msgs > this */ - -class Nest -{ -private: - bool nest; -public: - Nest(); - ~Nest(); - void open(Verbosity level, const FormatOrString & fs); - void close(); -}; - -void printMsg_(Verbosity level, const FormatOrString & fs); - -#define startNest(varName, level, f) \ - Nest varName; \ - if (level <= verbosity) { \ - varName.open(level, (f)); \ - } - -#define printMsg(level, f) \ - do { \ - if (level <= nix::verbosity) { \ - nix::printMsg_(level, (f)); \ - } \ - } while (0) - -#define debug(f) printMsg(lvlDebug, f) - -void warnOnce(bool & haveWarned, const FormatOrString & fs); - -void writeToStderr(const string & s); - -extern std::function<void(const unsigned char * buf, size_t count)> _writeToStderr; - - /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); @@ -380,6 +333,10 @@ template<class N> bool string2Float(const string & s, N & n) } +/* Return true iff `s' starts with `prefix'. */ +bool hasPrefix(const string & s, const string & prefix); + + /* Return true iff `s' ends in `suffix'. */ bool hasSuffix(const string & s, const string & suffix); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index efc67b6a8755..3c2e0521028c 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -33,29 +33,43 @@ using namespace nix; static FdSource from(STDIN_FILENO); static FdSink to(STDOUT_FILENO); -bool canSendStderr; +static bool canSendStderr; +static Logger * defaultLogger; -/* This function is called anytime we want to write something to - stderr. If we're in a state where the protocol allows it (i.e., - when canSendStderr), send the message to the client over the - socket. */ -static void tunnelStderr(const unsigned char * buf, size_t count) + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +class TunnelLogger : public Logger { - if (canSendStderr) { - try { - to << STDERR_NEXT; - writeString(buf, count, to); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - canSendStderr = false; - throw; - } - } else - writeFull(STDERR_FILENO, buf, count); -} + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + if (canSendStderr) { + try { + to << STDERR_NEXT << (fs.s + "\n"); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + canSendStderr = false; + throw; + } + } else + defaultLogger->log(lvl, fs); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; /* startWork() means that we're starting an operation for which we @@ -429,9 +443,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.useBuildHook = readInt(from) != 0; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { - settings.buildVerbosity = (Verbosity) readInt(from); - logType = (LogType) readInt(from); - settings.printBuildTrace = readInt(from) != 0; + settings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace } if (GET_PROTOCOL_MINOR(clientVersion) >= 6) settings.set("build-cores", std::to_string(readInt(from))); @@ -557,7 +571,8 @@ static void processConnection(bool trusted) MonitorFdHup monitor(from.fd); canSendStderr = false; - _writeToStderr = tunnelStderr; + defaultLogger = logger; + logger = new TunnelLogger(); /* Exchange the greeting. */ unsigned int magic = readInt(from); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a9d1ed024dd3..6bc8d79bc1bb 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -996,7 +996,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - startNest(nest, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); + Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); if (globals.prebuiltOnly && validPaths.find(i.queryOutPath()) == validPaths.end() && diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 81c1c8d5637c..7dce08400e82 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -19,7 +19,7 @@ using namespace nix; static Expr * parseStdin(EvalState & state) { - startNest(nest, lvlTalkative, format("parsing standard input")); + Activity act(*logger, lvlTalkative, format("parsing standard input")); return state.parseExprFromString(drainFD(0), absPath(".")); } diff --git a/src/nix-log2xml/local.mk b/src/nix-log2xml/local.mk deleted file mode 100644 index 09c848c17f40..000000000000 --- a/src/nix-log2xml/local.mk +++ /dev/null @@ -1,5 +0,0 @@ -programs += nix-log2xml - -nix-log2xml_DIR := $(d) - -nix-log2xml_SOURCES := $(d)/log2xml.cc diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc deleted file mode 100644 index 31cea60c3809..000000000000 --- a/src/nix-log2xml/log2xml.cc +++ /dev/null @@ -1,201 +0,0 @@ -#include <vector> -#include <iostream> -#include <cstdio> -#include <string> -#include <cstring> - -using namespace std; - - -struct Decoder -{ - enum { stTop, stEscape, stCSI } state; - string line; - bool inHeader; - int level; - vector<int> args; - bool newNumber; - int priority; - bool ignoreLF; - int lineNo, charNo; - bool warning; - bool error; - - Decoder() - { - state = stTop; - line = ""; - inHeader = false; - level = 0; - priority = 1; - ignoreLF = false; - lineNo = 1; - charNo = 0; - warning = false; - error = false; - } - - void pushChar(char c); - - void finishLine(); - - void decodeFile(istream & st); -}; - - -void Decoder::pushChar(char c) -{ - if (c == '\n') { - lineNo++; - charNo = 0; - } else charNo++; - - switch (state) { - - case stTop: - if (c == '\e') { - state = stEscape; - } else if (c == '\n' && !ignoreLF) { - finishLine(); - } else line += c; - break; - - case stEscape: - if (c == '[') { - state = stCSI; - args.clear(); - newNumber = true; - } else - state = stTop; /* !!! wrong */ - break; - - case stCSI: - if (c >= 0x40 && c != 0x7e) { - state = stTop; - switch (c) { - case 'p': - if (line.size()) finishLine(); - level++; - inHeader = true; - cout << "<nest>" << endl; - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'q': - if (line.size()) finishLine(); - if (level > 0) { - level--; - cout << "</nest>" << endl; - } else - cerr << "not enough nesting levels at line " - << lineNo << ", character " << charNo << endl; - break; - case 's': - if (line.size()) finishLine(); - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'a': - ignoreLF = true; - break; - case 'b': - ignoreLF = false; - break; - case 'e': - error = true; - break; - case 'w': - warning = true; - break; - } - } else if (c >= '0' && c <= '9') { - int n = 0; - if (!newNumber) { - n = args.back() * 10; - args.pop_back(); - } - n += c - '0'; - args.push_back(n); - } - break; - - } -} - - -void Decoder::finishLine() -{ - string storeDir = "/nix/store/"; - int sz = storeDir.size(); - string tag = inHeader ? "head" : "line"; - cout << "<" << tag; - if (priority != 1) cout << " priority='" << priority << "'"; - if (warning) cout << " warning='1'"; - if (error) cout << " error='1'"; - cout << ">"; - - for (unsigned int i = 0; i < line.size(); i++) { - - if (line[i] == '<') cout << "<"; - else if (line[i] == '&') cout << "&"; - else if (line[i] == '\r') ; /* ignore carriage return */ - else if (line[i] == '\n') cout << "\n"; - else if (line[i] >= 0 && line[i] < 32 && line[i] != 9) cout << "�"; - else if (i + sz + 33 < line.size() && - string(line, i, sz) == storeDir && - line[i + sz + 32] == '-') - { - int j = i + sz + 32; - /* skip name */ - while (!strchr("/\n\r\t ()[]:;?<>", line[j])) j++; - int k = j; - while (!strchr("\n\r\t ()[]:;?<>", line[k])) k++; - // !!! escaping - cout << "<storeref>" - << "<storedir>" - << string(line, i, sz) - << "</storedir>" - << "<hash>" - << string(line, i + sz, 32) - << "</hash>" - << "<name>" - << string(line, i + sz + 32, j - (i + sz + 32)) - << "</name>" - << "<path>" - << string(line, j, k - j) - << "</path>" - << "</storeref>"; - i = k - 1; - } else cout << line[i]; - } - - cout << "</" << tag << ">" << endl; - line = ""; - inHeader = false; - priority = 1; - warning = false; - error = false; -} - - -void Decoder::decodeFile(istream & st) -{ - int c; - - cout << "<logfile>" << endl; - - while ((c = st.get()) != EOF) { - pushChar(c); - } - - if (line.size()) finishLine(); - - while (level--) cout << "</nest>" << endl; - - cout << "</logfile>" << endl; -} - - -int main(int argc, char * * argv) -{ - Decoder dec; - dec.decodeFile(cin); -} diff --git a/src/nix-log2xml/logfile.css b/src/nix-log2xml/logfile.css deleted file mode 100644 index ed390d64a9ef..000000000000 --- a/src/nix-log2xml/logfile.css +++ /dev/null @@ -1,86 +0,0 @@ -body { - font-family: sans-serif; - background: white; -} - - -ul.nesting, ul.toplevel { - padding: 0; - margin: 0; -} - -ul.toplevel { - list-style-type: none; -} - -ul.nesting li.line, ul.nesting li.lastline { - position: relative; - list-style-type: none; -} - -ul.nesting li.line { - padding-left: 1.1em; -} - -ul.nesting li.lastline { - padding-left: 1.2em; // for the 0.1em border-left in .lastline > .lineconn -} - -li.line { - border-left: 0.1em solid #6185a0; -} - -li.line > span.lineconn, li.lastline > span.lineconn { - position: absolute; - height: 0.65em; - left: 0em; - width: 1em; - border-bottom: 0.1em solid #6185a0; -} - -li.lastline > span.lineconn { - border-left: 0.1em solid #6185a0; -} - - -em.storeref { - color: #500000; - position: relative; - width: 100%; -} - -em.storeref:hover { - background-color: #eeeeee; -} - -*.popup { - display: none; -/* background: url('http://losser.st-lab.cs.uu.nl/~mbravenb/menuback.png') repeat; */ - background: #ffffcd; - border: solid #555555 1px; - position: absolute; - top: 0em; - left: 0em; - margin: 0; - padding: 0; - z-index: 100; -} - -em.storeref:hover span.popup { - display: inline; -} - - -.toggle { - text-decoration: none; -} - -.showTree, .hideTree { - font-family: monospace; - font-size: larger; -} - -.error { - color: #ff0000; - font-weight: bold; -} \ No newline at end of file diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a7fcdf4e0e4..653a95f21679 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,6 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "xmlgraph.hh" +#include "compression.hh" #include <iostream> #include <algorithm> @@ -502,21 +503,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) } else if (pathExists(logBz2Path)) { - AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); - FILE * f = 0; - if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) - throw SysError(format("opening file ‘%1%’") % logBz2Path); - int err; - BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); - if (!bz) throw Error(format("cannot open bzip2 file ‘%1%’") % logBz2Path); - unsigned char buf[128 * 1024]; - do { - int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); - if (err != BZ_OK && err != BZ_STREAM_END) - throw Error(format("error reading bzip2 file ‘%1%’") % logBz2Path); - writeFull(STDOUT_FILENO, buf, n); - } while (err != BZ_STREAM_END); - BZ2_bzReadClose(&err, bz); + std::cout << *decompress("bzip2", make_ref<std::string>(readFile(logBz2Path))); found = true; break; } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index b5bd362d63a6..be51fee62712 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -1,5 +1,4 @@ #include "command.hh" -#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" #include "sync.hh" @@ -47,16 +46,9 @@ struct CmdCopy : StorePathsCommand ref<Store> srcStore = srcUri.empty() ? store : openStoreAt(srcUri); ref<Store> dstStore = dstUri.empty() ? store : openStoreAt(dstUri); - ProgressBar progressBar; + std::string copiedLabel = "copied"; - std::atomic<size_t> done{0}; - std::atomic<size_t> total{storePaths.size()}; - - auto showProgress = [&]() { - return (format("[%d/%d copied]") % done % total).str(); - }; - - progressBar.updateStatus(showProgress()); + logger->setExpected(copiedLabel, storePaths.size()); ThreadPool pool; @@ -71,7 +63,7 @@ struct CmdCopy : StorePathsCommand checkInterrupt(); if (!dstStore->isValidPath(storePath)) { - auto activity(progressBar.startActivity(format("copying ‘%s’...") % storePath)); + Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); StringSink sink; srcStore->exportPaths({storePath}, false, sink); @@ -79,16 +71,12 @@ struct CmdCopy : StorePathsCommand StringSource source(*sink.s); dstStore->importPaths(false, source, 0); - done++; + logger->incProgress(copiedLabel); } else - total--; - - progressBar.updateStatus(showProgress()); + logger->incExpected(copiedLabel, -1); }); pool.process(); - - progressBar.done(); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 2005ec5f9a6d..440ced97dfcc 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -7,6 +7,7 @@ #include "legacy.hh" #include "shared.hh" #include "store-api.hh" +#include "progress-bar.hh" namespace nix { @@ -26,6 +27,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { + settings.verboseBuild = false; + initNix(); initGC(); @@ -42,6 +45,8 @@ void mainWrapped(int argc, char * * argv) assert(args.command); + StartProgressBar bar; + args.command->prepare(); args.command->run(); } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 9347795f1cb9..c61fe7ff1e00 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -1,5 +1,4 @@ #include "command.hh" -#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 746b891a78df..659d6572ad93 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -1,72 +1,157 @@ #include "progress-bar.hh" +#include "util.hh" +#include "sync.hh" -#include <iostream> +#include <map> namespace nix { -ProgressBar::ProgressBar() +class ProgressBar : public Logger { - _writeToStderr = [&](const unsigned char * buf, size_t count) { - auto state_(state.lock()); - assert(!state_->done); - std::cerr << "\r\e[K" << std::string((const char *) buf, count); - render(*state_); +private: + + struct ActInfo + { + Activity * activity; + Verbosity lvl; + std::string s; }; -} -ProgressBar::~ProgressBar() -{ - done(); -} + struct Progress + { + uint64_t expected = 0, progress = 0; + }; -void ProgressBar::updateStatus(const std::string & s) -{ - auto state_(state.lock()); - assert(!state_->done); - state_->status = s; - render(*state_); -} + struct State + { + std::list<ActInfo> activities; + std::map<Activity *, std::list<ActInfo>::iterator> its; + std::map<std::string, Progress> progress; + }; -void ProgressBar::done() -{ - _writeToStderr = decltype(_writeToStderr)(); - auto state_(state.lock()); - assert(state_->activities.empty()); - state_->done = true; - std::cerr << "\r\e[K"; - std::cerr.flush(); -} + Sync<State> state_; -void ProgressBar::render(State & state_) -{ - std::cerr << '\r' << state_.status; - if (!state_.activities.empty()) { - if (!state_.status.empty()) std::cerr << ' '; - std::cerr << *state_.activities.rbegin(); +public: + + ~ProgressBar() + { + auto state(state_.lock()); + assert(state->activities.empty()); + writeToStderr("\r\e[K"); } - std::cerr << "\e[K"; - std::cerr.flush(); -} + void log(Verbosity lvl, const FormatOrString & fs) override + { + auto state(state_.lock()); + log(*state, lvl, fs.s); + } -ProgressBar::Activity ProgressBar::startActivity(const FormatOrString & fs) -{ - return Activity(*this, fs); -} + void log(State & state, Verbosity lvl, const std::string & s) + { + writeToStderr("\r\e[K" + s + "\n"); + update(state); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + auto state(state_.lock()); + state->activities.emplace_back(ActInfo{&activity, lvl, fs.s}); + state->its.emplace(&activity, std::prev(state->activities.end())); + update(*state); + } + + void stopActivity(Activity & activity) override + { + auto state(state_.lock()); + auto i = state->its.find(&activity); + if (i == state->its.end()) return; + state->activities.erase(i->second); + state->its.erase(i); + update(*state); + } + + void setExpected(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].expected = value; + } + + void setProgress(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].progress = value; + } + + void incExpected(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].expected += value; + } + + void incProgress(const std::string & label, uint64_t value) + { + auto state(state_.lock()); + state->progress[label].progress += value; + } + + void update() + { + auto state(state_.lock()); + } + + void update(State & state) + { + std::string line = "\r"; + + std::string status = getStatus(state); + if (!status.empty()) { + line += '['; + line += status; + line += "]"; + } + + if (!state.activities.empty()) { + if (!status.empty()) line += " "; + line += state.activities.rbegin()->s; + } + + line += "\e[K"; + writeToStderr(line); + } + + std::string getStatus(State & state) + { + std::string res; + for (auto & p : state.progress) + if (p.second.expected || p.second.progress) { + if (!res.empty()) res += ", "; + res += std::to_string(p.second.progress); + if (p.second.expected) { + res += "/"; + res += std::to_string(p.second.expected); + } + res += " "; res += p.first; + } + return res; + } +}; -ProgressBar::Activity::Activity(ProgressBar & pb, const FormatOrString & fs) - : pb(pb) +StartProgressBar::StartProgressBar() { - auto state_(pb.state.lock()); - state_->activities.push_back(fs.s); - it = state_->activities.end(); --it; - pb.render(*state_); + if (isatty(STDERR_FILENO)) { + prev = logger; + logger = new ProgressBar(); + } } -ProgressBar::Activity::~Activity() +StartProgressBar::~StartProgressBar() { - auto state_(pb.state.lock()); - state_->activities.erase(it); + if (prev) { + auto bar = logger; + logger = prev; + delete bar; + } } } diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh index 2dda24346c90..d2e44f7c4fd9 100644 --- a/src/nix/progress-bar.hh +++ b/src/nix/progress-bar.hh @@ -1,49 +1,15 @@ #pragma once -#include "sync.hh" -#include "util.hh" +#include "logging.hh" namespace nix { -class ProgressBar +class StartProgressBar { -private: - struct State - { - std::string status; - bool done = false; - std::list<std::string> activities; - }; - - Sync<State> state; - + Logger * prev = 0; public: - - ProgressBar(); - - ~ProgressBar(); - - void updateStatus(const std::string & s); - - void done(); - - class Activity - { - friend class ProgressBar; - private: - ProgressBar & pb; - std::list<std::string>::iterator it; - Activity(ProgressBar & pb, const FormatOrString & fs); - public: - ~Activity(); - }; - - Activity startActivity(const FormatOrString & fs); - -private: - - void render(State & state_); - + StartProgressBar(); + ~StartProgressBar(); }; } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 69073d8847f2..9932aa4a9eb0 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,5 +1,4 @@ #include "command.hh" -#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" @@ -38,21 +37,15 @@ struct CmdCopySigs : StorePathsCommand for (auto & s : substituterUris) substituters.push_back(openStoreAt(s)); - ProgressBar progressBar; - ThreadPool pool; - std::atomic<size_t> done{0}; + std::string doneLabel = "done"; std::atomic<size_t> added{0}; - auto showProgress = [&]() { - return (format("[%d/%d done]") % done % storePaths.size()).str(); - }; - - progressBar.updateStatus(showProgress()); + logger->setExpected(doneLabel, storePaths.size()); auto doPath = [&](const Path & storePath) { - auto activity(progressBar.startActivity(format("getting signatures for ‘%s’") % storePath)); + Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); checkInterrupt(); @@ -83,8 +76,7 @@ struct CmdCopySigs : StorePathsCommand added += newSigs.size(); } - done++; - progressBar.updateStatus(showProgress()); + logger->incProgress(doneLabel); }; for (auto & storePath : storePaths) @@ -92,8 +84,6 @@ struct CmdCopySigs : StorePathsCommand pool.process(); - progressBar.done(); - printMsg(lvlInfo, format("imported %d signatures") % added); } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 20ce26fa7d21..fd904f465687 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,5 +1,4 @@ #include "command.hh" -#include "progress-bar.hh" #include "shared.hh" #include "store-api.hh" #include "sync.hh" @@ -57,30 +56,16 @@ struct CmdVerify : StorePathsCommand auto publicKeys = getDefaultPublicKeys(); + std::atomic<size_t> done{0}; std::atomic<size_t> untrusted{0}; std::atomic<size_t> corrupted{0}; - std::atomic<size_t> done{0}; std::atomic<size_t> failed{0}; - ProgressBar progressBar; - - auto showProgress = [&](bool final) { - std::string s; - if (final) - s = (format("checked %d paths") % storePaths.size()).str(); - else - s = (format("[%d/%d checked") % done % storePaths.size()).str(); - if (corrupted > 0) - s += (format(", %d corrupted") % corrupted).str(); - if (untrusted > 0) - s += (format(", %d untrusted") % untrusted).str(); - if (failed > 0) - s += (format(", %d failed") % failed).str(); - if (!final) s += "]"; - return s; - }; - - progressBar.updateStatus(showProgress(false)); + std::string doneLabel("paths checked"); + std::string untrustedLabel("untrusted"); + std::string corruptedLabel("corrupted"); + std::string failedLabel("failed"); + logger->setExpected(doneLabel, storePaths.size()); ThreadPool pool; @@ -88,7 +73,7 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - auto activity(progressBar.startActivity(format("checking ‘%s’") % storePath)); + Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); auto info = store->queryPathInfo(storePath); @@ -100,6 +85,7 @@ struct CmdVerify : StorePathsCommand auto hash = sink.finish(); if (hash.first != info->narHash) { + logger->incProgress(corruptedLabel); corrupted = 1; printMsg(lvlError, format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") @@ -147,18 +133,19 @@ struct CmdVerify : StorePathsCommand } if (!good) { + logger->incProgress(untrustedLabel); untrusted++; printMsg(lvlError, format("path ‘%s’ is untrusted") % info->path); } } + logger->incProgress(doneLabel); done++; - progressBar.updateStatus(showProgress(false)); - } catch (Error & e) { printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logger->incProgress(failedLabel); failed++; } }; @@ -168,9 +155,8 @@ struct CmdVerify : StorePathsCommand pool.process(); - progressBar.done(); - - printMsg(lvlInfo, showProgress(true)); + printMsg(lvlInfo, format("%d paths checked, %d untrusted, %d corrupted, %d failed") + % done % untrusted % corrupted % failed); throw Exit( (corrupted ? 1 : 0) | diff --git a/tests/fallback.sh b/tests/fallback.sh deleted file mode 100644 index f3a6b50515bf..000000000000 --- a/tests/fallback.sh +++ /dev/null @@ -1,20 +0,0 @@ -source common.sh - -clearStore - -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -outPath=$(nix-store -q --fallback "$drvPath") -echo "output path is $outPath" - -# Build with a substitute that fails. This should fail. -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh -if nix-store -r "$drvPath"; then echo unexpected fallback; exit 1; fi - -# Build with a substitute that fails. This should fall back to a source build. -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh -nix-store -r --fallback "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hello World!"; then exit 1; fi diff --git a/tests/local.mk b/tests/local.mk index 471821b270e8..7c5a553d39e0 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,8 +3,7 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - build-hook.sh substitutes.sh substitutes2.sh \ - fallback.sh nix-push.sh gc.sh gc-concurrent.sh \ + build-hook.sh nix-push.sh gc.sh gc-concurrent.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ diff --git a/tests/logging.sh b/tests/logging.sh index 77b2337a9d00..86f32bade941 100644 --- a/tests/logging.sh +++ b/tests/logging.sh @@ -2,16 +2,7 @@ source common.sh clearStore -# Produce an escaped log file. -path=$(nix-build --log-type escapes -vv dependencies.nix --no-out-link 2> $TEST_ROOT/log.esc) - -# Convert it to an XML representation. -nix-log2xml < $TEST_ROOT/log.esc > $TEST_ROOT/log.xml - -# Is this well-formed XML? -if test "$xmllint" != "false"; then - $xmllint --noout $TEST_ROOT/log.xml || fail "malformed XML" -fi +path=$(nix-build dependencies.nix --no-out-link) # Test nix-store -l. [ "$(nix-store -l $path)" = FOO ] diff --git a/tests/substituter.sh b/tests/substituter.sh deleted file mode 100755 index 9aab295de87b..000000000000 --- a/tests/substituter.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/sh -e -echo -echo substituter args: $* >&2 - -if test $1 = "--query"; then - while read cmd args; do - echo "CMD = $cmd, ARGS = $args" >&2 - if test "$cmd" = "have"; then - for path in $args; do - read path - if grep -q "$path" $TEST_ROOT/sub-paths; then - echo $path - fi - done - echo - elif test "$cmd" = "info"; then - for path in $args; do - echo $path - echo "" # deriver - echo 0 # nr of refs - echo $((1 * 1024 * 1024)) # download size - echo $((2 * 1024 * 1024)) # nar size - done - echo - else - echo "bad command $cmd" - exit 1 - fi - done -elif test $1 = "--substitute"; then - mkdir $2 - echo "Hallo Wereld" > $2/hello - echo # no expected hash -else - echo "unknown substituter operation" - exit 1 -fi diff --git a/tests/substituter2.sh b/tests/substituter2.sh deleted file mode 100755 index 5d1763599c25..000000000000 --- a/tests/substituter2.sh +++ /dev/null @@ -1,33 +0,0 @@ -#! /bin/sh -e -echo -echo substituter2 args: $* >&2 - -if test $1 = "--query"; then - while read cmd args; do - if test "$cmd" = have; then - for path in $args; do - if grep -q "$path" $TEST_ROOT/sub-paths; then - echo $path - fi - done - echo - elif test "$cmd" = info; then - for path in $args; do - echo $path - echo "" # deriver - echo 0 # nr of refs - echo 0 # download size - echo 0 # nar size - done - echo - else - echo "bad command $cmd" - exit 1 - fi - done -elif test $1 = "--substitute"; then - exit 1 -else - echo "unknown substituter operation" - exit 1 -fi diff --git a/tests/substitutes.sh b/tests/substitutes.sh deleted file mode 100644 index 0c6adf2601fa..000000000000 --- a/tests/substitutes.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore - -# Instantiate. -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -# Find the output path. -outPath=$(nix-store -qvv "$drvPath") -echo "output path is $outPath" - -echo $outPath > $TEST_ROOT/sub-paths - -export NIX_SUBSTITUTERS=$(pwd)/substituter.sh - -nix-store -r "$drvPath" --dry-run 2>&1 | grep -q "1.00 MiB.*2.00 MiB" - -nix-store -rvv "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi diff --git a/tests/substitutes2.sh b/tests/substitutes2.sh deleted file mode 100644 index bd914575cca8..000000000000 --- a/tests/substitutes2.sh +++ /dev/null @@ -1,21 +0,0 @@ -source common.sh - -clearStore - -# Instantiate. -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -# Find the output path. -outPath=$(nix-store -qvvvvv "$drvPath") -echo "output path is $outPath" - -echo $outPath > $TEST_ROOT/sub-paths - -# First try a substituter that fails, then one that succeeds -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh:$(pwd)/substituter.sh - -nix-store -j0 -rvv "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi |