diff options
| -rw-r--r-- | doc/manual/expressions/builtins.xml | 74 | ||||
| -rw-r--r-- | release.nix | 12 | ||||
| -rw-r--r-- | src/libexpr/eval.cc | 2 | ||||
| -rw-r--r-- | src/libexpr/primops.cc | 93 | ||||
| -rw-r--r-- | src/libstore/build.cc | 2 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 5 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 6 | ||||
| -rw-r--r-- | src/libutil/logging.cc | 2 | ||||
| -rw-r--r-- | src/libutil/util.cc | 67 | ||||
| -rw-r--r-- | src/libutil/util.hh | 10 | ||||
| -rw-r--r-- | src/nix/progress-bar.cc | 43 | ||||
| -rw-r--r-- | tests/lang/data | 1 | ||||
| -rw-r--r-- | tests/lang/eval-okay-path.nix | 7 | ||||
| -rw-r--r-- | tests/misc.sh | 2 |
14 files changed, 222 insertions, 104 deletions
diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 5a3a8645c1d9..81770bcf6292 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -308,8 +308,9 @@ stdenv.mkDerivation { … } </varlistentry> - <varlistentry><term><function>builtins.filterSource</function> - <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + <varlistentry xml:id='builtin-filterSource'> + <term><function>builtins.filterSource</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> <listitem> @@ -768,6 +769,75 @@ Evaluates to <literal>[ "foo" ]</literal>. </varlistentry> + <varlistentry> + <term> + <function>builtins.path</function> + <replaceable>args</replaceable> + </term> + + <listitem> + <para> + An enrichment of the built-in path type, based on the attributes + present in <replaceable>args</replaceable>. All are optional + except <varname>path</varname>: + </para> + + <variablelist> + <varlistentry> + <term>path</term> + <listitem> + <para>The underlying path.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>name</term> + <listitem> + <para> + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like <literal>@</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>filter</term> + <listitem> + <para> + A function of the type expected by + <link linkend="builtin-filterSource">builtins.filterSource</link>, + with the same semantics. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>recursive</term> + <listitem> + <para> + When <literal>false</literal>, when + <varname>path</varname> is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, <varname>path</varname> must refer to a + regular file, not a directory. This allows similar + behavior to <literal>fetchurl</literal>. Defaults to + <literal>true</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>sha256</term> + <listitem> + <para> + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + <literal>builtins.path</literal> to be used even when the + <literal>pure-eval</literal> nix config option is on. + </para> + </listitem> + </varlistentry> + </variablelist> + </listitem> + </varlistentry> <varlistentry><term><function>builtins.pathExists</function> <replaceable>path</replaceable></term> diff --git a/release.nix b/release.nix index bdac283cfe3b..9e04f0b67f9d 100644 --- a/release.nix +++ b/release.nix @@ -224,11 +224,13 @@ let nix = build.x86_64-linux; system = "x86_64-linux"; }); - tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system: - import ./tests/setuid.nix rec { - inherit nixpkgs; - nix = build.${system}; inherit system; - }); + tests.setuid = pkgs.lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: + import ./tests/setuid.nix rec { + inherit nixpkgs; + nix = build.${system}; inherit system; + }); tests.binaryTarball = with import nixpkgs { system = "x86_64-linux"; }; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7775cbe53cc4..0b0a0f7b1790 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1578,7 +1578,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) dstPath = srcToStore[path]; else { dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(checkSourcePath(path)).first + ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); srcToStore[path] = dstPath; printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5fe7da216439..466fd13e8698 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1023,20 +1023,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu } -static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - - state.forceValue(*args[0]); - if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - - path = state.checkSourcePath(path); - - PathFilter filter = [&](const Path & path) { + const auto path = settings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); /* Call the filter function. The first argument is the path, @@ -1045,7 +1038,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args mkString(arg1, path); Value fun2; - state.callFunction(*args[0], arg1, fun2, noPos); + state.callFunction(*filterFun, arg1, fun2, noPos); Value arg2; mkString(arg2, @@ -1058,16 +1051,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args state.callFunction(fun2, arg2, res, noPos); return state.forceBool(res, pos); - }; + }) : defaultPathFilter; - Path dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first - : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair); + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); + } + } else + dstPath = expectedStorePath; mkString(v, dstPath, {dstPath}); } +static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + } + if (path.empty()) + throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) + name = baseNameOf(path); + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + + /************************************************************* * Sets *************************************************************/ @@ -2085,6 +2141,7 @@ void EvalState::createBaseEnv() addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); // Sets addPrimOp("__attrNames", 1, prim_attrNames); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5be7ce60dab9..9f669f7e4645 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3428,7 +3428,7 @@ void DerivationGoal::flushLine() else { if (settings.verboseBuild && (settings.printRepeatedBuilds || curRound == 1)) - printError(filterANSIEscapes(currentLogLine, true)); + printError(currentLogLine); else { logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 77ab87ef728e..7abb300a9bb8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash, } -std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) const +std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, h, name); return std::pair<Path, Hash>(dstPath, h); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index aa83c2ded8c8..563aa566bd37 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -307,9 +307,9 @@ public: /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter) const; + std::pair<Path, Hash> computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6924e0080475..27a631a37d10 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -44,7 +44,7 @@ public: prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n"); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 272997397794..f7a12d21b244 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1178,36 +1178,51 @@ void ignoreException() } -string filterANSIEscapes(const string & s, bool nixOnly) -{ - string t, r; - enum { stTop, stEscape, stCSI } state = stTop; - for (auto c : s) { - if (state == stTop) { - if (c == '\e') { - state = stEscape; - r = c; - } else - t += c; - } else if (state == stEscape) { - r += c; - if (c == '[') - state = stCSI; - else { - t += r; - state = stTop; +std::string filterANSIEscapes(const std::string & s, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; } - } else { - r += c; - if (c >= 0x40 && c <= 0x7e) { - if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b')) - t += r; - state = stTop; - r.clear(); + + if (last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; } } + + else if (*i == '\r') + // do nothing for now + ; + + else { + t += *i++; w++; + } } - t += r; + return t; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 75eb9751524e..47e02bc898a6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -388,10 +388,12 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" -/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is - set, only filter escape codes generated by Nixpkgs' stdenv (used to - denote nesting etc.). */ -string filterANSIEscapes(const string & s, bool nixOnly = false); +/* Truncate a string to 'width' printable characters. Certain ANSI + escape sequences (such as colour setting) are copied but not + included in the character count. Other ANSI escape sequences are + filtered. Also, tabs are expanded to spaces. */ +std::string filterANSIEscapes(const std::string & s, + unsigned int width = std::numeric_limits<unsigned int>::max()); /* Base64 encoding/decoding. */ diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 252d12c5d37f..e6553c06f4ae 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -23,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n) return fields[n].i; } -/* Truncate a string to 'width' printable characters. ANSI escape - sequences are copied but not included in the character count. Also, - tabs are expanded to spaces. */ -static std::string ansiTruncate(const std::string & s, int width) -{ - if (width <= 0) return s; - - std::string t; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - if (*i == '\e') { - t += *i++; - if (i != s.end() && *i == '[') { - t += *i++; - while (i != s.end() && (*i < 0x40 || *i > 0x7e)) { - t += *i++; - } - if (i != s.end()) t += *i++; - } - } - - else if (*i == '\t') { - t += ' '; w++; - while (w < (size_t) width && w & 8) { - t += ' '; w++; - } - } - - else { - t += *i++; w++; - } - } - - return t; -} - class ProgressBar : public Logger { private: @@ -343,7 +305,10 @@ public: } } - writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K"); + auto width = getWindowSize().second; + if (width <= 0) std::numeric_limits<decltype(width)>::max(); + + writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K"); } std::string getStatus(State & state) diff --git a/tests/lang/data b/tests/lang/data new file mode 100644 index 000000000000..257cc5642cb1 --- /dev/null +++ b/tests/lang/data @@ -0,0 +1 @@ +foo diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix new file mode 100644 index 000000000000..e67168cf3edf --- /dev/null +++ b/tests/lang/eval-okay-path.nix @@ -0,0 +1,7 @@ +builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + } diff --git a/tests/misc.sh b/tests/misc.sh index 6d0ab3adcec8..eda0164167f2 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. -nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$" +nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$" |