diff options
50 files changed, 860 insertions, 489 deletions
diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index f2dc6ed8540d..5fff64a18f93 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -204,7 +204,7 @@ printed.)</para> with <option>-K</option>, if an output path is not identical to the corresponding output from the previous build, the new output path is left in - <filename>/nix/store/<replaceable>name</replaceable>-check.</filename></para> + <filename>/nix/store/<replaceable>name</replaceable>.check.</filename></para> <para>See also the <option>build-repeat</option> configuration option, which repeats a derivation a number of times and prevents diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 7bc0d2667245..54f697d852e3 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -92,6 +92,36 @@ available as <function>builtins.derivation</function>.</para> </varlistentry> + <varlistentry><term><function>builtins.bitAnd</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise AND of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry><term><function>builtins.bitOr</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise OR of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry><term><function>builtins.bitXor</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise XOR of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + <varlistentry><term><varname>builtins</varname></term> <listitem><para>The set <varname>builtins</varname> contains all diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index fc39cdd9dfef..1fd6bafee7e3 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -21,4 +21,51 @@ in your <filename>~/.profile</filename> (or similar), like this:</para> <screen> source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen> -</chapter> \ No newline at end of file +<section xml:id="sec-nix-ssl-cert-file"> + +<title><envar>NIX_SSL_CERT_FILE</envar></title> + +<para>If you need to specify a custom certificate bundle to account +for an HTTPS-intercepting man in the middle proxy, you must specify +the path to the certificate bundle in the environment variable +<envar>NIX_SSL_CERT_FILE</envar>.</para> + + +<para>If you don't specify a <envar>NIX_SSL_CERT_FILE</envar> +manually, Nix will install and use its own certificate +bundle.</para> + +<procedure> + <step><para>Set the environment variable and install Nix</para> + <screen> +$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +$ curl https://nixos.org/nix/install | sh +</screen></step> + + <step><para>In the shell profile and rc files (for example, + <filename>/etc/bashrc</filename>, <filename>/etc/zshrc</filename>), + add the following line:</para> +<programlisting> +export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +</programlisting> +</step> +</procedure> + +<note><para>You must not add the export and then do the install, as +the Nix installer will detect the presense of Nix configuration, and +abort.</para></note> + +<section> +<title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title> + +<para>On macOS you must specify the environment variable for the Nix +daemon service, then restart it:</para> + +<screen> +$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt +$ sudo launchctl kickstart -k system/org.nixos.nix-daemon +</screen> +</section> + +</section> +</chapter> diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bbfb2934315b..ce553bb53ebc 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,7 +27,7 @@ static ref<Store> store() static std::shared_ptr<Store> _store; if (!_store) { try { - settings.loadConfFile(); + loadConfFile(); settings.lockCPU = false; _store = openStore(); } catch (Error & e) { diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5f6542355e0c..b4eec2fe94cd 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -727,11 +727,17 @@ setup_default_profile() { _sudo "to installing a bootstrapping Nix in to the default Profile" \ HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX" - _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ - HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" + if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then + _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" + export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt + fi + # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, + # otherwise it will be lost in environments where sudo doesn't pass + # all the environment variables by default. _sudo "to update the default channel in the default profile" \ - HOME="$ROOT_HOME" NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs + HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs } diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 43c7156062de..52070e36a1e2 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -49,6 +49,23 @@ if test -w $HOME; then fi fi -export NIX_SSL_CERT_FILE="@localstatedir@/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" -export NIX_PATH="@localstatedir@/nix/profiles/per-user/root/channels" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if [ ! -z "$NIX_SSL_CERT_FILE" ]; then + : # Allow users to override the NIX_SSL_CERT_FILE +elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem +elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +elif [ -e "$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the user's Nix profile + export NIX_SSL_CERT_FILE=$NIX_USER_PROFILE_DIR/etc/ssl/certs/ca-bundle.crt +elif [ -e "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in the default Nix profile + export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt +fi + +export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" export PATH="$HOME/.nix-profile/bin:$HOME/.nix-profile/lib/kde4/libexec:@localstatedir@/nix/profiles/default/bin:@localstatedir@/nix/profiles/default:@localstatedir@/nix/profiles/default/lib/kde4/libexec:$PATH" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 353097f89713..a3f411a5fb1d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -307,15 +307,17 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) assert(gcInitialised); + static_assert(sizeof(Env) == 16, "environment must be 16 bytes"); + /* Initialise the Nix expression search path. */ - if (!settings.pureEval) { + if (!evalSettings.pureEval) { Strings paths = parseNixPath(getEnv("NIX_PATH", "")); for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); } addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); - if (settings.restrictEval || settings.pureEval) { + if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); for (auto & i : searchPath) { @@ -352,6 +354,10 @@ Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) + return i->second; + bool found = false; for (auto & i : *allowedPaths) { @@ -369,8 +375,10 @@ Path EvalState::checkSourcePath(const Path & path_) Path path = canonPath(path_, true); for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; return path; + } } throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); @@ -379,13 +387,13 @@ Path EvalState::checkSourcePath(const Path & path_) void EvalState::checkURI(const std::string & uri) { - if (!settings.restrictEval) return; + if (!evalSettings.restrictEval) return; /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit access to https://github.com. Note: this allows 'http://' and 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : settings.allowedUris.get()) + for (auto & prefix : evalSettings.allowedUris.get()) if (uri == prefix || (uri.size() > prefix.size() && prefix.size() > 0 @@ -562,12 +570,12 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) if (!var.fromWith) return env->values[var.displ]; while (1) { - if (!env->haveWithAttrs) { + if (env->type == Env::HasWithExpr) { if (noEval) return 0; Value * v = allocValue(); evalAttrs(*env->up, (Expr *) env->values[0], *v); env->values[0] = v; - env->haveWithAttrs = true; + env->type = Env::HasWithAttrs; } Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { @@ -597,6 +605,7 @@ Env & EvalState::allocEnv(size_t size) nrValuesInEnvs += size; Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); env->size = (decltype(Env::size)) size; + env->type = Env::Plain; /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ @@ -1199,7 +1208,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(1)); env2.up = &env; env2.prevWith = prevWith; - env2.haveWithAttrs = false; + env2.type = Env::HasWithExpr; env2.values[0] = (Value *) attrs; body->eval(state, env2, v); @@ -1857,9 +1866,10 @@ size_t valueSize(Value & v) size_t sz = sizeof(Env) + sizeof(Value *) * env.size; - for (size_t i = 0; i < env.size; ++i) - if (env.values[i]) - sz += doValue(*env.values[i]); + if (env.type != Env::HasWithExpr) + for (size_t i = 0; i < env.size; ++i) + if (env.values[i]) + sz += doValue(*env.values[i]); if (env.up) sz += doEnv(*env.up); @@ -1888,4 +1898,9 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } +EvalSettings evalSettings; + +static GlobalConfig::Register r1(&evalSettings); + + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 86e93a5ac9ce..146f21255034 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -5,8 +5,10 @@ #include "nixexpr.hh" #include "symbol-table.hh" #include "hash.hh" +#include "config.hh" #include <map> +#include <unordered_map> namespace nix { @@ -34,8 +36,8 @@ struct Env { Env * up; unsigned short size; // used by ‘valueSize’ - unsigned short prevWith:15; // nr of levels up to next `with' environment - unsigned short haveWithAttrs:1; + unsigned short prevWith:14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -100,6 +102,9 @@ private: std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + /* Cache used by checkSourcePath(). */ + std::unordered_map<Path, Path> resolvedPaths; + public: EvalState(const Strings & _searchPath, ref<Store> store); @@ -316,4 +321,25 @@ struct InvalidPathError : EvalError #endif }; +struct EvalSettings : Config +{ + Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting<bool> restrictEval{this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting<bool> pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access to files specified by cryptographic hash."}; + + Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting<Strings> allowedUris{this, {}, "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; +}; + +extern EvalSettings evalSettings; + } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 8b1404595548..3f6017957782 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -110,7 +110,7 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) if (number_type == tFloat) mkFloat(v, stod(tmp_number)); else - mkInt(v, stoi(tmp_number)); + mkInt(v, stol(tmp_number)); } catch (std::invalid_argument e) { throw JSONParseError("invalid JSON number"); } catch (std::out_of_range e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 57dc7bd1279d..3a6c4035b8b8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -73,7 +73,7 @@ void EvalState::realiseContext(const PathSet & context) if (drvs.empty()) return; - if (!settings.enableImportFromDerivation) + if (!evalSettings.enableImportFromDerivation) throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); /* For performance, prefetch all substitute info. */ @@ -464,7 +464,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, settings.restrictEval || settings.pureEval ? "" : getEnv(name)); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name)); } @@ -1031,7 +1031,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) { - const auto path = settings.pureEval && expectedHash ? + const auto path = evalSettings.pureEval && expectedHash ? path_ : state.checkSourcePath(path_); PathFilter filter = filterFun ? ([&](const Path & path) { @@ -1676,6 +1676,20 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & } } +static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +} + +static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +} + +static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +} static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2042,7 +2056,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.checkURI(url); - if (settings.pureEval && !expectedHash) + if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); @@ -2110,12 +2124,12 @@ void EvalState::createBaseEnv() addConstant(name, v); }; - if (!settings.pureEval) { + if (!evalSettings.pureEval) { mkInt(v, time(0)); addConstant("__currentTime", v); } - if (!settings.pureEval) { + if (!evalSettings.pureEval) { mkString(v, settings.thisSystem); addConstant("__currentSystem", v); } @@ -2140,7 +2154,7 @@ void EvalState::createBaseEnv() mkApp(v, *vScopedImport, *v2); forceValue(v); addConstant("import", v); - if (settings.enableNativeCode) { + if (evalSettings.enableNativeCode) { addPrimOp("__importNative", 2, prim_importNative); addPrimOp("__exec", 1, prim_exec); } @@ -2167,7 +2181,7 @@ void EvalState::createBaseEnv() // Paths addPrimOp("__toPath", 1, prim_toPath); - if (settings.pureEval) + if (evalSettings.pureEval) addPurityError("__storePath"); else addPrimOp("__storePath", 1, prim_storePath); @@ -2221,6 +2235,9 @@ void EvalState::createBaseEnv() addPrimOp("__sub", 2, prim_sub); addPrimOp("__mul", 2, prim_mul); addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); addPrimOp("__lessThan", 2, prim_lessThan); // String manipulation diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 8bb74dad639e..7aa98e0bfab3 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -28,7 +28,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, std::experimental::optional<std::string> ref, std::string rev, const std::string & name) { - if (settings.pureEval && rev == "") + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index a75c5fc2ddff..9d35f6d0d6d7 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -27,7 +27,7 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); HgInfo exportMercurial(ref<Store> store, const std::string & uri, std::string rev, const std::string & name) { - if (settings.pureEval && rev == "") + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index bcc05c2cdad6..4c35a4199590 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -29,14 +29,14 @@ MixCommonArgs::MixCommonArgs(const string & programName) .arity(2) .handler([](std::vector<std::string> ss) { try { - settings.set(ss[0], ss[1]); + globalConfig.set(ss[0], ss[1]); } catch (UsageError & e) { warn(e.what()); } }); std::string cat = "config"; - settings.convertToArgs(*this, cat); + globalConfig.convertToArgs(*this, cat); // Backward compatibility hack: nix-env already had a --system flag. if (programName == "nix-env") longFlags.erase("system"); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 91a4eaf922a6..4ed34e54dc55 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -109,7 +109,7 @@ void initNix() opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); - settings.loadConfFile(); + loadConfFile(); startSignalHandlerThread(); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2e9a13e564ca..11fa3cae27a5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -54,17 +54,38 @@ void BinaryCacheStore::init() } } -std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) +void BinaryCacheStore::getFile(const std::string & path, + Callback<std::shared_ptr<std::string>> callback) +{ + try { + callback(getFile(path)); + } catch (...) { callback.rethrow(); } +} + +void BinaryCacheStore::getFile(const std::string & path, Sink & sink) { std::promise<std::shared_ptr<std::string>> promise; getFile(path, - [&](std::shared_ptr<std::string> result) { - promise.set_value(result); - }, - [&](std::exception_ptr exc) { - promise.set_exception(exc); - }); - return promise.get_future().get(); + {[&](std::future<std::shared_ptr<std::string>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink((unsigned char *) data->data(), data->size()); +} + +std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) +{ + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) @@ -196,30 +217,27 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast<const NarInfo>(); - auto nar = getFile(info->url); - - if (!nar) throw Error(format("file '%s' missing from binary cache") % info->url); + auto source = sinkToSource([this, url{info->url}](Sink & sink) { + getFile(url, sink); + }); stats.narRead++; - stats.narReadCompressedBytes += nar->size(); + //stats.narReadCompressedBytes += nar->size(); // FIXME uint64_t narSize = 0; - StringSource source(*nar); - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { sink(data, len); narSize += len; }); - decompress(info->compression, source, wrapperSink); + decompress(info->compression, *source, wrapperSink); stats.narReadBytes += narSize; } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<std::shared_ptr<ValidPathInfo>> callback) { auto uri = getUri(); auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, @@ -229,17 +247,22 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); getFile(narInfoFile, - [=](std::shared_ptr<std::string> data) { - if (!data) return success(0); + {[=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); - stats.narInfoRead++; + if (!data) return callback(nullptr); - callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>) - std::make_shared<NarInfo>(*this, *data, narInfoFile)); + stats.narInfoRead++; - (void) act; // force Activity into this lambda to ensure it stays alive - }, - failure); + callback((std::shared_ptr<ValidPathInfo>) + std::make_shared<NarInfo>(*this, *data, narInfoFile)); + + (void) act; // force Activity into this lambda to ensure it stays alive + } catch (...) { + callback.rethrow(); + } + }}); } Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index e20b968442b7..6bc83fc50ca1 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -38,11 +38,16 @@ public: const std::string & data, const std::string & mimeType) = 0; - /* Return the contents of the specified file, or null if it - doesn't exist. */ + /* Note: subclasses must implement at least one of the two + following getFile() methods. */ + + /* Dump the contents of the specified file to a sink. */ + virtual void getFile(const std::string & path, Sink & sink); + + /* Fetch the specified file and call the specified callback with + the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) = 0; + Callback<std::shared_ptr<std::string>> callback); std::shared_ptr<std::string> getFile(const std::string & path); @@ -71,8 +76,7 @@ public: { unsupported(); } void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) override; void queryReferrers(const Path & path, PathSet & referrers) override @@ -131,4 +135,6 @@ public: }; +MakeError(NoSuchBinaryCacheFile, Error); + } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f70ab8108fd7..07b533783931 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -672,8 +672,10 @@ HookInstance::HookInstance() toHook.readSide = -1; sink = FdSink(toHook.writeSide.get()); - for (auto & setting : settings.getSettings()) - sink << 1 << setting.first << setting.second; + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto & setting : settings) + sink << 1 << setting.first << setting.second.value; sink << 0; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 4ca4a838e3c4..1f4abd374f54 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -22,52 +22,60 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) return i->second; }; - auto fetch = [&](const string & url) { - /* No need to do TLS verification, because we check the hash of - the result anyway. */ - DownloadRequest request(url); - request.verifyTLS = false; - request.decompress = false; - - /* Note: have to use a fresh downloader here because we're in - a forked process. */ - auto data = makeDownloader()->download(request); - assert(data.data); - - return data.data; - }; + Path storePath = getAttr("out"); + auto mainUrl = getAttr("url"); + + /* Note: have to use a fresh downloader here because we're in + a forked process. */ + auto downloader = makeDownloader(); + + auto fetch = [&](const std::string & url) { + + auto source = sinkToSource([&](Sink & sink) { + + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadRequest request(url); + request.verifyTLS = false; + request.decompress = false; + + downloader->download(std::move(request), sink); + }); + + if (get(drv.env, "unpack", "") == "1") { - std::shared_ptr<std::string> data; + if (hasSuffix(mainUrl, ".xz")) { + auto source2 = sinkToSource([&](Sink & sink) { + decompress("xz", *source, sink); + }); + restorePath(storePath, *source2); + } else + restorePath(storePath, *source); + } else + writeFile(storePath, *source); + + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) + throw SysError(format("making '%1%' executable") % storePath); + } + }; + + /* Try the hashed mirrors first. */ if (getAttr("outputHashMode") == "flat") for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - data = fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); - break; + fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + return; } catch (Error & e) { debug(e.what()); } - if (!data) data = fetch(getAttr("url")); - - Path storePath = getAttr("out"); - - auto unpack = drv.env.find("unpack"); - if (unpack != drv.env.end() && unpack->second == "1") { - if (string(*data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data = decompress("xz", *data); - StringSource source(*data); - restorePath(storePath, source); - } else - writeFile(storePath, *data); - - auto executable = drv.env.find("executable"); - if (executable != drv.env.end() && executable->second == "1") { - if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); - } + /* Otherwise try the specified URL. */ + fetch(mainUrl); } } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 54f4dd218007..72a08ef0089c 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -7,6 +7,7 @@ #include "s3.hh" #include "compression.hh" #include "pathlocks.hh" +#include "finally.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -29,12 +30,25 @@ using namespace std::string_literals; namespace nix { -double getTime() +struct DownloadSettings : Config { - struct timeval tv; - gettimeofday(&tv, 0); - return tv.tv_sec + (tv.tv_usec / 1000000.0); -} + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; +}; + +static DownloadSettings downloadSettings; + +static GlobalConfig::Register r1(&downloadSettings); std::string resolveUri(const std::string & uri) { @@ -61,8 +75,6 @@ struct CurlDownloader : public Downloader std::random_device rd; std::mt19937 mt19937; - bool enableHttp2; - struct DownloadItem : public std::enable_shared_from_this<DownloadItem> { CurlDownloader & downloader; @@ -70,8 +82,7 @@ struct CurlDownloader : public Downloader DownloadResult result; Activity act; bool done = false; // whether either the success or failure function has been called - std::function<void(const DownloadResult &)> success; - std::function<void(std::exception_ptr exc)> failure; + Callback<DownloadResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -86,10 +97,13 @@ struct CurlDownloader : public Downloader std::string encoding; - DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) + DownloadItem(CurlDownloader & downloader, + const DownloadRequest & request, + Callback<DownloadResult> callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, fmt("downloading '%s'", request.uri), {request.uri}, request.parentAct) + , callback(callback) { if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); @@ -118,13 +132,16 @@ struct CurlDownloader : public Downloader { assert(!done); done = true; - callFailure(failure, std::make_exception_ptr(e)); + callback.rethrow(std::make_exception_ptr(e)); } size_t writeCallback(void * contents, size_t size, size_t nmemb) { size_t realSize = size * nmemb; - result.data->append((char *) contents, realSize); + if (request.dataCallback) + request.dataCallback((char *) contents, realSize); + else + result.data->append((char *) contents, realSize); return realSize; } @@ -228,12 +245,12 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str()); + (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloader.enableHttp2) + if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); @@ -265,7 +282,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); @@ -305,11 +322,11 @@ struct CurlDownloader : public Downloader try { if (request.decompress) result.data = decodeContent(encoding, ref<std::string>(result.data)); - callSuccess(success, failure, const_cast<const DownloadResult &>(result)); act.progress(result.data->size(), result.data->size()); + callback(std::move(result)); } catch (...) { done = true; - callFailure(failure, std::current_exception()); + callback.rethrow(); } } else { // We treat most errors as transient, but won't retry when hopeless @@ -408,11 +425,9 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - settings.binaryCachesParallelConnections.get()); + downloadSettings.httpConnections.get()); #endif - enableHttp2 = settings.enableHttp2; - wakeupPipe.create(); fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); @@ -561,13 +576,12 @@ struct CurlDownloader : public Downloader } void enqueueDownload(const DownloadRequest & request, - std::function<void(const DownloadResult &)> success, - std::function<void(std::exception_ptr exc)> failure) override + Callback<DownloadResult> callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { // FIXME: do this on a worker thread - sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult { + try { #ifdef ENABLE_S3 S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable auto slash = request.uri.find('/', 5); @@ -581,27 +595,22 @@ struct CurlDownloader : public Downloader if (!s3Res.data) throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; - return res; + callback(std::move(res)); #else throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); #endif - }); + } catch (...) { callback.rethrow(); } return; } - auto item = std::make_shared<DownloadItem>(*this, request); - item->success = success; - item->failure = failure; - enqueueItem(item); + enqueueItem(std::make_shared<DownloadItem>(*this, request, callback)); } }; ref<Downloader> getDownloader() { - static std::shared_ptr<Downloader> downloader; - static std::once_flag downloaderCreated; - std::call_once(downloaderCreated, [&]() { downloader = makeDownloader(); }); - return ref<Downloader>(downloader); + static ref<Downloader> downloader = makeDownloader(); + return downloader; } ref<Downloader> makeDownloader() @@ -613,8 +622,13 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & { auto promise = std::make_shared<std::promise<DownloadResult>>(); enqueueDownload(request, - [promise](const DownloadResult & result) { promise->set_value(result); }, - [promise](std::exception_ptr exc) { promise->set_exception(exc); }); + {[promise](std::future<DownloadResult> fut) { + try { + promise->set_value(fut.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + }}); return promise->get_future(); } @@ -623,6 +637,92 @@ DownloadResult Downloader::download(const DownloadRequest & request) return enqueueDownload(request).get(); } +void Downloader::download(DownloadRequest && request, Sink & sink) +{ + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the downloader + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + }; + + auto _state = std::make_shared<Sync<State>>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + request.dataCallback = [_state](char * buf, size_t len) { + + auto state(_state->lock()); + + if (state->quit) return; + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). Note: this does stall the download thread. */ + while (state->data.size() > 1024 * 1024) { + if (state->quit) return; + debug("download buffer is full; going to sleep"); + state.wait(state->request); + } + + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(buf, len); + state->avail.notify_one(); + }; + + enqueueDownload(request, + {[_state](std::future<DownloadResult> fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + }}); + + auto state(_state->lock()); + + while (true) { + checkInterrupt(); + + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + break; + } + + /* If no data is available, then wait for the download thread + to wake us up. */ + if (state->data.empty()) + state.wait(state->avail); + + /* If data is available, then flush it to the sink and wake up + the download thread if it's blocked on a full buffer. */ + if (!state->data.empty()) { + sink((unsigned char *) state->data.data(), state->data.size()); + state->data.clear(); + state->request.notify_one(); + } + } +} + Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 7ade756fc356..f56274b2353c 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -21,6 +21,7 @@ struct DownloadRequest bool decompress = true; std::shared_ptr<std::string> data; std::string mimeType; + std::function<void(char *, size_t)> dataCallback; DownloadRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } @@ -42,14 +43,17 @@ struct Downloader the download. The future may throw a DownloadError exception. */ virtual void enqueueDownload(const DownloadRequest & request, - std::function<void(const DownloadResult &)> success, - std::function<void(std::exception_ptr exc)> failure) = 0; + Callback<DownloadResult> callback) = 0; std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); /* Synchronously download a file. */ DownloadResult download(const DownloadRequest & request); + /* Download a file, writing its data to a sink. The sink will be + invoked on the thread of the caller. */ + void download(DownloadRequest && request, Sink & sink); + /* Check if the specified file is already in ~/.cache/nix/tarballs and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 544566e0b573..d95db56726cb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -28,9 +28,10 @@ namespace nix { Settings settings; +static GlobalConfig::Register r1(&settings); + Settings::Settings() - : Config({}) - , nixPrefix(NIX_PREFIX) + : nixPrefix(NIX_PREFIX) , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) @@ -69,20 +70,15 @@ Settings::Settings() allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); } -void Settings::loadConfFile() +void loadConfFile() { - applyConfigFile(nixConfDir + "/nix.conf"); + globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - resetOverriden(); + globalConfig.resetOverriden(); - applyConfigFile(getConfigDir() + "/nix/nix.conf"); -} - -void Settings::set(const string & name, const string & value) -{ - Config::set(name, value); + globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf"); } unsigned int Settings::getDefaultCores() @@ -162,23 +158,11 @@ void initPlugins() throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); } } - /* We handle settings registrations here, since plugins can add settings */ - if (RegisterSetting::settingRegistrations) { - for (auto & registration : *RegisterSetting::settingRegistrations) - settings.addSetting(registration); - delete RegisterSetting::settingRegistrations; - } - settings.handleUnknownSettings(); -} - -RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations; -RegisterSetting::RegisterSetting(AbstractSetting * s) -{ - if (!settingRegistrations) - settingRegistrations = new SettingRegistrations; - settingRegistrations->emplace_back(s); + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); } - } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9360096aae8c..f589078dbb98 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -13,26 +13,6 @@ namespace nix { typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; -extern bool useCaseHack; // FIXME - -struct CaseHackSetting : public BaseSetting<bool> -{ - CaseHackSetting(Config * options, - const std::string & name, - const std::string & description, - const std::set<std::string> & aliases = {}) - : BaseSetting<bool>(useCaseHack, name, description, aliases) - { - options->addSetting(this); - } - - void set(const std::string & str) override - { - BaseSetting<bool>::set(str); - nix::useCaseHack = value; - } -}; - struct MaxBuildJobsSetting : public BaseSetting<unsigned int> { MaxBuildJobsSetting(Config * options, @@ -56,10 +36,6 @@ public: Settings(); - void loadConfFile(); - - void set(const string & name, const string & value); - Path nixPrefix; /* The directory where we store sources and derived files. */ @@ -217,9 +193,6 @@ public: Setting<bool> showTrace{this, false, "show-trace", "Whether to show a stack trace on evaluation errors."}; - Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox", "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", {"build-use-chroot", "build-use-sandbox"}}; @@ -232,13 +205,6 @@ public: "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting<bool> restrictEval{this, false, "restrict-eval", - "Whether to restrict file system access to paths in $NIX_PATH, " - "and network access to the URI prefixes listed in 'allowed-uris'."}; - - Setting<bool> pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; - Setting<size_t> buildRepeat{this, 0, "repeat", "The number of times to repeat a build in order to verify determinism.", {"build-repeat"}}; @@ -280,13 +246,6 @@ public: Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", "Secret keys with which to sign local builds."}; - Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting<bool> enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; @@ -350,18 +309,6 @@ public: /* Path to the SSL CA file used */ Path caFile; - Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", - "Whether the evaluator allows importing the result of a derivation."}; - - CaseHackSetting useCaseHack{this, "use-case-hack", - "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - - Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - #if __linux__ Setting<bool> filterSyscalls{this, true, "filter-syscalls", "Whether to prevent certain dangerous system calls, such as " @@ -383,9 +330,6 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; - Setting<Strings> allowedUris{this, {}, "allowed-uris", - "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; - Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; }; @@ -398,15 +342,8 @@ extern Settings settings; anything else */ void initPlugins(); +void loadConfFile(); extern const string nixVersion; -struct RegisterSetting -{ - typedef std::vector<AbstractSetting *> SettingRegistrations; - static SettingRegistrations * settingRegistrations; - RegisterSetting(AbstractSetting * s); -}; - - } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index b9e9cd5daba5..6fdae40e3603 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -77,28 +77,42 @@ protected: } } - void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) override + DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); request.tries = 8; + return request; + } + + void getFile(const std::string & path, Sink & sink) override + { + auto request(makeRequest(path)); + try { + getDownloader()->download(std::move(request), sink); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + throw; + } + } + + void getFile(const std::string & path, + Callback<std::shared_ptr<std::string>> callback) override + { + auto request(makeRequest(path)); getDownloader()->enqueueDownload(request, - [success](const DownloadResult & result) { - success(result.data); - }, - [success, failure](std::exception_ptr exc) { + {[callback](std::future<DownloadResult> result) { try { - std::rethrow_exception(exc); + callback(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return success(0); - failure(exc); + return callback(std::shared_ptr<std::string>()); + callback.rethrow(); } catch (...) { - failure(exc); + callback.rethrow(); } - }); + }}); } }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5dee25308f7f..02d91ded04cd 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -84,10 +84,9 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override + Callback<std::shared_ptr<ValidPathInfo>> callback) override { - sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> { + try { auto conn(connections->get()); debug("querying remote host '%s' for info on '%s'", host, path); @@ -97,7 +96,7 @@ struct LegacySSHStore : public Store auto info = std::make_shared<ValidPathInfo>(); conn->from >> info->path; - if (info->path.empty()) return nullptr; + if (info->path.empty()) return callback(nullptr); assert(path == info->path); PathSet references; @@ -116,8 +115,8 @@ struct LegacySSHStore : public Store auto s = readString(conn->from); assert(s == ""); - return info; - }); + callback(std::move(info)); + } catch (...) { callback.rethrow(); } } void addToStore(const ValidPathInfo & info, Source & source, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 2577e90aef23..b7001795be4d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -34,18 +34,14 @@ protected: const std::string & data, const std::string & mimeType) override; - void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) override + void getFile(const std::string & path, Sink & sink) override { - sync2async<std::shared_ptr<std::string>>(success, failure, [&]() { - try { - return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)); - } catch (SysError & e) { - if (e.errNo == ENOENT) return std::shared_ptr<std::string>(); - throw; - } - }); + try { + readFile(binaryCacheDir + "/" + path, sink); + } catch (SysError & e) { + if (e.errNo == ENOENT) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + } } PathSet queryAllValidPaths() override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ef8c2811bd86..3b2ba65f3b46 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -629,17 +629,15 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<std::shared_ptr<ValidPathInfo>> callback) { - sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() { - + try { auto info = std::make_shared<ValidPathInfo>(); info->path = path; assertStorePath(path); - return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); /* Get the path info. */ @@ -679,8 +677,9 @@ void LocalStore::queryPathInfoUncached(const Path & path, info->references.insert(useQueryReferences.getStr(0)); return info; - }); - }); + })); + + } catch (...) { callback.rethrow(); } } @@ -976,7 +975,8 @@ const PublicKeys & LocalStore::getPublicKeys() void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { - assert(info.narHash); + if (!info.narHash) + throw Error("cannot add path '%s' because it lacks a hash", info.path); if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) throw Error("cannot add path '%s' because it lacks a valid signature", info.path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1209a06356f7..746bdbeed793 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,8 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a82aa4e9cfa5..adcce026fa1d 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -33,9 +33,11 @@ void Store::computeFSClosure(const PathSet & startPaths, state->pending++; } - queryPathInfo(path, - [&, path](ref<ValidPathInfo> info) { - // FIXME: calls to isValidPath() should be async + queryPathInfo(path, {[&, path](std::future<ref<ValidPathInfo>> fut) { + // FIXME: calls to isValidPath() should be async + + try { + auto info = fut.get(); if (flipDirection) { @@ -75,14 +77,13 @@ void Store::computeFSClosure(const PathSet & startPaths, if (!--state->pending) done.notify_one(); } - }, - - [&, path](std::exception_ptr exc) { + } catch (...) { auto state(state_.lock()); - if (!state->exc) state->exc = exc; + if (!state->exc) state->exc = std::current_exception(); assert(state->pending); if (!--state->pending) done.notify_one(); - }); + }; + }}); }; for (auto & startPath : startPaths) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 080cef93d214..ea86ef052f53 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -7,6 +7,7 @@ #include "globals.hh" #include "derivations.hh" #include "pool.hh" +#include "finally.hh" #include <sys/types.h> #include <sys/stat.h> @@ -187,10 +188,11 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - auto overrides = settings.getSettings(true); + std::map<std::string, Config::SettingInfo> overrides; + globalConfig.getSettings(overrides, true); conn.to << overrides.size(); for (auto & i : overrides) - conn.to << i.first << i.second; + conn.to << i.first << i.second.value; } conn.processStderr(); @@ -293,38 +295,40 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<std::shared_ptr<ValidPathInfo>> callback) { - sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() { - auto conn(connections->get()); - conn->to << wopQueryPathInfo << path; - try { - conn->processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); - } - auto info = std::make_shared<ValidPathInfo>(); - info->path = path; - info->deriver = readString(conn->from); - if (info->deriver != "") assertStorePath(info->deriver); - info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<PathSet>(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings<StringSet>(conn->from); - conn->from >> info->ca; + try { + std::shared_ptr<ValidPathInfo> info; + { + auto conn(connections->get()); + conn->to << wopQueryPathInfo << path; + try { + conn->processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.what()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; conn->from >> valid; + if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); + } + info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = readString(conn->from); + if (info->deriver != "") assertStorePath(info->deriver); + info->narHash = Hash(readString(conn->from), htSHA256); + info->references = readStorePaths<PathSet>(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings<StringSet>(conn->from); + conn->from >> info->ca; + } } - return info; - }); + callback(std::move(info)); + } catch (...) { callback.rethrow(); } } @@ -411,8 +415,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.references << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca << repair << !checkSigs; - copyNAR(source, conn->to); - conn->processStderr(); + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) copyNAR(source, conn->to); + conn->processStderr(0, tunnel ? &source : nullptr); } } @@ -435,8 +440,10 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, conn->to.written = 0; conn->to.warn = true; connections->incCapacity(); - dumpPath(srcPath, conn->to, filter); - connections->decCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + dumpPath(srcPath, conn->to, filter); + } conn->to.warn = false; conn->processStderr(); } catch (SysError & e) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 95fa59a2069d..b488e34ce263 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -40,8 +40,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 103f141a1a11..239739bae832 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -364,24 +364,23 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore uploadFile(path, data, mimeType, ""); } - void getFile(const std::string & path, - std::function<void(std::shared_ptr<std::string>)> success, - std::function<void(std::exception_ptr exc)> failure) override + void getFile(const std::string & path, Sink & sink) override { - sync2async<std::shared_ptr<std::string>>(success, failure, [&]() { - stats.get++; + stats.get++; - auto res = s3Helper.getObject(bucketName, path); + // FIXME: stream output to sink. + auto res = s3Helper.getObject(bucketName, path); - stats.getBytes += res.data ? res.data->size() : 0; - stats.getTimeMs += res.durationMs; + stats.getBytes += res.data ? res.data->size() : 0; + stats.getTimeMs += res.durationMs; - if (res.data) - printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + if (res.data) { + printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, res.data->size(), res.durationMs); - return res.data; - }); + sink((unsigned char *) res.data->data(), res.data->size()); + } else + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } PathSet queryAllValidPaths() override diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 42d40e71d8be..a061d64f36d8 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -10,6 +10,7 @@ namespace nix { [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) { int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); auto path = sqlite3_db_filename(db, nullptr); if (!path) path = "(in-memory)"; @@ -21,7 +22,7 @@ namespace nix { : fmt("SQLite database '%s' is busy", path)); } else - throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(err), path); + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } SQLite::SQLite(const Path & path) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a0d12ca78c2..9b0b7d6327e0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -253,6 +253,8 @@ std::string Store::getUri() bool Store::isValidPath(const Path & storePath) { + assertStorePath(storePath); + auto hashPart = storePathToHash(storePath); { @@ -303,20 +305,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) std::promise<ref<ValidPathInfo>> promise; queryPathInfo(storePath, - [&](ref<ValidPathInfo> info) { - promise.set_value(info); - }, - [&](std::exception_ptr exc) { - promise.set_exception(exc); - }); + {[&](std::future<ref<ValidPathInfo>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); return promise.get_future().get(); } void Store::queryPathInfo(const Path & storePath, - std::function<void(ref<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) + Callback<ref<ValidPathInfo>> callback) { auto hashPart = storePathToHash(storePath); @@ -328,7 +330,7 @@ void Store::queryPathInfo(const Path & storePath, stats.narInfoReadAverted++; if (!*res) throw InvalidPath(format("path '%s' is not valid") % storePath); - return success(ref<ValidPathInfo>(*res)); + return callback(ref<ValidPathInfo>(*res)); } } @@ -344,35 +346,36 @@ void Store::queryPathInfo(const Path & storePath, (res.second->path != storePath && storePathToName(storePath) != "")) throw InvalidPath(format("path '%s' is not valid") % storePath); } - return success(ref<ValidPathInfo>(res.second)); + return callback(ref<ValidPathInfo>(res.second)); } } - } catch (std::exception & e) { - return callFailure(failure); - } + } catch (...) { return callback.rethrow(); } queryPathInfoUncached(storePath, - [this, storePath, hashPart, success, failure](std::shared_ptr<ValidPathInfo> info) { + {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) { - if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, info); + try { + auto info = fut.get(); - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, info); - } + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, info); - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { - stats.narInfoMissing++; - return failure(std::make_exception_ptr(InvalidPath(format("path '%s' is not valid") % storePath))); - } + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } - callSuccess(success, failure, ref<ValidPathInfo>(info)); + if (!info + || (info->path != storePath && storePathToName(storePath) != "")) + { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } - }, failure); + callback(ref<ValidPathInfo>(info)); + } catch (...) { callback.rethrow(); } + }}); } @@ -392,26 +395,19 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti auto doQuery = [&](const Path & path ) { checkInterrupt(); - queryPathInfo(path, - [path, &state_, &wakeup](ref<ValidPathInfo> info) { - auto state(state_.lock()); + queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<ValidPathInfo>> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); state->valid.insert(path); - assert(state->left); - if (!--state->left) - wakeup.notify_one(); - }, - [path, &state_, &wakeup](std::exception_ptr exc) { - auto state(state_.lock()); - try { - std::rethrow_exception(exc); - } catch (InvalidPath &) { - } catch (...) { - state->exc = exc; - } - assert(state->left); - if (!--state->left) - wakeup.notify_one(); - }); + } catch (InvalidPath &) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (!--state->left) + wakeup.notify_one(); + }}); }; for (auto & path : paths) @@ -847,7 +843,7 @@ ref<Store> openStore(const std::string & uri_, for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { - store->handleUnknownSettings(); + store->warnUnknownSettings(); return ref<Store>(store); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ea259f07e8ab..6ee2d550679b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -355,14 +355,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - std::function<void(ref<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure); + Callback<ref<ValidPathInfo>> callback); protected: virtual void queryPathInfoUncached(const Path & path, - std::function<void(std::shared_ptr<ValidPathInfo>)> success, - std::function<void(std::exception_ptr exc)> failure) = 0; + Callback<std::shared_ptr<ValidPathInfo>> callback) = 0; public: diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 996e1d25355f..5ebdfaf134d6 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x114 +#define PROTOCOL_VERSION 0x115 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 154e2d20430c..1be8934a2eba 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -13,17 +13,25 @@ #include "archive.hh" #include "util.hh" - +#include "config.hh" namespace nix { +struct ArchiveSettings : Config +{ + Setting<bool> useCaseHack{this, + #if __APPLE__ + true, + #else + false, + #endif + "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; +}; -bool useCaseHack = -#if __APPLE__ - true; -#else - false; -#endif +static ArchiveSettings archiveSettings; + +static GlobalConfig::Register r1(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; @@ -78,7 +86,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) the case hack applied by restorePath(). */ std::map<string, string> unhacked; for (auto & i : readDirectory(path)) - if (useCaseHack) { + if (archiveSettings.useCaseHack) { string name(i.name); size_t pos = i.name.find(caseHackSuffix); if (pos != string::npos) { @@ -243,7 +251,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) if (name <= prevName) throw Error("NAR directory is not sorted"); prevName = name; - if (useCaseHack) { + if (archiveSettings.useCaseHack) { auto i = names.find(name); if (i != names.end()) { debug(format("case collision between '%1%' and '%2%'") % i->first % name); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 7a0e688e4201..25be426c1a4d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -78,10 +78,6 @@ void restorePath(const Path & path, Source & source); void copyNAR(Source & source, Sink & sink); -// FIXME: global variables are bad m'kay. -extern bool useCaseHack; - - extern const std::string narVersionMagic1; diff --git a/src/libutil/config.cc b/src/libutil/config.cc index ce6858f0d65a..9023cb1bb6de 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -4,15 +4,13 @@ namespace nix { -void Config::set(const std::string & name, const std::string & value) +bool Config::set(const std::string & name, const std::string & value) { auto i = _settings.find(name); - if (i == _settings.end()) { - extras.emplace(name, value); - } else { - i->second.setting->set(value); - i->second.setting->overriden = true; - } + if (i == _settings.end()) return false; + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; } void Config::addSetting(AbstractSetting * setting) @@ -23,46 +21,51 @@ void Config::addSetting(AbstractSetting * setting) bool set = false; - auto i = extras.find(setting->name); - if (i != extras.end()) { + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { setting->set(i->second); setting->overriden = true; - extras.erase(i); + unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = extras.find(alias); - if (i != extras.end()) { + auto i = unknownSettings.find(alias); + if (i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { setting->set(i->second); setting->overriden = true; - extras.erase(i); + unknownSettings.erase(i); set = true; } } } } -void Config::handleUnknownSettings() +void AbstractConfig::warnUnknownSettings() { - for (auto & s : extras) + for (auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } -StringMap Config::getSettings(bool overridenOnly) +void AbstractConfig::reapplyUnknownSettings() +{ + auto unknownSettings2 = std::move(unknownSettings); + for (auto & s : unknownSettings2) + set(s.first, s.second); +} + +void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) { - StringMap res; for (auto & opt : _settings) if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) - res.emplace(opt.first, opt.second.setting->to_string()); - return res; + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void Config::applyConfigFile(const Path & path) +void AbstractConfig::applyConfigFile(const Path & path) { try { string contents = readFile(path); @@ -287,4 +290,49 @@ void PathSetting::set(const std::string & str) value = canonPath(str); } +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overridenOnly); +} + +void GlobalConfig::resetOverriden() +{ + for (auto & config : *configRegistrations) + config->resetOverriden(); +} + +void GlobalConfig::toJSON(JSONObject & out) +{ + for (auto & config : *configRegistrations) + config->toJSON(out); +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index d2e7faf17434..d86c65ff033a 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -12,6 +12,40 @@ class AbstractSetting; class JSONPlaceholder; class JSONObject; +class AbstractConfig +{ +protected: + StringMap unknownSettings; + + AbstractConfig(const StringMap & initials = {}) + : unknownSettings(initials) + { } + +public: + + virtual bool set(const std::string & name, const std::string & value) = 0; + + struct SettingInfo + { + std::string value; + std::string description; + }; + + virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0; + + void applyConfigFile(const Path & path); + + virtual void resetOverriden() = 0; + + virtual void toJSON(JSONObject & out) = 0; + + virtual void convertToArgs(Args & args, const std::string & category) = 0; + + void warnUnknownSettings(); + + void reapplyUnknownSettings(); +}; + /* A class to simplify providing configuration settings. The typical use is to inherit Config and add Setting<T> members: @@ -27,7 +61,7 @@ class JSONObject; }; */ -class Config +class Config : public AbstractConfig { friend class AbstractSetting; @@ -48,31 +82,23 @@ private: Settings _settings; - StringMap extras; - public: - Config(const StringMap & initials) - : extras(initials) + Config(const StringMap & initials = {}) + : AbstractConfig(initials) { } - void set(const std::string & name, const std::string & value); + bool set(const std::string & name, const std::string & value) override; void addSetting(AbstractSetting * setting); - void handleUnknownSettings(); - - StringMap getSettings(bool overridenOnly = false); + void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; - const Settings & _getSettings() { return _settings; } - - void applyConfigFile(const Path & path); + void resetOverriden() override; - void resetOverriden(); + void toJSON(JSONObject & out) override; - void toJSON(JSONObject & out); - - void convertToArgs(Args & args, const std::string & category); + void convertToArgs(Args & args, const std::string & category) override; }; class AbstractSetting @@ -209,4 +235,27 @@ public: void operator =(const Path & v) { this->assign(v); } }; +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector<Config*> ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 33ae1ea389d7..21803edd056a 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -133,7 +133,7 @@ size_t FdSource::readUnbuffered(unsigned char * data, size_t len) ssize_t n; do { checkInterrupt(); - n = ::read(fd, (char *) data, bufSize); + n = ::read(fd, (char *) data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6e703c52a1e3..14b62fdb6774 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,10 +77,12 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - /* Underlying read call, to be overridden. */ - virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; bool hasData(); + +protected: + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; }; @@ -134,8 +136,9 @@ struct FdSource : BufferedSource return *this; } - size_t readUnbuffered(unsigned char * data, size_t len) override; bool good() override; +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override; private: bool _good = true; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 15962236ec65..6bc64ae75a42 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -311,6 +311,14 @@ string readFile(const Path & path, bool drain) } +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + void writeFile(const Path & path, const string & s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); @@ -320,6 +328,23 @@ void writeFile(const Path & path, const string & s, mode_t mode) } +void writeFile(const Path & path, Source & source, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + + std::vector<unsigned char> buf(64 * 1024); + + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } +} + + string readLine(int fd) { string s; @@ -593,7 +618,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("making file descriptor non-blocking"); } - std::vector<unsigned char> buf(4096); + std::vector<unsigned char> buf(64 * 1024); while (1) { checkInterrupt(); ssize_t rd = read(fd, buf.data(), buf.size()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 743d238611fc..fc25d27758c7 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -15,6 +15,7 @@ #include <map> #include <sstream> #include <experimental/optional> +#include <future> #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -97,10 +98,13 @@ unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); string readFile(const Path & path, bool drain = false); +void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ void writeFile(const Path & path, const string & s, mode_t mode = 0666); +void writeFile(const Path & path, Source & source, mode_t mode = 0666); + /* Read a line from a file descriptor. */ string readLine(int fd); @@ -424,44 +428,30 @@ string get(const T & map, const string & key, const string & def = "") } -/* Call ‘failure’ with the current exception as argument. If ‘failure’ - throws an exception, abort the program. */ -void callFailure(const std::function<void(std::exception_ptr exc)> & failure, - std::exception_ptr exc = std::current_exception()); +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future<T> to pass the value or + exception.) */ +template<typename T> +struct Callback +{ + std::function<void(std::future<T>)> fun; + Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } -/* Evaluate the function ‘f’. If it returns a value, call ‘success’ - with that value as its argument. If it or ‘success’ throws an - exception, call ‘failure’. If ‘failure’ throws an exception, abort - the program. */ -template<class T> -void sync2async( - const std::function<void(T)> & success, - const std::function<void(std::exception_ptr exc)> & failure, - const std::function<T()> & f) -{ - try { - success(f()); - } catch (...) { - callFailure(failure); + void operator()(T && t) const + { + std::promise<T> promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); } -} - -/* Call the function ‘success’. If it throws an exception, call - ‘failure’. If that throws an exception, abort the program. */ -template<class T> -void callSuccess( - const std::function<void(T)> & success, - const std::function<void(std::exception_ptr exc)> & failure, - T && arg) -{ - try { - success(arg); - } catch (...) { - callFailure(failure); + void rethrow(const std::exception_ptr & exc = std::current_exception()) const + { + std::promise<T> promise; + promise.set_exception(exc); + fun(promise.get_future()); } -} +}; /* Start a thread that handles various signals. Also block those signals diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 35603af7082a..423e6bb67893 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -120,8 +120,6 @@ struct TunnelLogger : public Logger want to send out stderr to the client. */ void startWork() { - std::vector<std::string> pendingMsgs; - auto state(state_.lock()); state->canSendStderr = true; @@ -197,7 +195,8 @@ struct TunnelSource : BufferedSource { Source & from; TunnelSource(Source & from) : from(from) { } - size_t readUnbuffered(unsigned char * data, size_t len) +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override { to << STDERR_READ << len; to.flush(); @@ -554,7 +553,7 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, ; else if (trusted || name == settings.buildTimeout.name - || name == settings.connectTimeout.name) + || name == "connect-timeout") settings.set(name, value); else if (setSubstituters(settings.substituters)) ; @@ -691,12 +690,22 @@ static void performOp(TunnelLogger * logger, ref<LocalStore> store, if (!trusted) info.ultimate = false; - TeeSink tee(from); - parseDump(tee, tee.source); + std::string saved; + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique<TunnelSource>(from); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique<StringSource>(saved); + } logger->startWork(); - store.cast<Store>()->addToStore(info, tee.source.data, (RepairFlag) repair, + + store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + logger->stopWork(); break; } diff --git a/src/nix/main.cc b/src/nix/main.cc index bb107ec7d3f6..9cd5d21c84b6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -34,9 +34,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .handler([&]() { std::cout << "The following configuration options are available:\n\n"; Table2 tbl; - for (const auto & s : settings._getSettings()) - if (!s.second.isAlias) - tbl.emplace_back(s.first, s.second.setting->description); + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + tbl.emplace_back(s.first, s.second.description); printTable(std::cout, tbl); throw Exit(); }); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index c64b12c8dd62..86638b50d2c6 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -27,10 +27,12 @@ struct CmdShowConfig : Command, MixJSON if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). JSONObject jsonObj(std::cout); - settings.toJSON(jsonObj); + globalConfig.toJSON(jsonObj); } else { - for (auto & s : settings.getSettings()) - std::cout << s.first + " = " + s.second + "\n"; + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + std::cout << s.first + " = " + s.second.value + "\n"; } } }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 758bbbc688bc..21892c31a893 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -46,7 +46,7 @@ struct CmdUpgradeNix : StoreCommand void run(ref<Store> store) override { - settings.pureEval = true; + evalSettings.pureEval = true; if (profileDir == "") profileDir = getProfileDir(store); diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/lang/eval-okay-arithmetic.exp index b195055b7a09..5c54d10b7b47 100644 --- a/tests/lang/eval-okay-arithmetic.exp +++ b/tests/lang/eval-okay-arithmetic.exp @@ -1 +1 @@ -2188 +2216 diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/lang/eval-okay-arithmetic.nix index bbbbc4691d75..7e9e6a0b666e 100644 --- a/tests/lang/eval-okay-arithmetic.nix +++ b/tests/lang/eval-okay-arithmetic.nix @@ -26,6 +26,10 @@ let { (56088 / 123 / 2) (3 + 4 * const 5 0 - 6 / id 2) + (builtins.bitAnd 12 10) # 0b1100 & 0b1010 = 8 + (builtins.bitOr 12 10) # 0b1100 | 0b1010 = 14 + (builtins.bitXor 12 10) # 0b1100 ^ 0b1010 = 6 + (if 3 < 7 then 1 else err) (if 7 < 3 then err else 1) (if 3 < 3 then err else 1) diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc index 8da15ebabd7d..c085d33295be 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/plugins/plugintest.cc @@ -1,16 +1,21 @@ -#include "globals.hh" +#include "config.hh" #include "primops.hh" using namespace nix; -static BaseSetting<bool> settingSet{false, "setting-set", +struct MySettings : Config +{ + Setting<bool> settingSet{this, false, "setting-set", "Whether the plugin-defined setting was set"}; +}; + +MySettings mySettings; -static RegisterSetting rs(&settingSet); +static GlobalConfig::Register rs(&mySettings); static void prim_anotherNull (EvalState & state, const Pos & pos, Value ** args, Value & v) { - if (settingSet) + if (mySettings.settingSet) mkNull(v); else mkBool(v, false); |