diff options
Diffstat (limited to 'src')
84 files changed, 4502 insertions, 1244 deletions
diff --git a/src/boost/format/feed_args.hpp b/src/boost/format/feed_args.hpp index 3d0b47b4a12e..cdd57fdf2bf1 100644 --- a/src/boost/format/feed_args.hpp +++ b/src/boost/format/feed_args.hpp @@ -41,6 +41,13 @@ namespace { std::streamsize w, const char c, std::ios::fmtflags f, + bool center) + __attribute__ ((unused)); + + void do_pad( std::string & s, + std::streamsize w, + const char c, + std::ios::fmtflags f, bool center) // applies centered / left / right padding to the string s. // Effects : string s is padded. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index d7aee288670a..7ffbdca7c0f4 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,6 +9,7 @@ #include <sys/time.h> #endif +#include "machines.hh" #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" @@ -22,131 +23,56 @@ using std::cin; static void handleAlarm(int sig) { } -class Machine { - const std::set<string> supportedFeatures; - const std::set<string> mandatoryFeatures; - -public: - const string hostName; - const std::vector<string> systemTypes; - const string sshKey; - const unsigned int maxJobs; - const unsigned int speedFactor; - bool enabled; - - bool allSupported(const std::set<string> & features) const { - return std::all_of(features.begin(), features.end(), - [&](const string & feature) { - return supportedFeatures.count(feature) || - mandatoryFeatures.count(feature); - }); - } - - bool mandatoryMet(const std::set<string> & features) const { - return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { - return features.count(feature); - }); - } - - Machine(decltype(hostName) hostName, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures) : - supportedFeatures(supportedFeatures), - mandatoryFeatures(mandatoryFeatures), - hostName(hostName), - systemTypes(systemTypes), - sshKey(sshKey), - maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), - enabled(true) - {}; -};; - -static std::vector<Machine> readConf() +std::string escapeUri(std::string uri) { - auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines"); - - auto machines = std::vector<Machine>{}; - auto lines = std::vector<string>{}; - try { - lines = tokenizeString<std::vector<string>>(readFile(conf), "\n"); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - for (auto line : lines) { - chomp(line); - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) { - continue; - } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 4) - throw FormatError("bad machines.conf file ‘%1%’", conf); - machines.emplace_back(tokens[0], - tokenizeString<std::vector<string>>(tokens[1], ","), - tokens[2], - stoull(tokens[3]), - sz >= 5 ? stoull(tokens[4]) : 1LL, - sz >= 6 ? - tokenizeString<std::set<string>>(tokens[5], ",") : - std::set<string>{}, - sz >= 7 ? - tokenizeString<std::set<string>>(tokens[6], ",") : - std::set<string>{}); - } - return machines; + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; } static string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) { - std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out); - fn_stream << "/"; - for (auto t : m.systemTypes) { - fn_stream << t << "-"; - } - fn_stream << m.hostName << "-" << slot; - return openLockFile(fn_stream.str(), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static char display_env[] = "DISPLAY="; -static char ssh_env[] = "SSH_ASKPASS="; - int main (int argc, char * * argv) { return handleExceptions(argv[0], [&]() { initNix(); /* Ensure we don't get any SSH passphrase or host key popups. */ - if (putenv(display_env) == -1 || - putenv(ssh_env) == -1) - throw SysError("setting SSH env vars"); + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); - if (argc != 4) + if (argc != 6) throw UsageError("called without required arguments"); auto store = openStore(); auto localSystem = argv[1]; - settings.maxSilentTime = stoull(string(argv[2])); - settings.buildTimeout = stoull(string(argv[3])); + settings.maxSilentTime = std::stoll(argv[2]); + settings.buildTimeout = std::stoll(argv[3]); + verbosity = (Verbosity) std::stoll(argv[4]); + settings.builders = argv[5]; - currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load"); + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on OS X. */ + currentLoad = settings.nixStateDir + "/current-load"; std::shared_ptr<Store> sshStore; AutoCloseFD bestSlotLock; - auto machines = readConf(); + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return; + } + string drvPath; - string hostName; + string storeUri; for (string line; getline(cin, line);) { auto tokens = tokenizeString<std::vector<string>>(line); auto sz = tokens.size(); @@ -173,6 +99,8 @@ int main (int argc, char * * argv) Machine * bestMachine = nullptr; unsigned long long bestLoad = 0; for (auto & m : machines) { + debug("considering building on ‘%s’", m.storeUri); + if (m.enabled && std::find(m.systemTypes.begin(), m.systemTypes.end(), neededSystem) != m.systemTypes.end() && @@ -233,16 +161,22 @@ int main (int argc, char * * argv) lock = -1; try { - sshStore = openStore("ssh-ng://" + bestMachine->hostName, - { {"ssh-key", bestMachine->sshKey }, - {"max-connections", "1" } }); - hostName = bestMachine->hostName; + + Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}}; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + } catch (std::exception & e) { printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...", - bestMachine->hostName, e.what()); + bestMachine->storeUri, e.what()); bestMachine->enabled = false; continue; } + goto connected; } } @@ -252,22 +186,32 @@ connected: string line; if (!getline(cin, line)) throw Error("hook caller didn't send inputs"); + auto inputs = tokenizeString<PathSet>(line); if (!getline(cin, line)) throw Error("hook caller didn't send outputs"); + auto outputs = tokenizeString<PathSet>(line); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true); + + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); if (!lockFile(uploadLock.get(), ltWrite, true)) printError("somebody is hogging the upload lock for ‘%s’, continuing..."); alarm(0); signal(SIGALRM, old); - copyPaths(store, ref<Store>(sshStore), inputs); + copyPaths(store, ref<Store>(sshStore), inputs, false, true); uploadLock = -1; - printError("building ‘%s’ on ‘%s’", drvPath, hostName); - sshStore->buildDerivation(drvPath, readDerivation(drvPath)); + BasicDerivation drv(readDerivation(drvPath)); + drv.inputSrcs = inputs; + + printError("building ‘%s’ on ‘%s’", drvPath, storeUri); + auto result = sshStore->buildDerivation(drvPath, drv); + + if (!result.success()) + throw Error("build of ‘%s’ on ‘%s’ failed: %s", drvPath, storeUri, result.errorMsg); PathSet missing; for (auto & path : outputs) @@ -275,7 +219,7 @@ connected: if (!missing.empty()) { setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing); + copyPaths(ref<Store>(sshStore), store, missing, false, true); } return; diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 62d5a010c247..64368a43ff73 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -7,5 +7,3 @@ build-remote_INSTALL_DIR := $(libexecdir)/nix build-remote_LIBS = libmain libutil libformat libstore build-remote_SOURCES := $(d)/build-remote.cc - -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d418ab4e43aa..0cdce602d7b2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -202,7 +202,7 @@ void initGC() GC_INIT(); - GC_oom_fn = oomHandler; + GC_set_oom_fn(oomHandler); /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases @@ -299,7 +299,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) { countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; - restricted = settings.get("restrict-eval", false); + restricted = settings.restrictEval; assert(gcInitialised); @@ -642,7 +642,7 @@ void EvalState::evalFile(const Path & path, Value & v) return; } - Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2); + printTalkative("evaluating file ‘%1%’", path2); Expr * e = parseExprFromFile(checkSourcePath(path2)); try { eval(e, v); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 5342739c53c4..4200e8fd6757 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -3,6 +3,7 @@ #include "eval-inline.hh" #include <cstring> +#include <regex> namespace nix { @@ -262,6 +263,9 @@ static string addToPath(const string & s1, const string & s2) } +static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); + + static void getDerivations(EvalState & state, Value & vIn, const string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, Done & done, @@ -285,7 +289,9 @@ static void getDerivations(EvalState & state, Value & vIn, bound to the attribute with the "lower" name should take precedence). */ for (auto & i : v.attrs->lexicographicOrder()) { - Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i->name); + debug("evaluating attribute ‘%1%’", i->name); + if (!std::regex_match(std::string(i->name), attrRegex)) + continue; string pathPrefix2 = addToPath(pathPrefix, i->name); if (combineChannels) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -304,7 +310,6 @@ static void getDerivations(EvalState & state, Value & vIn, else if (v.isList()) { for (unsigned int n = 0; n < v.listSize(); ++n) { - Activity act(*logger, lvlDebug, "evaluating list element"); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5b1ff0350cd1..40ca77258037 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -142,25 +142,34 @@ or { return OR_KW; } \{ { return '{'; } <INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { PUSH_STATE(STRING); return '"'; } +<INITIAL,INSIDE_DOLLAR_CURLY>\" { + PUSH_STATE(STRING); return '"'; + } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { - /* It is impossible to match strings ending with '$' with one - regex because trailing contexts are only valid at the end - of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); - return STR; - } + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext); + return STR; + } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -<STRING>\" { POP_STATE(); return '"'; } -<STRING>. return yytext[0]; /* just in case: shouldn't be reached */ +<STRING>\" { POP_STATE(); return '"'; } +<STRING>\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } <INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; } -<IND_STRING>\'\'\$ { +<IND_STRING>\'\'\$ | +<IND_STRING>\$ { yylval->e = new ExprIndStr("$"); return IND_STR; } @@ -178,7 +187,6 @@ or { return OR_KW; } yylval->e = new ExprIndStr("'"); return IND_STR; } -<IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ <INITIAL,INSIDE_DOLLAR_CURLY>{ diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 620050a13b05..daa3258f0d3c 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,8 +6,6 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_CXXFLAGS := -Wno-deprecated-register - libexpr_LIBS = libutil libstore libformat libexpr_LDFLAGS = diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index d07eedddaf6b..62982650a22e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -376,7 +376,7 @@ expr_simple $$ = stripIndentation(CUR_POS, data->symbols, *$2); } | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } - | HPATH { $$ = new ExprPath(getEnv("HOME", "") + string{$1 + 1}); } + | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); } | SPATH { string path($1 + 1, strlen($1) - 2); $$ = new ExprApp(CUR_POS, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 615cc8138433..99ffddaeb80c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -127,7 +127,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path); + printTalkative("evaluating file ‘%1%’", path); Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); @@ -326,8 +326,6 @@ typedef list<Value *> ValueList; static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { - Activity act(*logger, lvlDebug, "finding dependencies"); - state.forceAttrs(*args[0], pos); /* Get the start set. */ @@ -499,8 +497,6 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation. */ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) { - Activity act(*logger, lvlVomit, "evaluating derivation"); - state.forceAttrs(*args[0], pos); /* Figure out the name first (for stack backtraces). */ @@ -534,7 +530,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - string outputHash, outputHashAlgo; + std::experimental::optional<std::string> outputHash; + std::string outputHashAlgo; bool outputHashRecursive = false; StringSet outputs; @@ -543,7 +540,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : args[0]->attrs->lexicographicOrder()) { if (i->name == state.sIgnoreNulls) continue; string key = i->name; - Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); + vomit("processing attribute ‘%1%’", key); auto handleHashMode = [&](const std::string & s) { if (s == "recursive") outputHashRecursive = true; @@ -703,7 +700,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * throw EvalError(format("derivation names are not allowed to end in ‘%1%’, at %2%") % drvExtension % posDrvName); - if (outputHash != "") { + if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); @@ -711,13 +708,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = parseHashType(outputHashAlgo); if (ht == htUnknown) throw EvalError(format("unknown hash algorithm ‘%1%’, at %2%") % outputHashAlgo % posDrvName); - Hash h = parseHash16or32(ht, outputHash); + Hash h = parseHash16or32(ht, *outputHash); outputHash = printHash(h); if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash); + drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); } else { @@ -1712,26 +1709,33 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, ‘null’ or a list containing substring matches. */ static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::regex regex(state.forceStringNoCtx(*args[0], pos), std::regex::extended); + auto re = state.forceStringNoCtx(*args[0], pos); - PathSet context; - const std::string str = state.forceString(*args[1], context, pos); + try { + std::regex regex(re, std::regex::extended); - std::smatch match; - if (!std::regex_match(str, match, regex)) { - mkNull(v); - return; - } + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); - // the first match is the whole string - const size_t len = match.size() - 1; - state.mkList(v, len); - for (size_t i = 0; i < len; ++i) { - if (!match[i+1].matched) - mkNull(*(v.listElems()[i] = state.allocValue())); - else - mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + std::smatch match; + if (!std::regex_match(str, match, regex)) { + mkNull(v); + return; + } + + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i+1].matched) + mkNull(*(v.listElems()[i] = state.allocValue())); + else + mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + } + + } catch (std::regex_error &) { + throw EvalError("invalid regular expression ‘%s’, at %s", re, pos); } } diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc index 09e2c077baba..3e4ece2cffdc 100644 --- a/src/libexpr/primops/fetchgit.cc +++ b/src/libexpr/primops/fetchgit.cc @@ -17,7 +17,7 @@ Path exportGit(ref<Store> store, const std::string & uri, const std::string & re runProgram("git", true, { "init", "--bare", cacheDir }); } - Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); + //Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); std::string localRef = "pid-" + std::to_string(getpid()); Path localRefFile = cacheDir + "/refs/heads/" + localRef; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 802e8ed2ee75..9df516f062ef 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -220,6 +220,14 @@ static inline void mkApp(Value & v, Value & left, Value & right) } +static inline void mkPrimOpApp(Value & v, Value & left, Value & right) +{ + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} + + static inline void mkStringNoCopy(Value & v, const char * s) { v.type = tString; diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 98693d78a7f4..9a7a893138de 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -22,7 +22,11 @@ MixCommonArgs::MixCommonArgs(const string & programName) [](Strings ss) { auto name = ss.front(); ss.pop_front(); auto value = ss.front(); - settings.set(name, value); + try { + settings.set(name, value); + } catch (UsageError & e) { + warn(e.what()); + } }); } diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index 2c0d71edd815..a4de3dccf0a5 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -12,7 +12,7 @@ struct MixCommonArgs : virtual Args struct MixDryRun : virtual Args { - bool dryRun; + bool dryRun = false; MixDryRun() { @@ -20,4 +20,14 @@ struct MixDryRun : virtual Args } }; +struct MixJSON : virtual Args +{ + bool json = false; + + MixJSON() + { + mkFlag(0, "json", "produce JSON output", &json); + } +}; + } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a720afd6cdd4..d6c1c0c9cb49 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -106,8 +106,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - logger = makeDefaultLogger(); - /* Initialise OpenSSL locking. */ opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); @@ -140,9 +138,6 @@ void initNix() struct timeval tv; gettimeofday(&tv, 0); srandom(tv.tv_usec); - - if (char *pack = getenv("_NIX_OPTIONS")) - settings.unpack(pack); } @@ -158,13 +153,13 @@ struct LegacyArgs : public MixCommonArgs &settings.verboseBuild, false); mkFlag('K', "keep-failed", "keep temporary directories of failed builds", - &settings.keepFailed); + &(bool&) settings.keepFailed); mkFlag('k', "keep-going", "keep going after a build fails", - &settings.keepGoing); + &(bool&) settings.keepGoing); mkFlag(0, "fallback", "build from source if substitution fails", []() { - settings.set("build-fallback", "true"); + settings.tryFallback = true; }); mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { @@ -186,7 +181,7 @@ struct LegacyArgs : public MixCommonArgs &settings.readOnlyMode); mkFlag(0, "no-build-hook", "disable use of the build hook mechanism", - &settings.useBuildHook, false); + &(bool&) settings.useBuildHook, false); mkFlag(0, "show-trace", "show Nix expression stack trace in evaluation errors", &settings.showTrace); @@ -220,7 +215,6 @@ void parseCmdLine(int argc, char * * argv, std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) { LegacyArgs(baseNameOf(argv[0]), parseArg).parseCmdline(argvToStrings(argc, argv)); - settings.update(); } @@ -265,7 +259,7 @@ int handleExceptions(const string & programName, std::function<void()> fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - interruptThrown = true; + setInterruptThrown(); throw; } } catch (Exit & e) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 25ad0d75b70a..46c5aa21b2eb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -79,10 +79,7 @@ struct BinaryCacheStoreAccessor : public FSAccessor BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) - , compression(get(params, "compression", "xz")) - , writeNARListing(get(params, "write-nar-listing", "0") == "1") { - auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); @@ -117,11 +114,6 @@ void BinaryCacheStore::init() } } -void BinaryCacheStore::notImpl() -{ - throw Error("operation not implemented for binary cache stores"); -} - std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) { std::promise<std::shared_ptr<std::string>> promise; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index d42b1abd2455..87d4aa43838e 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,20 +13,20 @@ struct NarInfo; class BinaryCacheStore : public Store { -private: +public: - std::unique_ptr<SecretKey> secretKey; + const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - std::string compression; +private: - bool writeNARListing; + std::unique_ptr<SecretKey> secretKey; protected: BinaryCacheStore(const Params & params); - [[noreturn]] void notImpl(); - public: virtual bool fileExists(const std::string & path) = 0; @@ -63,7 +63,7 @@ public: bool isValidPathUncached(const Path & path) override; PathSet queryAllValidPaths() override - { notImpl(); } + { unsupported(); } void queryPathInfoUncached(const Path & path, std::function<void(std::shared_ptr<ValidPathInfo>)> success, @@ -71,16 +71,16 @@ public: void queryReferrers(const Path & path, PathSet & referrers) override - { notImpl(); } + { unsupported(); } PathSet queryDerivationOutputs(const Path & path) override - { notImpl(); } + { unsupported(); } StringSet queryDerivationOutputNames(const Path & path) override - { notImpl(); } + { unsupported(); } Path queryPathFromHashPart(const string & hashPart) override - { notImpl(); } + { unsupported(); } bool wantMassQuery() override { return wantMassQuery_; } @@ -97,32 +97,29 @@ public: void narFromPath(const Path & path, Sink & sink) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { notImpl(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { notImpl(); } + { unsupported(); } void ensurePath(const Path & path) override - { notImpl(); } + { unsupported(); } void addTempRoot(const Path & path) override - { notImpl(); } + { unsupported(); } void addIndirectRoot(const Path & path) override - { notImpl(); } + { unsupported(); } Roots findRoots() override - { notImpl(); } + { unsupported(); } void collectGarbage(const GCOptions & options, GCResults & results) override - { notImpl(); } + { unsupported(); } ref<FSAccessor> getFSAccessor() override; void addSignatures(const Path & storePath, const StringSet & sigs) override - { notImpl(); } + { unsupported(); } std::shared_ptr<std::string> getBuildLog(const Path & path) override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 968e291129f3..44cae3431fc2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -34,13 +34,6 @@ #include <pwd.h> #include <grp.h> -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - /* Includes required for chroot support. */ #if __linux__ #include <sys/socket.h> @@ -127,6 +120,8 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + Activity act; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -175,7 +170,8 @@ public: virtual string key() = 0; protected: - void amDone(ExitCode result); + + virtual void amDone(ExitCode result); }; @@ -469,7 +465,7 @@ UserLock::UserLock() assert(settings.buildUsersGroup != ""); /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) throw Error(format("the group ‘%1%’ specified in ‘build-users-group’ does not exist") % settings.buildUsersGroup); @@ -590,11 +586,7 @@ struct HookInstance HookInstance::HookInstance() { - debug("starting build hook"); - - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; - buildHook = canonPath(buildHook); + debug("starting build hook ‘%s’", settings.buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -621,15 +613,17 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(buildHook), + baseNameOf(settings.buildHook), settings.thisSystem, - (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.buildTimeout).str() + std::to_string(settings.maxSilentTime), + std::to_string(settings.buildTimeout), + std::to_string(verbosity), + settings.builders }; - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError(format("executing ‘%1%’") % buildHook); + throw SysError("executing ‘%s’", settings.buildHook); }); pid.setSeparatePG(true); @@ -911,6 +905,12 @@ private: void repairClosure(); + void amDone(ExitCode result) + { + logger->event(evBuildFinished, act, result == ecSuccess); + Goal::amDone(result); + } + void done(BuildResult::Status status, const string & msg = ""); }; @@ -929,6 +929,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut state = &DerivationGoal::getDerivation; name = (format("building of ‘%1%’") % drvPath).str(); trace("created"); + + logger->event(evBuildCreated, act, drvPath); } @@ -944,6 +946,8 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); trace("created"); + logger->event(evBuildCreated, act, drvPath); + /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ worker.store.addTempRoot(drvPath); @@ -1075,12 +1079,8 @@ void DerivationGoal::haveDerivation() /* Reject doing a hash build of anything other than a fixed-output derivation. */ - if (buildMode == bmHash) { - if (drv->outputs.size() != 1 || - drv->outputs.find("out") == drv->outputs.end() || - drv->outputs["out"].hashAlgo == "") - throw Error(format("cannot do a hash build of non-fixed-output derivation ‘%1%’") % drvPath); - } + if (buildMode == bmHash && !drv->isFixedOutput()) + throw Error("cannot do a hash build of non-fixed-output derivation ‘%1%’", drvPath); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1279,7 +1279,7 @@ void DerivationGoal::inputsRealised() /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = fixedOutput ? 1 : settings.get("build-repeat", 0) + 1; + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -1575,7 +1575,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; + if (!settings.useBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1608,8 +1608,15 @@ HookReply DerivationGoal::tryBuildHook() debug(format("hook reply is ‘%1%’") % reply); - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + settings.useBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; else if (reply != "accept") throw Error(format("bad hook reply ‘%1%’") % reply); @@ -1628,23 +1635,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - worker.store.computeFSClosure(drvPath, allInputs); - - string s; - for (auto & i : allInputs) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + remote system. */ + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - s = ""; - for (auto & i : missingPaths) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); hook->toHook.writeSide = -1; @@ -1697,12 +1693,7 @@ void DerivationGoal::startBuilder() /* Are we doing a chroot build? */ { - string x = settings.get("build-use-sandbox", - /* deprecated alias */ - settings.get("build-use-chroot", string("false"))); - if (x != "true" && x != "false" && x != "relaxed") - throw Error("option ‘build-use-sandbox’ must be set to one of ‘true’, ‘false’ or ‘relaxed’"); - if (x == "true") { + if (settings.sandboxMode == smEnabled) { if (get(drv->env, "__noChroot") == "1") throw Error(format("derivation ‘%1%’ has ‘__noChroot’ set, " "but that's not allowed when ‘build-use-sandbox’ is ‘true’") % drvPath); @@ -1713,9 +1704,9 @@ void DerivationGoal::startBuilder() #endif useChroot = true; } - else if (x == "false") + else if (settings.sandboxMode == smDisabled) useChroot = false; - else if (x == "relaxed") + else if (settings.sandboxMode == smRelaxed) useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; } @@ -1739,7 +1730,14 @@ void DerivationGoal::startBuilder() /* In a sandbox, for determinism, always use the same temporary directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#elif __APPLE__ + // On Darwin, we canonize /tmp because its probably a symlink to /private/tmp. tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ @@ -1756,21 +1754,10 @@ void DerivationGoal::startBuilder() if (useChroot) { - string defaultChrootDirs; -#if __linux__ - if (worker.store.isInStore(BASH_PATH)) - defaultChrootDirs = "/bin/sh=" BASH_PATH; -#endif - /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = tokenizeString<StringSet>( - settings.get("build-sandbox-paths", - /* deprecated alias with lower priority */ - settings.get("build-chroot-dirs", defaultChrootDirs))); - PathSet dirs2 = tokenizeString<StringSet>( - settings.get("build-extra-chroot-dirs", - settings.get("build-extra-sandbox-paths", string("")))); + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; dirs.insert(dirs2.begin(), dirs2.end()); dirsInChroot.clear(); @@ -1796,14 +1783,14 @@ void DerivationGoal::startBuilder() try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + } catch (InvalidPath & e) { } catch (Error & e) { throw Error(format("while processing ‘build-sandbox-paths’: %s") % e.what()); } for (auto & i : closure) dirsInChroot[i] = i; - string allowed = settings.get("allowed-impure-host-deps", string(DEFAULT_ALLOWED_IMPURE_PREFIXES)); - PathSet allowedPaths = tokenizeString<StringSet>(allowed); + PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); @@ -1823,7 +1810,7 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps (‘%3%’)") % drvPath % i % allowed); + throw Error(format("derivation ‘%1%’ requested impure path ‘%2%’, but it was not in allowed-impure-host-deps") % drvPath % i); dirsInChroot[i] = i; } @@ -1859,11 +1846,11 @@ void DerivationGoal::startBuilder() Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - writeFile(chrootRootDir + "/etc/passwd", - (format( - "root:x:0:0:Nix build user:/:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n") % sandboxUid % sandboxGid).str()); + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -1900,6 +1887,7 @@ void DerivationGoal::startBuilder() dirsInChroot[i] = r; else { Path p = chrootRootDir + i; + debug("linking ‘%1%’ to ‘%2%’", p, r); if (link(r.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), @@ -2137,6 +2125,8 @@ void DerivationGoal::startBuilder() } debug(msg); } + + logger->event(evBuildStarted, act); } @@ -2382,10 +2372,8 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); -#ifdef __linux__ if (pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); -#endif ss.push_back("/dev/null"); ss.push_back("/dev/random"); ss.push_back("/dev/tty"); @@ -2414,17 +2402,14 @@ void DerivationGoal::runChild() /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - for (auto & i : dirsInChroot) { - struct stat st; - Path source = i.second.source; - Path target = chrootRootDir + i.first; - if (source == "/proc") continue; // backwards compatibility + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { debug(format("bind mounting ‘%1%’ to ‘%2%’") % source % target); + struct stat st; if (stat(source.c_str(), &st) == -1) { - if (i.second.optional && errno == ENOENT) - continue; + if (optional && errno == ENOENT) + return; else - throw SysError(format("getting attributes of path ‘%1%’") % source); + throw SysError("getting attributes of path ‘%1%’", source); } if (S_ISDIR(st.st_mode)) createDirs(target); @@ -2433,7 +2418,12 @@ void DerivationGoal::runChild() writeFile(target, ""); } if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError(format("bind mount from ‘%1%’ to ‘%2%’ failed") % source % target); + throw SysError("bind mount from ‘%1%’ to ‘%2%’ failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ @@ -2444,7 +2434,7 @@ void DerivationGoal::runChild() /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, - fmt("size=%s", settings.get("sandbox-dev-shm-size", std::string("50%"))).c_str()) == -1) + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) throw SysError("mounting /dev/shm"); /* Mount a new devpts on /dev/pts. Note that this @@ -2455,13 +2445,19 @@ void DerivationGoal::runChild() !pathExists(chrootRootDir + "/dev/ptmx") && !dirsInChroot.count("/dev/pts")) { - if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == -1) - throw SysError("mounting /dev/pts"); - createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", "/dev/pts"); + doBind("/dev/ptmx", "/dev/ptmx"); + } } /* Do the chroot(). */ @@ -2602,7 +2598,7 @@ void DerivationGoal::runChild() sandboxProfile += "(version 1)\n"; /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.get("darwin-log-sandbox-violations", false)) { + if (settings.darwinLogSandboxViolations) { sandboxProfile += "(deny default)\n"; } else { sandboxProfile += "(deny default (with no-log))\n"; @@ -2749,7 +2745,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.get("run-diff-hook", false); + bool runDiffHook = settings.runDiffHook; bool keepPreviousRound = settings.keepFailed || runDiffHook; /* Check whether the output paths were created, and grep each @@ -2876,7 +2872,7 @@ void DerivationGoal::registerOutputs() contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); + debug("scanning for references inside ‘%1%’", path); HashResult hash; PathSet references = scanForReferences(actualPath, allPaths, hash); @@ -2990,7 +2986,7 @@ void DerivationGoal::registerOutputs() ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); - auto diffHook = settings.get("diff-hook", std::string("")); + auto diffHook = settings.diffHook; if (prevExists && diffHook != "" && runDiffHook) { try { auto diff = runProgram(diffHook, true, {prev, i->path}); @@ -3001,7 +2997,7 @@ void DerivationGoal::registerOutputs() } } - if (settings.get("enforce-determinism", true)) + if (settings.enforceDeterminism) throw NotDeterministic(msg); printError(msg); @@ -3051,13 +3047,11 @@ Path DerivationGoal::openLogFile() string baseName = baseNameOf(drvPath); /* Create a log file. */ - Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % worker.store.drvsLogDir % string(baseName, 0, 2)).str(); + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); createDirs(dir); - Path logFileName = (format("%1%/%2%%3%") - % dir - % string(baseName, 2) - % (settings.compressLog ? ".bz2" : "")).str(); + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); if (!fdLogFile) throw SysError(format("creating log file ‘%1%’") % logFileName); @@ -3131,7 +3125,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide.get()) - printError(data); // FIXME? + printError(chomp(data)); } @@ -3151,6 +3145,7 @@ void DerivationGoal::flushLine() logTail.push_back(currentLogLine); if (logTail.size() > settings.logLines) logTail.pop_front(); } + logger->event(evBuildOutput, act, currentLogLine); currentLogLine = ""; currentLogLinePos = 0; } @@ -3265,6 +3260,12 @@ public: void handleEOF(int fd); Path getStorePath() { return storePath; } + + void amDone(ExitCode result) + { + logger->event(evSubstitutionFinished, act, result == ecSuccess); + Goal::amDone(result); + } }; @@ -3277,6 +3278,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool state = &SubstitutionGoal::init; name = (format("substitution of ‘%1%’") % storePath).str(); trace("created"); + logger->event(evSubstitutionCreated, act, storePath); } @@ -3402,16 +3404,18 @@ void SubstitutionGoal::tryToRun() trace("trying to run"); /* Make sure that we are allowed to start a build. Note that even - is maxBuildJobs == 0 (no local builds allowed), we still allow + if maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= (settings.maxBuildJobs == 0 ? 1 : settings.maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } printInfo(format("fetching path ‘%1%’...") % storePath); + logger->event(evSubstitutionStarted, act); + outPipe.create(); promise = std::promise<void>(); @@ -3658,7 +3662,7 @@ void Worker::run(const Goals & _topGoals) { for (auto & i : _topGoals) topGoals.insert(i); - Activity act(*logger, lvlDebug, "entered goal loop"); + debug("entered goal loop"); while (1) { @@ -3686,7 +3690,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && settings.maxBuildJobs == 0) throw Error( + if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( "unable to start any build; either increase ‘--max-jobs’ " "or enable distributed builds"); assert(!awake.empty()); @@ -3723,9 +3727,9 @@ void Worker::waitForInput() auto nearest = steady_time_point::max(); // nearest deadline for (auto & i : children) { if (!i.respectTimeouts) continue; - if (settings.maxSilentTime != 0) + if (0 != settings.maxSilentTime) nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (settings.buildTimeout != 0) + if (0 != settings.buildTimeout) nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { @@ -3803,7 +3807,7 @@ void Worker::waitForInput() } if (goal->getExitCode() == Goal::ecBusy && - settings.maxSilentTime != 0 && + 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { @@ -3814,7 +3818,7 @@ void Worker::waitForInput() } else if (goal->getExitCode() == Goal::ecBusy && - settings.buildTimeout != 0 && + 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index c5dbd57f8bc6..8a5cf3327d44 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -28,9 +28,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) DownloadRequest request(url); request.verifyTLS = false; - /* Show a progress indicator, even though stderr is not a tty. */ - request.showProgress = DownloadRequest::yes; - /* Note: have to use a fresh downloader here because we're in a forked process. */ auto data = makeDownloader()->download(request); diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 0fc86a1fe921..f56a6adab9c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -105,14 +105,12 @@ PublicKeys getDefaultPublicKeys() // FIXME: filter duplicates - for (auto s : settings.get("binary-cache-public-keys", - Strings{"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="})) - { + for (auto s : settings.binaryCachePublicKeys.get()) { PublicKey key(s); publicKeys.emplace(key.name, key); } - for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + for (auto secretKeyFile : settings.secretKeyFiles.get()) { try { SecretKey secretKey(readFile(secretKeyFile)); publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index d1f760fdc301..63e498f0603a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -63,6 +63,7 @@ struct CurlDownloader : public Downloader CurlDownloader & downloader; DownloadRequest request; DownloadResult result; + Activity act; bool done = false; // whether either the success or failure function has been called std::function<void(const DownloadResult &)> success; std::function<void(std::exception_ptr exc)> failure; @@ -70,10 +71,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - bool showProgress = false; - double prevProgressTime{0}, startTime{0}; - unsigned int moveBack{1}; - unsigned int attempt = 0; /* Don't start this download until the specified time point @@ -87,12 +84,10 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) : downloader(downloader), request(request) { - showProgress = - request.showProgress == DownloadRequest::yes || - (request.showProgress == DownloadRequest::automatic && isatty(STDERR_FILENO)); - if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + + logger->event(evDownloadCreated, act, request.uri); } ~DownloadItem() @@ -109,6 +104,7 @@ struct CurlDownloader : public Downloader } catch (...) { ignoreException(); } + logger->event(evDownloadDestroyed, act); } template<class T> @@ -171,19 +167,7 @@ struct CurlDownloader : public Downloader int progressCallback(double dltotal, double dlnow) { - if (showProgress) { - double now = getTime(); - if (prevProgressTime <= now - 1) { - string s = (format(" [%1$.0f/%2$.0f KiB, %3$.1f KiB/s]") - % (dlnow / 1024.0) - % (dltotal / 1024.0) - % (now == startTime ? 0 : dlnow / 1024.0 / (now - startTime))).str(); - std::cerr << "\e[" << moveBack << "D" << s; - moveBack = s.size(); - std::cerr.flush(); - prevProgressTime = now; - } - } + logger->event(evDownloadProgress, act, dltotal, dlnow); return _isInterrupted; } @@ -201,13 +185,6 @@ struct CurlDownloader : public Downloader void init() { - // FIXME: handle parallel downloads. - if (showProgress) { - std::cerr << (format("downloading ‘%1%’... ") % request.uri); - std::cerr.flush(); - startTime = getTime(); - } - if (!req) req = curl_easy_init(); curl_easy_reset(req); @@ -220,7 +197,9 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion).c_str()); + curl_easy_setopt(req, CURLOPT_USERAGENT, + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + (settings.userAgentSuffix != "" ? " " + settings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif @@ -249,9 +228,11 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, settings.connectTimeout.get()); + /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ - curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.c_str()); + curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); result.data = std::make_shared<std::string>(); @@ -259,10 +240,6 @@ struct CurlDownloader : public Downloader void finish(CURLcode code) { - if (showProgress) - //std::cerr << "\e[" << moveBack << "D\e[K\n"; - std::cerr << "\n"; - long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); @@ -288,6 +265,7 @@ struct CurlDownloader : public Downloader try { result.data = decodeContent(encoding, ref<std::string>(result.data)); callSuccess(success, failure, const_cast<const DownloadResult &>(result)); + logger->event(evDownloadSucceeded, act, result.data->size()); } catch (...) { done = true; callFailure(failure, std::current_exception()); @@ -300,6 +278,11 @@ struct CurlDownloader : public Downloader || httpStatus == 504 || httpStatus == 522 || httpStatus == 524 || code == CURLE_COULDNT_RESOLVE_HOST || code == CURLE_RECV_ERROR + + // this seems to occur occasionally for retriable reasons, and shows up in an error like this: + // curl: (23) Failed writing body (315 != 16366) + || code == CURLE_WRITE_ERROR + // this is a generic SSL failure that in some cases (e.g., certificate error) is permanent but also appears in transient cases, so we consider it retryable || code == CURLE_SSL_CONNECT_ERROR #if LIBCURL_VERSION_NUM >= 0x073200 @@ -364,9 +347,9 @@ struct CurlDownloader : public Downloader curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); #endif curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - settings.get("binary-caches-parallel-connections", 25)); + settings.binaryCachesParallelConnections.get()); - enableHttp2 = settings.get("enable-http2", true); + enableHttp2 = settings.enableHttp2; wakeupPipe.create(); fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); @@ -606,7 +589,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa string expectedETag; - int ttl = settings.get("tarball-ttl", 60 * 60); + int ttl = settings.tarballTtl; bool skip = false; if (pathExists(fileLink) && pathExists(dataFile)) { @@ -645,6 +628,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(false, hash); store->addToStore(info, sink.s, false, true); storePath = info.path; @@ -682,7 +666,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa } if (expectedStorePath != "" && storePath != expectedStorePath) - throw nix::Error(format("hash mismatch in file downloaded from ‘%s’") % url); + throw nix::Error("store path mismatch in file downloaded from ‘%s’", url); return storePath; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index e2e16b361036..7d8982d64c4c 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -13,9 +13,8 @@ struct DownloadRequest std::string uri; std::string expectedETag; bool verifyTLS = true; - enum { yes, no, automatic } showProgress = yes; bool head = false; - size_t tries = 1; + size_t tries = 5; unsigned int baseRetryTimeMs = 250; DownloadRequest(const std::string & uri) : uri(uri) { } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 2b8ab063e18e..6e8bc692cdff 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -30,13 +30,13 @@ void Store::exportPaths(const Paths & paths, Sink & sink) std::reverse(sorted.begin(), sorted.end()); std::string doneLabel("paths exported"); - logger->incExpected(doneLabel, sorted.size()); + //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path); + //Activity act(*logger, lvlInfo, format("exporting path ‘%s’") % path); sink << 1; exportPath(path, sink); - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); } sink << 0; @@ -81,7 +81,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, info.path = readStorePath(*this, source); - Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path); + //Activity act(*logger, lvlInfo, format("importing path ‘%s’") % info.path); info.references = readStorePaths<PathSet>(*this, source); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8e90913cc3f1..3cdbb114a79d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -426,22 +426,26 @@ void LocalStore::findRuntimeRoots(PathSet & roots) throw SysError("iterating /proc"); } +#if !defined(__linux__) try { - auto lsofRegex = std::regex(R"(^n(/.*)$)"); + std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = - tokenizeString<std::vector<string>>(runProgram("lsof", true, { "-n", "-w", "-F", "n" }), "\n"); + tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); for (const auto & line : lsofLines) { - auto match = std::smatch{}; + std::smatch match; if (std::regex_match(line, match, lsofRegex)) paths.emplace(match[1]); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ } +#endif +#if defined(__linux__) readFileRoots("/proc/sys/kernel/modprobe", paths); readFileRoots("/proc/sys/kernel/fbsplash", paths); readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); +#endif for (auto & i : paths) if (isInStore(i)) { @@ -611,7 +615,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) auto realPath = realStoreDir + "/" + baseNameOf(path); if (realPath == linksDir || realPath == trashDir) return; - Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); + //Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); if (!isStorePath(path) || !isValidPath(path)) { /* A lock file belonging to a path that we're building right @@ -679,7 +683,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting ‘%1%’") % path); - state.results.bytesFreed += st.st_blocks * 512; + state.results.bytesFreed += st.st_blocks * 512ULL; } struct stat st; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8c900be77b8f..3dd2508a26d3 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,6 +1,7 @@ #include "globals.hh" #include "util.hh" #include "archive.hh" +#include "args.hh" #include <algorithm> #include <map> @@ -17,255 +18,91 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" +#else + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" +#endif Settings settings; - Settings::Settings() + : Config({}) + , nixPrefix(NIX_PREFIX) + , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) + , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) + , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { - nixPrefix = NIX_PREFIX; - nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR))); - nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR)); - nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR)); - nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR)); - nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)); - nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); - nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); - nixDaemonSocketFile = canonPath(nixStateDir + DEFAULT_SOCKET_PATH); - - // should be set with the other config options, but depends on nixLibexecDir -#ifdef __APPLE__ - preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies"; -#endif - - keepFailed = false; - keepGoing = false; - tryFallback = false; - maxBuildJobs = 1; - buildCores = std::max(1U, std::thread::hardware_concurrency()); - readOnlyMode = false; - thisSystem = SYSTEM; - maxSilentTime = 0; - buildTimeout = 0; - useBuildHook = true; - reservedSize = 8 * 1024 * 1024; - fsyncMetadata = true; - useSQLiteWAL = true; - syncBeforeRegistering = false; - useSubstitutes = true; buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - useSshSubstituter = true; - impersonateLinux26 = false; - keepLog = true; - compressLog = true; - maxLogSize = 0; - pollInterval = 5; - checkRootReachability = false; - gcKeepOutputs = false; - gcKeepDerivations = true; - autoOptimiseStore = false; - envKeepDerivations = false; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; - showTrace = false; - enableNativeCode = false; - netrcFile = fmt("%s/%s", nixConfDir, "netrc"); caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt")); - enableImportFromDerivation = true; -} - - -void Settings::loadConfFile() -{ - Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str(); - if (!pathExists(settingsFile)) return; - string contents = readFile(settingsFile); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector<string> tokens = tokenizeString<vector<string> >(line); - if (tokens.empty()) continue; - if (tokens.size() < 2 || tokens[1] != "=") - throw Error(format("illegal configuration line ‘%1%’ in ‘%2%’") % line % settingsFile); - - string name = tokens[0]; - - vector<string>::iterator i = tokens.begin(); - advance(i, 2); - settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow - }; -} - - -void Settings::set(const string & name, const string & value) -{ - settings[name] = value; - overrides[name] = value; -} - - -string Settings::get(const string & name, const string & def) -{ - auto i = settings.find(name); - if (i == settings.end()) return def; - return i->second; -} - - -Strings Settings::get(const string & name, const Strings & def) -{ - auto i = settings.find(name); - if (i == settings.end()) return def; - return tokenizeString<Strings>(i->second); -} - - -bool Settings::get(const string & name, bool def) -{ - bool res = def; - _get(res, name); - return res; -} + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s != "") builderFiles = tokenizeString<Strings>(s, ":"); +#if defined(__linux__) && defined(SANDBOX_SHELL) + sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); +#endif -int Settings::get(const string & name, int def) -{ - int res = def; - _get(res, name); - return res; + allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); } - -void Settings::update() +void Settings::loadConfFile() { - _get(tryFallback, "build-fallback"); + applyConfigFile(nixConfDir + "/nix.conf"); - auto s = get("build-max-jobs", std::string("1")); - if (s == "auto") - maxBuildJobs = std::max(1U, std::thread::hardware_concurrency()); - else - if (!string2Int(s, maxBuildJobs)) - throw Error("configuration setting ‘build-max-jobs’ should be ‘auto’ or an integer"); + /* We only want to send overrides to the daemon, i.e. stuff from + ~/.nix/nix.conf or the command line. */ + resetOverriden(); - _get(buildCores, "build-cores"); - _get(thisSystem, "system"); - _get(maxSilentTime, "build-max-silent-time"); - _get(buildTimeout, "build-timeout"); - _get(reservedSize, "gc-reserved-space"); - _get(fsyncMetadata, "fsync-metadata"); - _get(useSQLiteWAL, "use-sqlite-wal"); - _get(syncBeforeRegistering, "sync-before-registering"); - _get(useSubstitutes, "build-use-substitutes"); - _get(buildUsersGroup, "build-users-group"); - _get(impersonateLinux26, "build-impersonate-linux-26"); - _get(keepLog, "build-keep-log"); - _get(compressLog, "build-compress-log"); - _get(maxLogSize, "build-max-log-size"); - _get(pollInterval, "build-poll-interval"); - _get(checkRootReachability, "gc-check-reachability"); - _get(gcKeepOutputs, "gc-keep-outputs"); - _get(gcKeepDerivations, "gc-keep-derivations"); - _get(autoOptimiseStore, "auto-optimise-store"); - _get(envKeepDerivations, "env-keep-derivations"); - _get(sshSubstituterHosts, "ssh-substituter-hosts"); - _get(useSshSubstituter, "use-ssh-substituter"); - _get(enableNativeCode, "allow-unsafe-native-code-during-evaluation"); - _get(useCaseHack, "use-case-hack"); - _get(preBuildHook, "pre-build-hook"); - _get(keepGoing, "keep-going"); - _get(keepFailed, "keep-failed"); - _get(netrcFile, "netrc-file"); - _get(enableImportFromDerivation, "allow-import-from-derivation"); + applyConfigFile(getConfigDir() + "/nix/nix.conf"); } - -void Settings::_get(string & res, const string & name) +void Settings::set(const string & name, const string & value) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = i->second; + Config::set(name, value); } - -void Settings::_get(bool & res, const string & name) +unsigned int Settings::getDefaultCores() { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (i->second == "true") res = true; - else if (i->second == "false") res = false; - else throw Error(format("configuration option ‘%1%’ should be either ‘true’ or ‘false’, not ‘%2%’") - % name % i->second); + return std::max(1U, std::thread::hardware_concurrency()); } +const string nixVersion = PACKAGE_VERSION; -void Settings::_get(StringSet & res, const string & name) -{ - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res.clear(); - Strings ss = tokenizeString<Strings>(i->second); - res.insert(ss.begin(), ss.end()); -} - -void Settings::_get(Strings & res, const string & name) +template<> void BaseSetting<SandboxMode>::set(const std::string & str) { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - res = tokenizeString<Strings>(i->second); + if (str == "true") value = smEnabled; + else if (str == "relaxed") value = smRelaxed; + else if (str == "false") value = smDisabled; + else throw UsageError("option '%s' has invalid value '%s'", name, str); } - -template<class N> void Settings::_get(N & res, const string & name) +template<> std::string BaseSetting<SandboxMode>::to_string() { - SettingsMap::iterator i = settings.find(name); - if (i == settings.end()) return; - if (!string2Int(i->second, res)) - throw Error(format("configuration setting ‘%1%’ should have an integer value") % name); + if (value == smEnabled) return "true"; + else if (value == smRelaxed) return "relaxed"; + else if (value == smDisabled) return "false"; + else abort(); } - -string Settings::pack() +template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) { - string s; - for (auto & i : settings) { - if (i.first.find('\n') != string::npos || - i.first.find('=') != string::npos || - i.second.find('\n') != string::npos) - throw Error("illegal option name/value"); - s += i.first; s += '='; s += i.second; s += '\n'; - } - return s; + AbstractSetting::toJSON(out); } - -void Settings::unpack(const string & pack) { - Strings lines = tokenizeString<Strings>(pack, "\n"); - for (auto & i : lines) { - string::size_type eq = i.find('='); - if (eq == string::npos) - throw Error("illegal option name/value"); - set(i.substr(0, eq), i.substr(eq + 1)); - } -} - - -Settings::SettingsMap Settings::getOverrides() +void MaxBuildJobsSetting::set(const std::string & str) { - return overrides; + if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + else if (!string2Int(str, value)) + throw UsageError("configuration setting ‘%s’ should be ‘auto’ or an integer", name); } - -const string nixVersion = PACKAGE_VERSION; - - } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ccec300f776e..af37ec61d7a1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,7 +1,7 @@ #pragma once #include "types.hh" -#include "logging.hh" +#include "config.hh" #include <map> #include <sys/types.h> @@ -9,10 +9,48 @@ namespace nix { +typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; + +extern bool useCaseHack; // FIXME + +struct CaseHackSetting : public BaseSetting<bool> +{ + CaseHackSetting(Config * options, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<bool>(useCaseHack, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str) override + { + BaseSetting<bool>::set(str); + nix::useCaseHack = true; + } +}; + +struct MaxBuildJobsSetting : public BaseSetting<unsigned int> +{ + MaxBuildJobsSetting(Config * options, + unsigned int def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<unsigned int>(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str) override; +}; -struct Settings { +class Settings : public Config { - typedef std::map<string, string> SettingsMap; + unsigned int getDefaultCores(); + +public: Settings(); @@ -20,29 +58,13 @@ struct Settings { void set(const string & name, const string & value); - string get(const string & name, const string & def); - - Strings get(const string & name, const Strings & def); - - bool get(const string & name, bool def); - - int get(const string & name, int def); - - void update(); - - string pack(); - - void unpack(const string & pack); - - SettingsMap getOverrides(); + Path nixPrefix; /* The directory where we store sources and derived files. */ Path nixStore; Path nixDataDir; /* !!! fix */ - Path nixPrefix; - /* The directory where we log various operations. */ Path nixLogDir; @@ -61,17 +83,14 @@ struct Settings { /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; - /* Whether to keep temporary directories of failed builds. */ - bool keepFailed; + Setting<bool> keepFailed{this, false, "keep-failed", + "Whether to keep temporary directories of failed builds."}; - /* Whether to keep building subgoals when a sibling (another - subgoal of the same goal) fails. */ - bool keepGoing; + Setting<bool> keepGoing{this, false, "keep-going", + "Whether to keep building derivations when another build fails."}; - /* Whether, if we cannot realise the known closure corresponding - to a derivation, we should try to normalise the derivation - instead. */ - bool tryFallback; + Setting<bool> tryFallback{this, false, "build-fallback", + "Whether to fall back to building when substitution fails."}; /* Whether to show build log output in real time. */ bool verboseBuild = true; @@ -80,132 +99,228 @@ struct Settings { the log to show if a build fails. */ size_t logLines = 10; - /* Maximum number of parallel build jobs. 0 means unlimited. */ - unsigned int maxBuildJobs; + MaxBuildJobsSetting maxBuildJobs{this, 1, "build-max-jobs", + "Maximum number of parallel build jobs. \"auto\" means use number of cores."}; - /* Number of CPU cores to utilize in parallel within a build, - i.e. by passing this number to Make via '-j'. 0 means that the - number of actual CPU cores on the local host ought to be - auto-detected. */ - unsigned int buildCores; + Setting<unsigned int> buildCores{this, getDefaultCores(), "build-cores", + "Number of CPU cores to utilize in parallel within a build, " + "i.e. by passing this number to Make via '-j'. 0 means that the " + "number of actual CPU cores on the local host ought to be " + "auto-detected."}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ - bool readOnlyMode; + bool readOnlyMode = false; + + Setting<std::string> thisSystem{this, SYSTEM, "system", + "The canonical Nix system name."}; - /* The canonical system name, as returned by config.guess. */ - string thisSystem; + Setting<time_t> maxSilentTime{this, 0, "build-max-silent-time", + "The maximum time in seconds that a builer can go without " + "producing any output on stdout/stderr before it is killed. " + "0 means infinity."}; - /* The maximum time in seconds that a builer can go without - producing any output on stdout/stderr before it is killed. 0 - means infinity. */ - time_t maxSilentTime; + Setting<time_t> buildTimeout{this, 0, "build-timeout", + "The maximum duration in seconds that a builder can run. " + "0 means infinity."}; - /* The maximum duration in seconds that a builder can run. 0 - means infinity. */ - time_t buildTimeout; + Setting<bool> useBuildHook{this, true, "remote-builds", + "Whether to use build hooks (for distributed builds)."}; - /* Whether to use build hooks (for distributed builds). Sometimes - users want to disable this from the command-line. */ - bool useBuildHook; + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + "The path of the helper program that executes builds to remote machines."}; - /* Amount of reserved space for the garbage collector - (/nix/var/nix/db/reserved). */ - off_t reservedSize; + Setting<std::string> builders{this, "", "builders", + "A semicolon-separated list of build machines, in the format of nix.machines."}; - /* Whether SQLite should use fsync. */ - bool fsyncMetadata; + Setting<Strings> builderFiles{this, + {nixConfDir + "/machines"}, "builder-files", + "A list of files specifying build machines."}; - /* Whether SQLite should use WAL mode. */ - bool useSQLiteWAL; + Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", + "Amount of reserved disk space for the garbage collector."}; - /* Whether to call sync() before registering a path as valid. */ - bool syncBeforeRegistering; + Setting<bool> fsyncMetadata{this, true, "fsync-metadata", + "Whether SQLite should use fsync()."}; - /* Whether to use substitutes. */ - bool useSubstitutes; + Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", + "Whether SQLite should use WAL mode."}; - /* The Unix group that contains the build users. */ - string buildUsersGroup; + Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering", + "Whether to call sync() before registering a path as valid."}; - /* Set of ssh connection strings for the ssh substituter */ - Strings sshSubstituterHosts; + Setting<bool> useSubstitutes{this, true, "build-use-substitutes", + "Whether to use substitutes."}; - /* Whether to use the ssh substituter at all */ - bool useSshSubstituter; + Setting<std::string> buildUsersGroup{this, "", "build-users-group", + "The Unix group that contains the build users."}; - /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ - bool impersonateLinux26; + Setting<bool> impersonateLinux26{this, false, "build-impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels."}; - /* Whether to store build logs. */ - bool keepLog; + Setting<bool> keepLog{this, true, "build-keep-log", + "Whether to store build logs."}; - /* Whether to compress logs. */ - bool compressLog; + Setting<bool> compressLog{this, true, "build-compress-log", + "Whether to compress logs."}; - /* Maximum number of bytes a builder can write to stdout/stderr - before being killed (0 means no limit). */ - unsigned long maxLogSize; + Setting<unsigned long> maxLogSize{this, 0, "build-max-log-size", + "Maximum number of bytes a builder can write to stdout/stderr " + "before being killed (0 means no limit)."}; /* When build-repeat > 0 and verboseBuild == true, whether to print repeated builds (i.e. builds other than the first one) to stderr. Hack to prevent Hydra logs from being polluted. */ bool printRepeatedBuilds = true; - /* How often (in seconds) to poll for locks. */ - unsigned int pollInterval; + Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", + "How often (in seconds) to poll for locks."}; - /* Whether to check if new GC roots can in fact be found by the - garbage collector. */ - bool checkRootReachability; + Setting<bool> checkRootReachability{this, false, "gc-check-reachability", + "Whether to check if new GC roots can in fact be found by the " + "garbage collector."}; - /* Whether the garbage collector should keep outputs of live - derivations. */ - bool gcKeepOutputs; + Setting<bool> gcKeepOutputs{this, false, "gc-keep-outputs", + "Whether the garbage collector should keep outputs of live derivations."}; - /* Whether the garbage collector should keep derivers of live - paths. */ - bool gcKeepDerivations; + Setting<bool> gcKeepDerivations{this, true, "gc-keep-derivations", + "Whether the garbage collector should keep derivers of live paths."}; - /* Whether to automatically replace files with identical contents - with hard links. */ - bool autoOptimiseStore; + Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", + "Whether to automatically replace files with identical contents with hard links."}; - /* Whether to add derivations as a dependency of user environments - (to prevent them from being GCed). */ - bool envKeepDerivations; + Setting<bool> envKeepDerivations{this, false, "env-keep-derivations", + "Whether to add derivations as a dependency of user environments " + "(to prevent them from being GCed)."}; /* Whether to lock the Nix client and worker to the same CPU. */ bool lockCPU; /* Whether to show a stack trace if Nix evaluation fails. */ - bool showTrace; + bool showTrace = false; + + Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting<SandboxMode> sandboxMode{this, smDisabled, "build-use-sandbox", + "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", + {"build-use-chroot"}}; + + Setting<PathSet> sandboxPaths{this, {}, "build-sandbox-paths", + "The paths to make available inside the build sandbox.", + {"build-chroot-dirs"}}; + + Setting<PathSet> extraSandboxPaths{this, {}, "build-extra-sandbox-paths", + "Additional paths to make available inside the build sandbox.", + {"build-extra-chroot-dirs"}}; + + Setting<bool> restrictEval{this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and to disallow fetching files from the network."}; - /* Whether native-code enabling primops should be enabled */ - bool enableNativeCode; + Setting<size_t> buildRepeat{this, 0, "build-repeat", + "The number of times to repeat a build in order to verify determinism."}; - /* The hook to run just before a build to set derivation-specific - build settings */ - Path preBuildHook; +#if __linux__ + Setting<std::string> sandboxShmSize{this, "50%", "sandbox-dev-shm-size", + "The size of /dev/shm in the build sandbox."}; - /* Path to the netrc file used to obtain usernames/passwords for - downloads. */ - Path netrcFile; + Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", + "The build directory inside the sandbox."}; +#endif + + Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; + +#if __APPLE__ + Setting<bool> darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", + "Whether to log Darwin sandbox access violations to the system log."}; +#endif + + Setting<bool> runDiffHook{this, false, "run-diff-hook", + "Whether to run the program specified by the diff-hook setting " + "repeated builds produce a different result. Typically used to " + "plug in diffoscope."}; + + PathSetting diffHook{this, true, "", "diff-hook", + "A program that prints out the differences between the two paths " + "specified on its command line."}; + + Setting<bool> enforceDeterminism{this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output."}; + + Setting<Strings> binaryCachePublicKeys{this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "binary-cache-public-keys", + "Trusted public keys for secure substitution."}; + + Setting<Strings> secretKeyFiles{this, {}, "secret-key-files", + "Secret keys with which to sign local builds."}; + + Setting<size_t> binaryCachesParallelConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<bool> enableHttp2{this, true, "enable-http2", + "Whether to enable HTTP/2 support."}; + + Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", + "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + + Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches", + "Obsolete."}; + + Setting<Strings> substituters{this, + nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + "substituters", + "The URIs of substituters (such as https://cache.nixos.org/).", + {"binary-caches"}}; + + // FIXME: provide a way to add to option values. + Setting<Strings> extraSubstituters{this, {}, "extra-substituters", + "Additional URIs of substituters.", + {"extra-binary-caches"}}; + + Setting<StringSet> trustedSubstituters{this, {}, "trusted-substituters", + "Disabled substituters that may be enabled via the substituters option by untrusted users.", + {"trusted-binary-caches"}}; + + Setting<Strings> trustedUsers{this, {"root"}, "trusted-users", + "Which users or groups are trusted to ask the daemon to do unsafe things."}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting<Strings> allowedUsers{this, {"*"}, "allowed-users", + "Which users or groups are allowed to connect to the daemon."}; + + Setting<bool> printMissing{this, true, "print-missing", + "Whether to print what paths need to be built or downloaded."}; + + Setting<std::string> preBuildHook{this, +#if __APPLE__ + nixLibexecDir + "/nix/resolve-system-dependencies", +#else + "", +#endif + "pre-build-hook", + "A program to run just before a build to set derivation-specific build settings."}; + + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", + "Path to the netrc file used to obtain usernames/passwords for downloads."}; /* Path to the SSL CA file used */ Path caFile; - /* Whether we allow import-from-derivation */ - bool enableImportFromDerivation; + Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + CaseHackSetting useCaseHack{this, "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; -private: - SettingsMap settings, overrides; + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; - void _get(string & res, const string & name); - void _get(bool & res, const string & name); - void _get(StringSet & res, const string & name); - void _get(Strings & res, const string & name); - template<class N> void _get(N & res, const string & name); + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 37a7d6ace142..cead81514ab4 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -50,7 +50,6 @@ protected: { try { DownloadRequest request(cacheUri + "/" + path); - request.showProgress = DownloadRequest::no; request.head = true; request.tries = 5; getDownloader()->download(request); @@ -76,7 +75,6 @@ protected: std::function<void(std::exception_ptr exc)> failure) override { DownloadRequest request(cacheUri + "/" + path); - request.showProgress = DownloadRequest::no; request.tries = 8; getDownloader()->enqueueDownload(request, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0e838846c794..e09932e3d182 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "worker-protocol.hh" #include "ssh.hh" +#include "derivations.hh" namespace nix { @@ -12,11 +13,19 @@ static std::string uriScheme = "ssh://"; struct LegacySSHStore : public Store { + const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; + + // Hack for getting remote build log output. + const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + struct Connection { std::unique_ptr<SSHMaster::Connection> sshConn; FdSink to; FdSource from; + int remoteVersion; }; std::string host; @@ -29,16 +38,17 @@ struct LegacySSHStore : public Store : Store(params) , host(host) , connections(make_ref<Pool<Connection>>( - std::max(1, std::stoi(get(params, "max-connections", "1"))), + std::max(1, (int) maxConnections), [this]() { return openConnection(); }, [](const ref<Connection> & r) { return true; } )) , master( host, - get(params, "ssh-key", ""), + sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - get(params, "compress", "") == "true") + compress, + logFD) { } @@ -49,8 +59,6 @@ struct LegacySSHStore : public Store conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - int remoteVersion; - try { conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; conn->to.flush(); @@ -58,8 +66,8 @@ struct LegacySSHStore : public Store unsigned int magic = readInt(conn->from); if (magic != SERVE_MAGIC_2) throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host); - remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host); } catch (EndOfFile & e) { @@ -144,12 +152,6 @@ struct LegacySSHStore : public Store sink(*savedNAR.data); } - /* Unsupported methods. */ - [[noreturn]] void unsupported() - { - throw Error("operation not supported on SSH stores"); - } - PathSet queryAllValidPaths() override { unsupported(); } void queryReferrers(const Path & path, PathSet & referrers) override @@ -173,12 +175,36 @@ struct LegacySSHStore : public Store const PathSet & references, bool repair) override { unsupported(); } - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { unsupported(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { unsupported(); } + { + auto conn(connections->get()); + + conn->to + << cmdBuildDerivation + << drvPath + << drv + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to + << settings.buildRepeat + << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status) readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + + return status; + } void ensurePath(const Path & path) override { unsupported(); } @@ -201,9 +227,6 @@ struct LegacySSHStore : public Store void addSignatures(const Path & storePath, const StringSet & sigs) override { unsupported(); } - bool isTrusted() override - { return true; } - void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -239,6 +262,11 @@ struct LegacySSHStore : public Store return readStorePaths<PathSet>(*this, conn->from); } + + void connect() override + { + auto conn(connections->get()); + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 57e1b8a09fe6..bf28a1c70c62 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -9,9 +9,6 @@ namespace nix { LocalFSStore::LocalFSStore(const Params & params) : Store(params) - , rootDir(get(params, "root")) - , stateDir(canonPath(get(params, "state", rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir))) - , logDir(canonPath(get(params, "log", rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir))) { } @@ -34,7 +31,7 @@ struct LocalStoreAccessor : public FSAccessor auto realPath = toRealPath(path); struct stat st; - if (lstat(path.c_str(), &st)) { + if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; throw SysError(format("getting status of ‘%1%’") % path); } @@ -54,7 +51,7 @@ struct LocalStoreAccessor : public FSAccessor { auto realPath = toRealPath(path); - auto entries = nix::readDirectory(path); + auto entries = nix::readDirectory(realPath); StringSet res; for (auto & entry : entries) @@ -76,7 +73,8 @@ struct LocalStoreAccessor : public FSAccessor ref<FSAccessor> LocalFSStore::getFSAccessor() { - return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); + return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( + std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } void LocalFSStore::narFromPath(const Path & path, Sink & sink) @@ -88,6 +86,8 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink) const string LocalFSStore::drvsLogDir = "drvs"; + + std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) { auto path(path_); @@ -110,8 +110,8 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) Path logPath = j == 0 - ? (format("%1%/%2%/%3%/%4%") % logDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() - : (format("%1%/%2%/%3%") % logDir % drvsLogDir % baseName).str(); + ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8610841d7229..207e8a40b6d3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -38,13 +38,14 @@ namespace nix { LocalStore::LocalStore(const Params & params) : Store(params) , LocalFSStore(params) - , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir)) + , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"} + , realStoreDir(realStoreDir_) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") , trashDir(realStoreDir + "/trash") - , requireSigs(trim(settings.get("signed-binary-caches", std::string("*"))) != "") // FIXME: rename option , publicKeys(getDefaultPublicKeys()) { auto state(_state.lock()); @@ -74,7 +75,7 @@ LocalStore::LocalStore(const Params & params) mode_t perm = 01775; - struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError(format("warning: the group ‘%1%’ specified in ‘build-users-group’ does not exist") % settings.buildUsersGroup); @@ -914,10 +915,16 @@ void LocalStore::invalidatePath(State & state, const Path & path) void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { + assert(info.narHash); + Hash h = hashString(htSHA256, *nar); if (h != info.narHash) - throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % - info.path % info.narHash.to_string() % h.to_string()); + throw Error("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’", + info.path, info.narHash.to_string(), h.to_string()); + + if (nar->size() != info.narSize) + throw Error("szie mismatch importing path ‘%s’; expected %s, got %s", + info.path, info.narSize, nar->size()); if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys)) throw Error("cannot add path ‘%s’ because it lacks a valid signature", info.path); @@ -1003,7 +1010,6 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, info.path = dstPath; info.narHash = hash.first; info.narSize = hash.second; - info.ultimate = true; info.ca = makeFixedOutputCA(recursive, h); registerValidPath(info); } @@ -1066,7 +1072,6 @@ Path LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = references; - info.ultimate = true; info.ca = "text:" + hash.to_string(); registerValidPath(info); } @@ -1332,9 +1337,9 @@ void LocalStore::signPathInfo(ValidPathInfo & info) { // FIXME: keep secret keys in memory. - auto secretKeyFiles = settings.get("secret-key-files", Strings()); + auto secretKeyFiles = settings.secretKeyFiles; - for (auto & secretKeyFile : secretKeyFiles) { + for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); info.sign(secretKey); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 28e9a31c9feb..f2c40e96464b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -26,14 +26,9 @@ struct Derivation; struct OptimiseStats { - unsigned long filesLinked; - unsigned long long bytesFreed; - unsigned long long blocksFreed; - OptimiseStats() - { - filesLinked = 0; - bytesFreed = blocksFreed = 0; - } + unsigned long filesLinked = 0; + unsigned long long bytesFreed = 0; + unsigned long long blocksFreed = 0; }; @@ -72,6 +67,8 @@ private: public: + PathSetting realStoreDir_; + const Path realStoreDir; const Path dbDir; const Path linksDir; @@ -81,7 +78,9 @@ public: private: - bool requireSigs; + Setting<bool> requireSigs{(Store*) this, + settings.signedBinaryCaches != "", // FIXME + "require-sigs", "whether store paths should have a trusted signature on import"}; PublicKeys publicKeys; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 9d5c04dca0c5..e06002587f94 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -27,7 +27,8 @@ libstore_CXXFLAGS = \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ - -DBASH_PATH="\"$(bash)\"" + -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ + -DLSOF=\"$(lsof)\" $(d)/local-store.cc: $(d)/schema.sql.hh diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc new file mode 100644 index 000000000000..7491037b2d79 --- /dev/null +++ b/src/libstore/machines.cc @@ -0,0 +1,91 @@ +#include "machines.hh" +#include "util.hh" +#include "globals.hh" + +#include <algorithm> + +namespace nix { + +Machine::Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) : + storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) +{} + +bool Machine::allSupported(const std::set<string> & features) const { + return std::all_of(features.begin(), features.end(), + [&](const string & feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); +} + +bool Machine::mandatoryMet(const std::set<string> & features) const { + return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string & feature) { + return features.count(feature); + }); +} + +void parseMachines(const std::string & s, Machines & machines) +{ + for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + chomp(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; + auto tokens = tokenizeString<std::vector<string>>(line); + auto sz = tokens.size(); + if (sz < 1) + throw FormatError("bad machine specification ‘%s’", line); + + auto isSet = [&](size_t n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; + + machines.emplace_back(tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() +{ + Machines machines; + + for (auto & file : settings.builderFiles.get()) { + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + } + } + + parseMachines(settings.builders, machines); + + return machines; +} + +} diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh new file mode 100644 index 000000000000..de92eb924e4a --- /dev/null +++ b/src/libstore/machines.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct Machine { + + const string storeUri; + const std::vector<string> systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set<string> supportedFeatures; + const std::set<string> mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set<string> & features) const; + + bool mandatoryMet(const std::set<string> & features) const; + + Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); +}; + +typedef std::vector<Machine> Machines; + +void parseMachines(const std::string & s, Machines & machines); + +Machines getMachines(); + +} diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 4cb5de7449ea..82595e76a9b5 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -2,6 +2,8 @@ #include "archive.hh" #include <map> +#include <stack> +#include <algorithm> namespace nix { @@ -16,16 +18,16 @@ struct NarMember size_t start, size; std::string target; + + /* If this is a directory, all the children of the directory. */ + std::map<std::string, NarMember> children; }; struct NarIndexer : ParseSink, StringSource { - // FIXME: should store this as a tree. Now we're vulnerable to - // O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}). - typedef std::map<Path, NarMember> Members; - Members members; + NarMember root; + std::stack<NarMember*> parents; - Path currentPath; std::string currentStart; bool isExec = false; @@ -33,28 +35,45 @@ struct NarIndexer : ParseSink, StringSource { } + void createMember(const Path & path, NarMember member) { + size_t level = std::count(path.begin(), path.end(), '/'); + while(parents.size() > level) { + parents.pop(); + } + + if(parents.empty()) { + root = std::move(member); + parents.push(&root); + } else { + if(parents.top()->type != FSAccessor::Type::tDirectory) { + throw Error(format("NAR file missing parent directory of path ‘%1%’") % path); + } + auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); + parents.push(&result.first->second); + } + } + void createDirectory(const Path & path) override { - members.emplace(path, - NarMember{FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 }); } void createRegularFile(const Path & path) override { - currentPath = path; + createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 }); } void isExecutable() override { - isExec = true; + parents.top()->isExecutable = true; } void preallocateContents(unsigned long long size) override { currentStart = string(s, pos, 16); assert(size <= std::numeric_limits<size_t>::max()); - members.emplace(currentPath, - NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size}); + parents.top()->size = (size_t)size; + parents.top()->start = pos; } void receiveContents(unsigned char * data, unsigned int len) override @@ -68,16 +87,42 @@ struct NarIndexer : ParseSink, StringSource void createSymlink(const Path & path, const string & target) override { - members.emplace(path, + createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } - Members::iterator find(const Path & path) + NarMember* find(const Path & path) { - auto i = members.find(path); - if (i == members.end()) + Path canon = path == "" ? "" : canonPath(path); + NarMember* current = &root; + auto end = path.end(); + for(auto it = path.begin(); it != end; ) { + // because it != end, the remaining component is non-empty so we need + // a directory + if(current->type != FSAccessor::Type::tDirectory) return nullptr; + + // skip slash (canonPath above ensures that this is always a slash) + assert(*it == '/'); + it += 1; + + // lookup current component + auto next = std::find(it, end, '/'); + auto child = current->children.find(std::string(it, next)); + if(child == current->children.end()) return nullptr; + current = &child->second; + + it = next; + } + + return current; + } + + NarMember& at(const Path & path) { + auto result = find(path); + if(result == nullptr) { throw Error(format("NAR file does not contain path ‘%1%’") % path); - return i; + } + return *result; } }; @@ -93,44 +138,41 @@ struct NarAccessor : public FSAccessor Stat stat(const Path & path) override { - auto i = indexer.members.find(path); - if (i == indexer.members.end()) + auto i = indexer.find(path); + if (i == nullptr) return {FSAccessor::Type::tMissing, 0, false}; - return {i->second.type, i->second.size, i->second.isExecutable}; + return {i->type, i->size, i->isExecutable}; } StringSet readDirectory(const Path & path) override { - auto i = indexer.find(path); + auto i = indexer.at(path); - if (i->second.type != FSAccessor::Type::tDirectory) + if (i.type != FSAccessor::Type::tDirectory) throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path); - ++i; StringSet res; - while (i != indexer.members.end() && isInDir(i->first, path)) { - // FIXME: really bad performance. - if (i->first.find('/', path.size() + 1) == std::string::npos) - res.insert(std::string(i->first, path.size() + 1)); - ++i; + for(auto&& child : i.children) { + res.insert(child.first); + } return res; } std::string readFile(const Path & path) override { - auto i = indexer.find(path); - if (i->second.type != FSAccessor::Type::tRegular) + auto i = indexer.at(path); + if (i.type != FSAccessor::Type::tRegular) throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path); - return std::string(*nar, i->second.start, i->second.size); + return std::string(*nar, i.start, i.size); } std::string readLink(const Path & path) override { - auto i = indexer.find(path); - if (i->second.type != FSAccessor::Type::tSymlink) + auto i = indexer.at(path); + if (i.type != FSAccessor::Type::tSymlink) throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path); - return i->second.target; + return i.target; } }; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cf234e35d373..56167c4dfae8 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -220,8 +220,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - if (st.st_size) - printInfo(format("‘%1%’ has maximum number of links") % linkPath); + debug("‘%s’ has reached maximum number of links", linkPath); return; } throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path); @@ -241,7 +240,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) for (auto & i : paths) { addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); + //Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c9c590787450..be8819bbc004 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -43,7 +43,7 @@ template Paths readStorePaths(Store & store, Source & from); RemoteStore::RemoteStore(const Params & params) : Store(params) , connections(make_ref<Pool<Connection>>( - std::max(1, std::stoi(get(params, "max-connections", "1"))), + std::max(1, (int) maxConnections), [this]() { return openConnectionWrapper(); }, [](const ref<Connection> & r) { return r->to.good() && r->from.good(); } )) @@ -100,7 +100,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() throw Error(format("socket path ‘%1%’ is too long") % socketPath); strcpy(addr.sun_path, socketPath.c_str()); - if (connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); conn->from.fd = conn->fd.get(); @@ -166,9 +166,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - Settings::SettingsMap overrides = settings.getOverrides(); - if (overrides["ssh-auth-sock"] == "") - overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK"); + auto overrides = settings.getSettings(true); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second; @@ -416,7 +414,9 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, try { conn->to.written = 0; conn->to.warn = true; + connections->incCapacity(); dumpPath(srcPath, conn->to, filter); + connections->decCapacity(); conn->to.warn = false; conn->processStderr(); } catch (SysError & e) { @@ -613,6 +613,12 @@ void RemoteStore::queryMissing(const PathSet & targets, } +void RemoteStore::connect() +{ + auto conn(connections->get()); +} + + RemoteStore::Connection::~Connection() { try { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index db8da7eaa8c5..ed430e4cabb6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,6 +22,9 @@ class RemoteStore : public virtual Store { public: + const Setting<int> maxConnections{(Store*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -89,6 +92,8 @@ public: PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) override; + void connect() override; + protected: struct Connection diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 3053f908c4e2..245455296013 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -125,22 +125,22 @@ S3Helper::DownloadResult S3Helper::getObject( struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { + const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; + const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + std::string bucketName; Stats stats; S3Helper s3Helper; - std::string narinfoCompression, lsCompression, logCompression; - S3BinaryCacheStoreImpl( const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1)) - , narinfoCompression(get(params, "narinfo-compression", "")) - , lsCompression(get(params, "ls-compression", "")) - , logCompression(get(params, "log-compression", "")) + , s3Helper(region) { diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 2a81a8b1ebe5..bb536fadfd51 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -14,16 +14,19 @@ class SSHStore : public RemoteStore { public: + const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; + SSHStore(const std::string & host, const Params & params) : Store(params) , RemoteStore(params) , host(host) , master( host, - get(params, "ssh-key", ""), + sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - get(params, "compress", "") == "true") + compress) { } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index e54f3f4ba284..6edabaa3a1d9 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -31,6 +31,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string throw SysError("duping over stdin"); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) + throw SysError("duping over stderr"); Strings args = { "ssh", host.c_str(), "-x", "-a" }; addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index b4396467e54e..18dea227ad1f 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -13,6 +13,7 @@ private: const std::string keyFile; const bool useMaster; const bool compress; + const int logFD; struct State { @@ -27,11 +28,12 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress) + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1) : host(host) , keyFile(keyFile) , useMaster(useMaster) , compress(compress) + , logFD(logFD) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 59348c5d0b5f..e6cbd53dc80a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -167,7 +167,7 @@ void checkStoreName(const string & name) collisions (for security). For instance, it shouldn't be feasible to come up with a derivation whose output path collides with the path for a copied source. The former would have a <s> starting with - "output:out:", while the latter would have a <2> starting with + "output:out:", while the latter would have a <s> starting with "source:". */ @@ -241,8 +241,8 @@ Path Store::computeStorePathForText(const string & name, const string & s, Store::Store(const Params & params) - : storeDir(get(params, "store", settings.nixStore)) - , state({std::stoi(get(params, "path-info-cache-size", "65536"))}) + : Config(params) + , state({(size_t) pathInfoCacheSize}) { } @@ -482,21 +482,23 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths if (showClosureSize) jsonPath.attr("closureSize", getClosureSize(storePath)); - if (!includeImpureInfo) continue; + if (includeImpureInfo) { - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); + if (info->deriver != "") + jsonPath.attr("deriver", info->deriver); - if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); - if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + if (info->ultimate) + jsonPath.attr("ultimate", info->ultimate); + + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto & sig : info->sigs) + jsonSigs.elem(sig); + } - if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); - for (auto & sig : info->sigs) - jsonSigs.elem(sig); } } } @@ -523,6 +525,17 @@ const Store::Stats & Store::getStats() } +void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & path : paths) + if (isDerivation(path)) + unsupported(); + + if (queryValidPaths(paths).size() != paths.size()) + unsupported(); +} + + void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, bool repair, bool dontCheckSigs) { @@ -531,15 +544,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); - if (srcStore->isTrusted()) - dontCheckSigs = true; - if (!info->narHash && dontCheckSigs) { auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); info = info2; } + assert(info->narHash); + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + assert(info->narHash); + dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); } @@ -698,10 +718,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStore(const std::string & uri_) +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) { auto uri(uri_); - Store::Params params; + Store::Params params(extraParams); auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { @@ -711,17 +732,16 @@ ref<Store> openStore(const std::string & uri_) } uri = uri_.substr(0, q); } - return openStore(uri, params); -} -ref<Store> openStore(const std::string & uri, const Store::Params & params) -{ for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); - if (store) return ref<Store>(store); + if (store) { + store->warnUnknownSettings(); + return ref<Store>(store); + } } - throw Error(format("don't know how to open Nix store ‘%s’") % uri); + throw Error("don't know how to open Nix store ‘%s’", uri); } @@ -731,7 +751,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir) return tDaemon; } else if (uri == "local") { return tLocal; - } else if (uri == "") { + } else if (uri == "" || uri == "auto") { if (access(stateDir.c_str(), R_OK | W_OK) == 0) return tLocal; else if (pathExists(settings.nixDaemonSocketFile)) @@ -779,14 +799,10 @@ std::list<ref<Store>> getDefaultSubstituters() state->stores.push_back(openStore(uri)); }; - Strings defaultSubstituters; - if (settings.nixStore == "/nix/store") - defaultSubstituters.push_back("https://cache.nixos.org/"); - - for (auto uri : settings.get("substituters", settings.get("binary-caches", defaultSubstituters))) + for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.get("extra-binary-caches", Strings())) + for (auto uri : settings.extraSubstituters.get()) addStore(uri); state->done = true; @@ -795,7 +811,8 @@ std::list<ref<Store>> getDefaultSubstituters() } -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute) +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute, bool dontCheckSigs) { PathSet valid = to->queryValidPaths(storePaths, substitute); @@ -805,7 +822,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool std::string copiedLabel = "copied"; - logger->setExpected(copiedLabel, missing.size()); + //logger->setExpected(copiedLabel, missing.size()); ThreadPool pool; @@ -821,13 +838,14 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool checkInterrupt(); if (!to->isValidPath(storePath)) { - Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); + //Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); - copyStorePath(from, to, storePath); + copyStorePath(from, to, storePath, false, dontCheckSigs); - logger->incProgress(copiedLabel); + //logger->incProgress(copiedLabel); } else - logger->incExpected(copiedLabel, -1); + ; + //logger->incExpected(copiedLabel, -1); }); pool.process(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 68c59a9f2922..929c95a0f2f8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -6,6 +6,7 @@ #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" +#include "config.hh" #include <atomic> #include <limits> @@ -17,6 +18,12 @@ namespace nix { +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) +MakeError(Unsupported, Error) + + struct BasicDerivation; struct Derivation; class FSAccessor; @@ -81,12 +88,7 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed; - - GCResults() - { - bytesFreed = 0; - } + unsigned long long bytesFreed = 0; }; @@ -111,9 +113,8 @@ struct ValidPathInfo uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it was built - locally or is content-addressable (e.g. added via addToStore() - or the result of a fixed-output derivation). */ + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ bool ultimate = false; StringSet sigs; // note: not necessarily verified @@ -229,13 +230,17 @@ struct BuildResult }; -class Store : public std::enable_shared_from_this<Store> +class Store : public std::enable_shared_from_this<Store>, public Config { public: typedef std::map<std::string, std::string> Params; - const Path storeDir; + const PathSetting storeDir_{this, false, settings.nixStore, + "store", "path to the Nix store"}; + const Path storeDir = storeDir_; + + const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; protected: @@ -414,7 +419,7 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for @@ -564,10 +569,6 @@ public: const Stats & getStats(); - /* Whether this store paths from this store can be imported even - if they lack a signature. */ - virtual bool isTrusted() { return false; } - /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr<std::string> getBuildLog(const Path & path) @@ -580,19 +581,39 @@ public: state.lock()->pathInfoCache.clear(); } + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect() { }; + protected: Stats stats; + /* Unsupported methods. */ + [[noreturn]] void unsupported() + { + throw Unsupported("requested operation is not supported by store ‘%s’", getUri()); + } + }; class LocalFSStore : public virtual Store { public: - const Path rootDir; - const Path stateDir; - const Path logDir; + + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to emit the call to the Option constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; + const static string drvsLogDir; LocalFSStore(const Params & params); @@ -646,23 +667,35 @@ void removeTempRoots(); /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: - * ‘direct’: The Nix store in /nix/store and database in + * ‘local’: The Nix store in /nix/store and database in /nix/var/nix/db, accessed directly. * ‘daemon’: The Nix store accessed via a Unix domain socket connection to nix-daemon. + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + * ‘file://<path>’: A binary cache stored in <path>. - If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on - whether the user has write access to the local Nix store/database. - set to true *unless* you're going to collect garbage. */ -ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); + * ‘https://<path>’: A binary cache accessed via HTTP. + + * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]<host>’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. -ref<Store> openStore(const std::string & uri, const Store::Params & params); + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), + const Store::Params & extraParams = Store::Params()); -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false); +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute = false, bool dontCheckSigs = false); enum StoreType { tDaemon, @@ -710,10 +743,4 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); - -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) - - } diff --git a/src/libutil/config.cc b/src/libutil/config.cc new file mode 100644 index 000000000000..f7a46bfee63f --- /dev/null +++ b/src/libutil/config.cc @@ -0,0 +1,231 @@ +#include "config.hh" +#include "args.hh" +#include "json.hh" + +namespace nix { + +void Config::set(const std::string & name, const std::string & value) +{ + auto i = _settings.find(name); + if (i == _settings.end()) + throw UsageError("unknown setting '%s'", name); + i->second.setting->set(value); + i->second.setting->overriden = true; +} + +void Config::addSetting(AbstractSetting * setting) +{ + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData(true, setting)); + + bool set = false; + + auto i = initials.find(setting->name); + if (i != initials.end()) { + setting->set(i->second); + setting->overriden = true; + initials.erase(i); + set = true; + } + + for (auto & alias : setting->aliases) { + auto i = initials.find(alias); + if (i != initials.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { + setting->set(i->second); + setting->overriden = true; + initials.erase(i); + set = true; + } + } + } +} + +void Config::warnUnknownSettings() +{ + for (auto & i : initials) + warn("unknown setting '%s'", i.first); +} + +StringMap Config::getSettings(bool overridenOnly) +{ + StringMap res; + for (auto & opt : _settings) + if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + res.emplace(opt.first, opt.second.setting->to_string()); + return res; +} + +void Config::applyConfigFile(const Path & path, bool fatal) +{ + try { + string contents = readFile(path); + + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2 || tokens[1] != "=") + throw UsageError("illegal configuration line ‘%1%’ in ‘%2%’", line, path); + + string name = tokens[0]; + + vector<string>::iterator i = tokens.begin(); + advance(i, 2); + + try { + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + } catch (UsageError & e) { + if (fatal) throw; + warn("in configuration file '%s': %s", path, e.what()); + } + }; + } catch (SysError &) { } +} + +void Config::resetOverriden() +{ + for (auto & s : _settings) + s.second.setting->overriden = false; +} + +void Config::toJSON(JSONObject & out) +{ + for (auto & s : _settings) + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } +} + +AbstractSetting::AbstractSetting( + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases) + : name(name), description(description), aliases(aliases) +{ +} + +void AbstractSetting::toJSON(JSONPlaceholder & out) +{ + out.write(to_string()); +} + +template<typename T> +void BaseSetting<T>::toJSON(JSONPlaceholder & out) +{ + out.write(value); +} + +template<> void BaseSetting<std::string>::set(const std::string & str) +{ + value = str; +} + +template<> std::string BaseSetting<std::string>::to_string() +{ + return value; +} + +template<typename T> +void BaseSetting<T>::set(const std::string & str) +{ + static_assert(std::is_integral<T>::value, "Integer required."); + if (!string2Int(str, value)) + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template<typename T> +std::string BaseSetting<T>::to_string() +{ + static_assert(std::is_integral<T>::value, "Integer required."); + return std::to_string(value); +} + +template<> void BaseSetting<bool>::set(const std::string & str) +{ + if (str == "true" || str == "yes" || str == "1") + value = true; + else if (str == "false" || str == "no" || str == "0") + value = false; + else + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); +} + +template<> std::string BaseSetting<bool>::to_string() +{ + return value ? "true" : "false"; +} + +template<> void BaseSetting<Strings>::set(const std::string & str) +{ + value = tokenizeString<Strings>(str); +} + +template<> std::string BaseSetting<Strings>::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template<> void BaseSetting<StringSet>::set(const std::string & str) +{ + value = tokenizeString<StringSet>(str); +} + +template<> std::string BaseSetting<StringSet>::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template class BaseSetting<int>; +template class BaseSetting<unsigned int>; +template class BaseSetting<long>; +template class BaseSetting<unsigned long>; +template class BaseSetting<long long>; +template class BaseSetting<unsigned long long>; +template class BaseSetting<bool>; +template class BaseSetting<std::string>; + +void PathSetting::set(const std::string & str) +{ + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); +} + +} diff --git a/src/libutil/config.hh b/src/libutil/config.hh new file mode 100644 index 000000000000..77620d47d371 --- /dev/null +++ b/src/libutil/config.hh @@ -0,0 +1,192 @@ +#include <map> +#include <set> + +#include "types.hh" + +#pragma once + +namespace nix { + +class Args; +class AbstractSetting; +class JSONPlaceholder; +class JSONObject; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting<T> members: + + class MyClass : private Config + { + Setting<int> foo{this, 123, "foo", "the number of foos to use"}; + Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config +{ + friend class AbstractSetting; + + struct SettingData + { + bool isAlias; + AbstractSetting * setting; + SettingData(bool isAlias, AbstractSetting * setting) + : isAlias(isAlias), setting(setting) + { } + }; + + std::map<std::string, SettingData> _settings; + + StringMap initials; + +public: + + Config(const StringMap & initials) + : initials(initials) + { } + + void set(const std::string & name, const std::string & value); + + void addSetting(AbstractSetting * setting); + + void warnUnknownSettings(); + + StringMap getSettings(bool overridenOnly = false); + + void applyConfigFile(const Path & path, bool fatal = false); + + void resetOverriden(); + + void toJSON(JSONObject & out); +}; + +class AbstractSetting +{ + friend class Config; + +public: + + const std::string name; + const std::string description; + const std::set<std::string> aliases; + + int created = 123; + + bool overriden = false; + +protected: + + AbstractSetting( + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases); + + virtual ~AbstractSetting() + { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } + + virtual void set(const std::string & value) = 0; + + virtual std::string to_string() = 0; + + virtual void toJSON(JSONPlaceholder & out); + + bool isOverriden() { return overriden; } +}; + +/* A setting of type T. */ +template<typename T> +class BaseSetting : public AbstractSetting +{ +protected: + + T value; + +public: + + BaseSetting(const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : AbstractSetting(name, description, aliases) + , value(def) + { } + + operator const T &() const { return value; } + operator T &() { return value; } + const T & get() const { return value; } + bool operator ==(const T & v2) const { return value == v2; } + bool operator !=(const T & v2) const { return value != v2; } + void operator =(const T & v) { assign(v); } + virtual void assign(const T & v) { value = v; } + + void set(const std::string & str) override; + + std::string to_string() override; + + void toJSON(JSONPlaceholder & out) override; +}; + +template<typename T> +std::ostream & operator <<(std::ostream & str, const BaseSetting<T> & opt) +{ + str << (const T &) opt; + return str; +} + +template<typename T> +bool operator ==(const T & v1, const BaseSetting<T> & v2) { return v1 == (const T &) v2; } + +template<typename T> +class Setting : public BaseSetting<T> +{ +public: + Setting(Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<T>(def, name, description, aliases) + { + options->addSetting(this); + } + + void operator =(const T & v) { this->assign(v); } +}; + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public BaseSetting<Path> +{ + bool allowEmpty; + +public: + + PathSetting(Config * options, + bool allowEmpty, + const Path & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<Path>(def, name, description, aliases) + , allowEmpty(allowEmpty) + { + options->addSetting(this); + } + + void set(const std::string & str) override; + + Path operator +(const char * p) const { return value + p; } + + void operator =(const Path & v) { this->assign(v); } +}; + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9f4afd93c2fc..fa1bb5d97183 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -224,7 +224,7 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, unsigned int len) + const unsigned char * bytes, size_t len) { if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 6023d1d4fb84..b8b8ef9c8cca 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -19,49 +19,32 @@ void toJSON(std::ostream & str, const char * start, const char * end) str << '"'; } -void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} - void toJSON(std::ostream & str, const char * s) { if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); } -void toJSON(std::ostream & str, unsigned long long n) -{ - str << n; -} - -void toJSON(std::ostream & str, unsigned long n) -{ - str << n; -} - -void toJSON(std::ostream & str, long n) -{ - str << n; -} +template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; } +template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; } +template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; } +template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; } +template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; } +template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; } -void toJSON(std::ostream & str, unsigned int n) +template<> void toJSON<std::string>(std::ostream & str, const std::string & s) { - str << n; -} - -void toJSON(std::ostream & str, int n) -{ - str << n; + toJSON(str, s.c_str(), s.c_str() + s.size()); } -void toJSON(std::ostream & str, double f) +template<> void toJSON<bool>(std::ostream & str, const bool & b) { - str << f; + str << (b ? "true" : "false"); } -void toJSON(std::ostream & str, bool b) +template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b) { - str << (b ? "true" : "false"); + str << "null"; } JSONWriter::JSONWriter(std::ostream & str, bool indent) diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 03eecb732586..595e9bbe3491 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -7,15 +7,10 @@ namespace nix { void toJSON(std::ostream & str, const char * start, const char * end); -void toJSON(std::ostream & str, const std::string & s); void toJSON(std::ostream & str, const char * s); -void toJSON(std::ostream & str, unsigned long long n); -void toJSON(std::ostream & str, unsigned long n); -void toJSON(std::ostream & str, long n); -void toJSON(std::ostream & str, unsigned int n); -void toJSON(std::ostream & str, int n); -void toJSON(std::ostream & str, double f); -void toJSON(std::ostream & str, bool b); + +template<typename T> +void toJSON(std::ostream & str, const T & n); class JSONWriter { diff --git a/src/libutil/lazy.hh b/src/libutil/lazy.hh new file mode 100644 index 000000000000..d073e486c2eb --- /dev/null +++ b/src/libutil/lazy.hh @@ -0,0 +1,48 @@ +#include <exception> +#include <functional> +#include <mutex> + +namespace nix { + +/* A helper class for lazily-initialized variables. + + Lazy<T> var([]() { return value; }); + + declares a variable of type T that is initialized to 'value' (in a + thread-safe way) on first use, that is, when var() is first + called. If the initialiser code throws an exception, then all + subsequent calls to var() will rethrow that exception. */ +template<typename T> +class Lazy +{ + + typedef std::function<T()> Init; + + Init init; + + std::once_flag done; + + T value; + + std::exception_ptr ex; + +public: + + Lazy(Init init) : init(init) + { } + + const T & operator () () + { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) std::rethrow_exception(ex); + return value; + } +}; + +} diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d9e8d22d7685..2d0acca24216 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,9 +1,16 @@ #include "logging.hh" #include "util.hh" +#include <atomic> + namespace nix { -Logger * logger = 0; +Logger * logger = makeDefaultLogger(); + +void Logger::warn(const std::string & msg) +{ + log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); +} class SimpleLogger : public Logger { @@ -37,12 +44,7 @@ public: writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - log(lvl, fs); - } - - void stopActivity(Activity & activity) override + void event(const Event & ev) override { } }; @@ -52,7 +54,7 @@ Verbosity verbosity = lvlInfo; void warnOnce(bool & haveWarned, const FormatOrString & fs) { if (!haveWarned) { - printError(format("warning: %1%") % fs.s); + warn(fs.s); haveWarned = true; } } @@ -74,4 +76,8 @@ Logger * makeDefaultLogger() return new SimpleLogger(); } +std::atomic<uint64_t> Activity::nextId{(uint64_t) getpid() << 32}; + +Activity::Activity() : id(nextId++) { }; + } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 3f83664794f7..ddfc336fee07 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -13,7 +13,64 @@ typedef enum { lvlVomit } Verbosity; -class Activity; +class Activity +{ + static std::atomic<uint64_t> nextId; +public: + typedef uint64_t t; + const t id; + Activity(); + Activity(const Activity & act) : id(act.id) { }; + Activity(uint64_t id) : id(id) { }; +}; + +typedef enum { + evBuildCreated = 0, + evBuildStarted = 1, + evBuildOutput = 2, + evBuildFinished = 3, + evDownloadCreated = 4, + evDownloadDestroyed = 5, + evDownloadProgress = 6, + evDownloadSucceeded = 7, + evSubstitutionCreated = 8, + evSubstitutionStarted = 9, + evSubstitutionFinished = 10, +} EventType; + +struct Event +{ + struct Field + { + // FIXME: use std::variant. + enum { tInt, tString } type; + uint64_t i = 0; + std::string s; + Field(const std::string & s) : type(tString), s(s) { } + Field(const char * s) : type(tString), s(s) { } + Field(const uint64_t & i) : type(tInt), i(i) { } + Field(const Activity & act) : type(tInt), i(act.id) { } + }; + + typedef std::vector<Field> Fields; + + EventType type; + Fields fields; + + std::string getS(size_t n) const + { + assert(n < fields.size()); + assert(fields[n].type == Field::tString); + return fields[n].s; + } + + uint64_t getI(size_t n) const + { + assert(n < fields.size()); + assert(fields[n].type == Field::tInt); + return fields[n].i; + } +}; class Logger { @@ -30,34 +87,18 @@ public: log(lvlInfo, fs); } - virtual void setExpected(const std::string & label, uint64_t value = 1) { } - virtual void setProgress(const std::string & label, uint64_t value = 1) { } - virtual void incExpected(const std::string & label, uint64_t value = 1) { } - virtual void incProgress(const std::string & label, uint64_t value = 1) { } + virtual void warn(const std::string & msg); -private: - - virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; - - virtual void stopActivity(Activity & activity) = 0; - -}; - -class Activity -{ -public: - Logger & logger; - - Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) - : logger(logger) + template<typename... Args> + void event(EventType type, const Args & ... args) { - logger.startActivity(*this, lvl, fs); + Event ev; + ev.type = type; + nop{(ev.fields.emplace_back(Event::Field(args)), 1)...}; + event(ev); } - ~Activity() - { - logger.stopActivity(*this); - } + virtual void event(const Event & ev) = 0; }; extern Logger * logger; @@ -82,6 +123,14 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) +template<typename... Args> +inline void warn(const std::string & fs, Args... args) +{ + boost::format f(fs); + nop{boost::io::detail::feed(f, args)...}; + logger->warn(f.str()); +} + void warnOnce(bool & haveWarned, const FormatOrString & fs); void writeToStderr(const string & s); diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index 20df21948849..7033090020ef 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -68,6 +68,22 @@ public: state_->max = max; } + void incCapacity() + { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() + { + auto state_(state.lock()); + state_->max--; + } + ~Pool() { auto state_(state.lock()); diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 97d79af9b5d6..9f32d31addbf 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -7,6 +7,7 @@ #include <list> #include <set> #include <memory> +#include <map> #include <boost/format.hpp> @@ -31,6 +32,11 @@ using std::vector; using boost::format; +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template<typename... T> nop(T...) {} }; + + struct FormatOrString { string s; @@ -45,16 +51,6 @@ struct FormatOrString ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion takes place). */ -inline void formatHelper(boost::format & f) -{ -} - -template<typename T, typename... Args> -inline void formatHelper(boost::format & f, T x, Args... args) -{ - formatHelper(f % x, args...); -} - inline std::string fmt(const std::string & s) { return s; @@ -74,7 +70,7 @@ template<typename... Args> inline std::string fmt(const std::string & fs, Args... args) { boost::format f(fs); - formatHelper(f, args...); + nop{boost::io::detail::feed(f, args)...}; return f.str(); } @@ -141,6 +137,7 @@ private: typedef list<string> Strings; typedef set<string> StringSet; +typedef std::map<std::string, std::string> StringMap; /* Paths are just strings. */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a640a64c724e..16f4b232e6c5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,3 +1,4 @@ +#include "lazy.hh" #include "util.hh" #include "affinity.hh" #include "sync.hh" @@ -13,10 +14,12 @@ #include <thread> #include <future> -#include <sys/wait.h> -#include <unistd.h> #include <fcntl.h> #include <limits.h> +#include <pwd.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> #ifdef __APPLE__ #include <sys/syscall.h> @@ -96,6 +99,8 @@ Path absPath(Path path, Path dir) Path canonPath(const Path & path, bool resolveSymlinks) { + assert(path != ""); + string s; if (path[0] != '/') @@ -367,7 +372,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); + //Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -415,18 +420,50 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +static Lazy<Path> getHome2([]() { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) { + char buf[16384]; + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(getuid(), &pwbuf, buf, sizeof(buf), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return homeDir; +}); + +Path getHome() { return getHome2(); } + + Path getCacheDir() { Path cacheDir = getEnv("XDG_CACHE_HOME"); - if (cacheDir.empty()) { - Path homeDir = getEnv("HOME"); - if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set"); - cacheDir = homeDir + "/.cache"; - } + if (cacheDir.empty()) + cacheDir = getHome() + "/.cache"; return cacheDir; } +Path getConfigDir() +{ + Path configDir = getEnv("XDG_CONFIG_HOME"); + if (configDir.empty()) + configDir = getHome() + "/.config"; + return configDir; +} + + +Path getDataDir() +{ + Path dataDir = getEnv("XDG_DATA_HOME"); + if (dataDir.empty()) + dataDir = getHome() + "/.local/share"; + return dataDir; +} + + Paths createDirs(const Path & path) { Paths created; @@ -932,7 +969,12 @@ void closeOnExec(int fd) bool _isInterrupted = false; -thread_local bool interruptThrown = false; +static thread_local bool interruptThrown = false; + +void setInterruptThrown() +{ + interruptThrown = true; +} void _interrupted() { @@ -1047,9 +1089,9 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & suffix) +bool hasPrefix(const string & s, const string & prefix) { - return s.compare(0, suffix.size(), suffix) == 0; + return s.compare(0, prefix.size(), prefix) == 0; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0e6941e4a8db..7ea32e8d9f14 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -110,9 +110,18 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Return the path to $XDG_CACHE_HOME/.cache. */ +/* Return $HOME or the user's home directory from /etc/passwd. */ +Path getHome(); + +/* Return $XDG_CACHE_HOME or $HOME/.cache. */ Path getCacheDir(); +/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +Path getConfigDir(); + +/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +Path getDataDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -264,7 +273,7 @@ void closeOnExec(int fd); extern bool _isInterrupted; -extern thread_local bool interruptThrown; +void setInterruptThrown(); void _interrupted(); @@ -355,6 +364,8 @@ void ignoreException(); #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_BLUE "\e[34;1m" /* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE new file mode 100644 index 000000000000..18e814865a54 --- /dev/null +++ b/src/linenoise/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com> +Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/linenoise/linenoise.c b/src/linenoise/linenoise.c new file mode 100644 index 000000000000..fce14a7c53a3 --- /dev/null +++ b/src/linenoise/linenoise.c @@ -0,0 +1,1199 @@ +/* linenoise.c -- guerrilla line editing library against the idea that a + * line editing lib needs to be 20,000 lines of C code. + * + * You can find the latest source code at: + * + * http://github.com/antirez/linenoise + * + * Does a number of crazy assumptions that happen to be true in 99.9999% of + * the 2010 UNIX computers around. + * + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2016, Salvatore Sanfilippo <antirez at gmail dot com> + * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * ------------------------------------------------------------------------ + * + * References: + * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html + * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html + * + * Todo list: + * - Filter bogus Ctrl+<char> combinations. + * - Win32 support + * + * Bloat: + * - History search like Ctrl+r in readline? + * + * List of escape sequences used by this program, we do everything just + * with three sequences. In order to be so cheap we may have some + * flickering effect with some slow terminal, but the lesser sequences + * the more compatible. + * + * EL (Erase Line) + * Sequence: ESC [ n K + * Effect: if n is 0 or missing, clear from cursor to end of line + * Effect: if n is 1, clear from beginning of line to cursor + * Effect: if n is 2, clear entire line + * + * CUF (CUrsor Forward) + * Sequence: ESC [ n C + * Effect: moves cursor forward n chars + * + * CUB (CUrsor Backward) + * Sequence: ESC [ n D + * Effect: moves cursor backward n chars + * + * The following is used to get the terminal width if getting + * the width with the TIOCGWINSZ ioctl fails + * + * DSR (Device Status Report) + * Sequence: ESC [ 6 n + * Effect: reports the current cusor position as ESC [ n ; m R + * where n is the row and m is the column + * + * When multi line mode is enabled, we also use an additional escape + * sequence. However multi line editing is disabled by default. + * + * CUU (Cursor Up) + * Sequence: ESC [ n A + * Effect: moves cursor up of n chars. + * + * CUD (Cursor Down) + * Sequence: ESC [ n B + * Effect: moves cursor down of n chars. + * + * When linenoiseClearScreen() is called, two additional escape sequences + * are used in order to clear the screen and position the cursor at home + * position. + * + * CUP (Cursor position) + * Sequence: ESC [ H + * Effect: moves the cursor to upper left corner + * + * ED (Erase display) + * Sequence: ESC [ 2 J + * Effect: clear the whole screen + * + */ + +#include <termios.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <stdlib.h> +#include <ctype.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/ioctl.h> +#include <unistd.h> +#include "linenoise.h" + +#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 +#define LINENOISE_MAX_LINE 4096 +static char *unsupported_term[] = {"dumb","cons25","emacs",NULL}; +static linenoiseCompletionCallback *completionCallback = NULL; +static linenoiseHintsCallback *hintsCallback = NULL; +static linenoiseFreeHintsCallback *freeHintsCallback = NULL; + +static struct termios orig_termios; /* In order to restore at exit.*/ +static int rawmode = 0; /* For atexit() function to check if restore is needed*/ +static int mlmode = 0; /* Multi line mode. Default is single line. */ +static int atexit_registered = 0; /* Register atexit just 1 time. */ +static int history_max_len = LINENOISE_DEFAULT_HISTORY_MAX_LEN; +static int history_len = 0; +static char **history = NULL; + +/* The linenoiseState structure represents the state during line editing. + * We pass this state to functions implementing specific editing + * functionalities. */ +struct linenoiseState { + int ifd; /* Terminal stdin file descriptor. */ + int ofd; /* Terminal stdout file descriptor. */ + char *buf; /* Edited line buffer. */ + size_t buflen; /* Edited line buffer size. */ + const char *prompt; /* Prompt to display. */ + size_t plen; /* Prompt length. */ + size_t pos; /* Current cursor position. */ + size_t oldpos; /* Previous refresh cursor position. */ + size_t len; /* Current edited line length. */ + size_t cols; /* Number of columns in terminal. */ + size_t maxrows; /* Maximum num of rows used so far (multiline mode) */ + int history_index; /* The history index we are currently editing. */ +}; + +enum KEY_ACTION{ + KEY_NULL = 0, /* NULL */ + CTRL_A = 1, /* Ctrl+a */ + CTRL_B = 2, /* Ctrl-b */ + CTRL_C = 3, /* Ctrl-c */ + CTRL_D = 4, /* Ctrl-d */ + CTRL_E = 5, /* Ctrl-e */ + CTRL_F = 6, /* Ctrl-f */ + CTRL_H = 8, /* Ctrl-h */ + TAB = 9, /* Tab */ + CTRL_K = 11, /* Ctrl+k */ + CTRL_L = 12, /* Ctrl+l */ + ENTER = 13, /* Enter */ + CTRL_N = 14, /* Ctrl-n */ + CTRL_P = 16, /* Ctrl-p */ + CTRL_T = 20, /* Ctrl-t */ + CTRL_U = 21, /* Ctrl+u */ + CTRL_W = 23, /* Ctrl+w */ + ESC = 27, /* Escape */ + BACKSPACE = 127 /* Backspace */ +}; + +static void linenoiseAtExit(void); +int linenoiseHistoryAdd(const char *line); +static void refreshLine(struct linenoiseState *l); + +/* Debugging macro. */ +#if 0 +FILE *lndebug_fp = NULL; +#define lndebug(...) \ + do { \ + if (lndebug_fp == NULL) { \ + lndebug_fp = fopen("/tmp/lndebug.txt","a"); \ + fprintf(lndebug_fp, \ + "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \ + (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \ + (int)l->maxrows,old_rows); \ + } \ + fprintf(lndebug_fp, ", " __VA_ARGS__); \ + fflush(lndebug_fp); \ + } while (0) +#else +#define lndebug(fmt, ...) +#endif + +/* ======================= Low level terminal handling ====================== */ + +/* Set if to use or not the multi line mode. */ +void linenoiseSetMultiLine(int ml) { + mlmode = ml; +} + +/* Return true if the terminal name is in the list of terminals we know are + * not able to understand basic escape sequences. */ +static int isUnsupportedTerm(void) { + char *term = getenv("TERM"); + int j; + + if (term == NULL) return 0; + for (j = 0; unsupported_term[j]; j++) + if (!strcasecmp(term,unsupported_term[j])) return 1; + return 0; +} + +/* Raw mode: 1960 magic shit. */ +static int enableRawMode(int fd) { + struct termios raw; + + if (!isatty(STDIN_FILENO)) goto fatal; + if (!atexit_registered) { + atexit(linenoiseAtExit); + atexit_registered = 1; + } + if (tcgetattr(fd,&orig_termios) == -1) goto fatal; + + raw = orig_termios; /* modify the original mode */ + /* input modes: no break, no CR to NL, no parity check, no strip char, + * no start/stop output control. */ + raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON); + /* output modes - disable post processing */ + raw.c_oflag &= ~(OPOST); + /* control modes - set 8 bit chars */ + raw.c_cflag |= (CS8); + /* local modes - choing off, canonical off, no extended functions, + * no signal chars (^Z,^C) */ + raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); + /* control chars - set return condition: min number of bytes and timer. + * We want read to return every single byte, without timeout. */ + raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ + + /* put terminal in raw mode after flushing */ + if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal; + rawmode = 1; + return 0; + +fatal: + errno = ENOTTY; + return -1; +} + +static void disableRawMode(int fd) { + /* Don't even check the return value as it's too late. */ + if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1) + rawmode = 0; +} + +/* Use the ESC [6n escape sequence to query the horizontal cursor position + * and return it. On error -1 is returned, on success the position of the + * cursor. */ +static int getCursorPosition(int ifd, int ofd) { + char buf[32]; + int cols, rows; + unsigned int i = 0; + + /* Report cursor location */ + if (write(ofd, "\x1b[6n", 4) != 4) return -1; + + /* Read the response: ESC [ rows ; cols R */ + while (i < sizeof(buf)-1) { + if (read(ifd,buf+i,1) != 1) break; + if (buf[i] == 'R') break; + i++; + } + buf[i] = '\0'; + + /* Parse it. */ + if (buf[0] != ESC || buf[1] != '[') return -1; + if (sscanf(buf+2,"%d;%d",&rows,&cols) != 2) return -1; + return cols; +} + +/* Try to get the number of columns in the current terminal, or assume 80 + * if it fails. */ +static int getColumns(int ifd, int ofd) { + struct winsize ws; + + if (ioctl(1, TIOCGWINSZ, &ws) == -1 || ws.ws_col == 0) { + /* ioctl() failed. Try to query the terminal itself. */ + int start, cols; + + /* Get the initial position so we can restore it later. */ + start = getCursorPosition(ifd,ofd); + if (start == -1) goto failed; + + /* Go to right margin and get position. */ + if (write(ofd,"\x1b[999C",6) != 6) goto failed; + cols = getCursorPosition(ifd,ofd); + if (cols == -1) goto failed; + + /* Restore position. */ + if (cols > start) { + char seq[32]; + snprintf(seq,32,"\x1b[%dD",cols-start); + if (write(ofd,seq,strlen(seq)) == -1) { + /* Can't recover... */ + } + } + return cols; + } else { + return ws.ws_col; + } + +failed: + return 80; +} + +/* Clear the screen. Used to handle ctrl+l */ +void linenoiseClearScreen(void) { + if (write(STDOUT_FILENO,"\x1b[H\x1b[2J",7) <= 0) { + /* nothing to do, just to avoid warning. */ + } +} + +/* Beep, used for completion when there is nothing to complete or when all + * the choices were already shown. */ +static void linenoiseBeep(void) { + fprintf(stderr, "\x7"); + fflush(stderr); +} + +/* ============================== Completion ================================ */ + +/* Free a list of completion option populated by linenoiseAddCompletion(). */ +static void freeCompletions(linenoiseCompletions *lc) { + size_t i; + for (i = 0; i < lc->len; i++) + free(lc->cvec[i]); + if (lc->cvec != NULL) + free(lc->cvec); +} + +/* This is an helper function for linenoiseEdit() and is called when the + * user types the <tab> key in order to complete the string currently in the + * input. + * + * The state of the editing is encapsulated into the pointed linenoiseState + * structure as described in the structure definition. */ +static int completeLine(struct linenoiseState *ls) { + linenoiseCompletions lc = { 0, NULL }; + int nread, nwritten; + char c = 0; + + completionCallback(ls->buf,&lc); + if (lc.len == 0) { + linenoiseBeep(); + } else { + size_t stop = 0, i = 0; + + while(!stop) { + /* Show completion or original buffer */ + if (i < lc.len) { + struct linenoiseState saved = *ls; + + ls->len = ls->pos = strlen(lc.cvec[i]); + ls->buf = lc.cvec[i]; + refreshLine(ls); + ls->len = saved.len; + ls->pos = saved.pos; + ls->buf = saved.buf; + } else { + refreshLine(ls); + } + + nread = read(ls->ifd,&c,1); + if (nread <= 0) { + freeCompletions(&lc); + return -1; + } + + switch(c) { + case 9: /* tab */ + i = (i+1) % (lc.len+1); + if (i == lc.len) linenoiseBeep(); + break; + case 27: /* escape */ + /* Re-show original buffer */ + if (i < lc.len) refreshLine(ls); + stop = 1; + break; + default: + /* Update buffer and return */ + if (i < lc.len) { + nwritten = snprintf(ls->buf,ls->buflen,"%s",lc.cvec[i]); + ls->len = ls->pos = nwritten; + } + stop = 1; + break; + } + } + } + + freeCompletions(&lc); + return c; /* Return last read character */ +} + +/* Register a callback function to be called for tab-completion. */ +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { + completionCallback = fn; +} + +/* Register a hits function to be called to show hits to the user at the + * right of the prompt. */ +void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) { + hintsCallback = fn; +} + +/* Register a function to free the hints returned by the hints callback + * registered with linenoiseSetHintsCallback(). */ +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) { + freeHintsCallback = fn; +} + +/* This function is used by the callback function registered by the user + * in order to add completion options given the input string when the + * user typed <tab>. See the example.c source code for a very easy to + * understand example. */ +void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { + size_t len = strlen(str); + char *copy, **cvec; + + copy = malloc(len+1); + if (copy == NULL) return; + memcpy(copy,str,len+1); + cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); + if (cvec == NULL) { + free(copy); + return; + } + lc->cvec = cvec; + lc->cvec[lc->len++] = copy; +} + +/* =========================== Line editing ================================= */ + +/* We define a very simple "append buffer" structure, that is an heap + * allocated string where we can append to. This is useful in order to + * write all the escape sequences in a buffer and flush them to the standard + * output in a single call, to avoid flickering effects. */ +struct abuf { + char *b; + int len; +}; + +static void abInit(struct abuf *ab) { + ab->b = NULL; + ab->len = 0; +} + +static void abAppend(struct abuf *ab, const char *s, int len) { + char *new = realloc(ab->b,ab->len+len); + + if (new == NULL) return; + memcpy(new+ab->len,s,len); + ab->b = new; + ab->len += len; +} + +static void abFree(struct abuf *ab) { + free(ab->b); +} + +/* Helper of refreshSingleLine() and refreshMultiLine() to show hints + * to the right of the prompt. */ +void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { + char seq[64]; + if (hintsCallback && plen+l->len < l->cols) { + int color = -1, bold = 0; + char *hint = hintsCallback(l->buf,&color,&bold); + if (hint) { + int hintlen = strlen(hint); + int hintmaxlen = l->cols-(plen+l->len); + if (hintlen > hintmaxlen) hintlen = hintmaxlen; + if (bold == 1 && color == -1) color = 37; + if (color != -1 || bold != 0) + snprintf(seq,64,"\033[%d;%d;49m",bold,color); + abAppend(ab,seq,strlen(seq)); + abAppend(ab,hint,hintlen); + if (color != -1 || bold != 0) + abAppend(ab,"\033[0m",4); + /* Call the function to free the hint returned. */ + if (freeHintsCallback) freeHintsCallback(hint); + } + } +} + +/* Single line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshSingleLine(struct linenoiseState *l) { + char seq[64]; + size_t plen = strlen(l->prompt); + int fd = l->ofd; + char *buf = l->buf; + size_t len = l->len; + size_t pos = l->pos; + struct abuf ab; + + while((plen+pos) >= l->cols) { + buf++; + len--; + pos--; + } + while (plen+len > l->cols) { + len--; + } + + abInit(&ab); + /* Cursor to left edge */ + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + abAppend(&ab,buf,len); + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + /* Erase to right */ + snprintf(seq,64,"\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + /* Move cursor to original position. */ + snprintf(seq,64,"\r\x1b[%dC", (int)(pos+plen)); + abAppend(&ab,seq,strlen(seq)); + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Multi line low level line refresh. + * + * Rewrite the currently edited line accordingly to the buffer content, + * cursor position, and number of columns of the terminal. */ +static void refreshMultiLine(struct linenoiseState *l) { + char seq[64]; + int plen = strlen(l->prompt); + int rows = (plen+l->len+l->cols-1)/l->cols; /* rows used by current buf. */ + int rpos = (plen+l->oldpos+l->cols)/l->cols; /* cursor relative row. */ + int rpos2; /* rpos after refresh. */ + int col; /* colum position, zero-based. */ + int old_rows = l->maxrows; + int fd = l->ofd, j; + struct abuf ab; + + /* Update maxrows if needed. */ + if (rows > (int)l->maxrows) l->maxrows = rows; + + /* First step: clear all the lines used before. To do so start by + * going to the last row. */ + abInit(&ab); + if (old_rows-rpos > 0) { + lndebug("go down %d", old_rows-rpos); + snprintf(seq,64,"\x1b[%dB", old_rows-rpos); + abAppend(&ab,seq,strlen(seq)); + } + + /* Now for every row clear it, go up. */ + for (j = 0; j < old_rows-1; j++) { + lndebug("clear+up"); + snprintf(seq,64,"\r\x1b[0K\x1b[1A"); + abAppend(&ab,seq,strlen(seq)); + } + + /* Clean the top line. */ + lndebug("clear"); + snprintf(seq,64,"\r\x1b[0K"); + abAppend(&ab,seq,strlen(seq)); + + /* Write the prompt and the current buffer content */ + abAppend(&ab,l->prompt,strlen(l->prompt)); + abAppend(&ab,l->buf,l->len); + + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + + /* If we are at the very end of the screen with our prompt, we need to + * emit a newline and move the prompt to the first column. */ + if (l->pos && + l->pos == l->len && + (l->pos+plen) % l->cols == 0) + { + lndebug("<newline>"); + abAppend(&ab,"\n",1); + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + rows++; + if (rows > (int)l->maxrows) l->maxrows = rows; + } + + /* Move cursor to right position. */ + rpos2 = (plen+l->pos+l->cols)/l->cols; /* current cursor relative row. */ + lndebug("rpos2 %d", rpos2); + + /* Go up till we reach the expected positon. */ + if (rows-rpos2 > 0) { + lndebug("go-up %d", rows-rpos2); + snprintf(seq,64,"\x1b[%dA", rows-rpos2); + abAppend(&ab,seq,strlen(seq)); + } + + /* Set column. */ + col = (plen+(int)l->pos) % (int)l->cols; + lndebug("set col %d", 1+col); + if (col) + snprintf(seq,64,"\r\x1b[%dC", col); + else + snprintf(seq,64,"\r"); + abAppend(&ab,seq,strlen(seq)); + + lndebug("\n"); + l->oldpos = l->pos; + + if (write(fd,ab.b,ab.len) == -1) {} /* Can't recover from write error. */ + abFree(&ab); +} + +/* Calls the two low level functions refreshSingleLine() or + * refreshMultiLine() according to the selected mode. */ +static void refreshLine(struct linenoiseState *l) { + if (mlmode) + refreshMultiLine(l); + else + refreshSingleLine(l); +} + +/* Insert the character 'c' at cursor current position. + * + * On error writing to the terminal -1 is returned, otherwise 0. */ +int linenoiseEditInsert(struct linenoiseState *l, char c) { + if (l->len < l->buflen) { + if (l->len == l->pos) { + l->buf[l->pos] = c; + l->pos++; + l->len++; + l->buf[l->len] = '\0'; + if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { + /* Avoid a full update of the line in the + * trivial case. */ + if (write(l->ofd,&c,1) == -1) return -1; + } else { + refreshLine(l); + } + } else { + memmove(l->buf+l->pos+1,l->buf+l->pos,l->len-l->pos); + l->buf[l->pos] = c; + l->len++; + l->pos++; + l->buf[l->len] = '\0'; + refreshLine(l); + } + } + return 0; +} + +/* Move cursor on the left. */ +void linenoiseEditMoveLeft(struct linenoiseState *l) { + if (l->pos > 0) { + l->pos--; + refreshLine(l); + } +} + +/* Move cursor on the right. */ +void linenoiseEditMoveRight(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos++; + refreshLine(l); + } +} + +/* Move cursor to the start of the line. */ +void linenoiseEditMoveHome(struct linenoiseState *l) { + if (l->pos != 0) { + l->pos = 0; + refreshLine(l); + } +} + +/* Move cursor to the end of the line. */ +void linenoiseEditMoveEnd(struct linenoiseState *l) { + if (l->pos != l->len) { + l->pos = l->len; + refreshLine(l); + } +} + +/* Substitute the currently edited line with the next or previous history + * entry as specified by 'dir'. */ +#define LINENOISE_HISTORY_NEXT 0 +#define LINENOISE_HISTORY_PREV 1 +void linenoiseEditHistoryNext(struct linenoiseState *l, int dir) { + if (history_len > 1) { + /* Update the current history entry before to + * overwrite it with the next one. */ + free(history[history_len - 1 - l->history_index]); + history[history_len - 1 - l->history_index] = strdup(l->buf); + /* Show the new entry */ + l->history_index += (dir == LINENOISE_HISTORY_PREV) ? 1 : -1; + if (l->history_index < 0) { + l->history_index = 0; + return; + } else if (l->history_index >= history_len) { + l->history_index = history_len-1; + return; + } + strncpy(l->buf,history[history_len - 1 - l->history_index],l->buflen); + l->buf[l->buflen-1] = '\0'; + l->len = l->pos = strlen(l->buf); + refreshLine(l); + } +} + +/* Delete the character at the right of the cursor without altering the cursor + * position. Basically this is what happens with the "Delete" keyboard key. */ +void linenoiseEditDelete(struct linenoiseState *l) { + if (l->len > 0 && l->pos < l->len) { + memmove(l->buf+l->pos,l->buf+l->pos+1,l->len-l->pos-1); + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Backspace implementation. */ +void linenoiseEditBackspace(struct linenoiseState *l) { + if (l->pos > 0 && l->len > 0) { + memmove(l->buf+l->pos-1,l->buf+l->pos,l->len-l->pos); + l->pos--; + l->len--; + l->buf[l->len] = '\0'; + refreshLine(l); + } +} + +/* Delete the previosu word, maintaining the cursor at the start of the + * current word. */ +void linenoiseEditDeletePrevWord(struct linenoiseState *l) { + size_t old_pos = l->pos; + size_t diff; + + while (l->pos > 0 && l->buf[l->pos-1] == ' ') + l->pos--; + while (l->pos > 0 && l->buf[l->pos-1] != ' ') + l->pos--; + diff = old_pos - l->pos; + memmove(l->buf+l->pos,l->buf+old_pos,l->len-old_pos+1); + l->len -= diff; + refreshLine(l); +} + +/* This function is the core of the line editing capability of linenoise. + * It expects 'fd' to be already in "raw mode" so that every key pressed + * will be returned ASAP to read(). + * + * The resulting string is put into 'buf' when the user type enter, or + * when ctrl+d is typed. + * + * The function returns the length of the current buffer. */ +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +{ + struct linenoiseState l; + + /* Populate the linenoise state that we pass to functions implementing + * specific editing functionalities. */ + l.ifd = stdin_fd; + l.ofd = stdout_fd; + l.buf = buf; + l.buflen = buflen; + l.prompt = prompt; + l.plen = strlen(prompt); + l.oldpos = l.pos = 0; + l.len = 0; + l.cols = getColumns(stdin_fd, stdout_fd); + l.maxrows = 0; + l.history_index = 0; + + /* Buffer starts empty. */ + l.buf[0] = '\0'; + l.buflen--; /* Make sure there is always space for the nulterm */ + + /* The latest history entry is always our current buffer, that + * initially is just an empty string. */ + linenoiseHistoryAdd(""); + + if (write(l.ofd,prompt,l.plen) == -1) return -1; + while(1) { + char c; + int nread; + char seq[3]; + + nread = read(l.ifd,&c,1); + if (nread <= 0) return l.len; + + /* Only autocomplete when the callback is set. It returns < 0 when + * there was an error reading from fd. Otherwise it will return the + * character that should be handled next. */ + if (c == 9 && completionCallback != NULL) { + c = completeLine(&l); + /* Return on errors */ + if (c < 0) return l.len; + /* Read next character when 0 */ + if (c == 0) continue; + } + + switch(c) { + case ENTER: /* enter */ + history_len--; + free(history[history_len]); + if (mlmode) linenoiseEditMoveEnd(&l); + if (hintsCallback) { + /* Force a refresh without hints to leave the previous + * line as the user typed it after a newline. */ + linenoiseHintsCallback *hc = hintsCallback; + hintsCallback = NULL; + refreshLine(&l); + hintsCallback = hc; + } + return (int)l.len; + case CTRL_C: /* ctrl-c */ + errno = EAGAIN; + return -1; + case BACKSPACE: /* backspace */ + case 8: /* ctrl-h */ + linenoiseEditBackspace(&l); + break; + case CTRL_D: /* ctrl-d, remove char at right of cursor, or if the + line is empty, act as end-of-file. */ + if (l.len > 0) { + linenoiseEditDelete(&l); + } else { + history_len--; + free(history[history_len]); + return -1; + } + break; + case CTRL_T: /* ctrl-t, swaps current character with previous. */ + if (l.pos > 0 && l.pos < l.len) { + int aux = buf[l.pos-1]; + buf[l.pos-1] = buf[l.pos]; + buf[l.pos] = aux; + if (l.pos != l.len-1) l.pos++; + refreshLine(&l); + } + break; + case CTRL_B: /* ctrl-b */ + linenoiseEditMoveLeft(&l); + break; + case CTRL_F: /* ctrl-f */ + linenoiseEditMoveRight(&l); + break; + case CTRL_P: /* ctrl-p */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case CTRL_N: /* ctrl-n */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case ESC: /* escape sequence */ + /* Read the next two bytes representing the escape sequence. + * Use two calls to handle slow terminals returning the two + * chars at different times. */ + if (read(l.ifd,seq,1) == -1) break; + if (read(l.ifd,seq+1,1) == -1) break; + + /* ESC [ sequences. */ + if (seq[0] == '[') { + if (seq[1] >= '0' && seq[1] <= '9') { + /* Extended escape, read additional byte. */ + if (read(l.ifd,seq+2,1) == -1) break; + if (seq[2] == '~') { + switch(seq[1]) { + case '3': /* Delete key. */ + linenoiseEditDelete(&l); + break; + } + } + } else { + switch(seq[1]) { + case 'A': /* Up */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_PREV); + break; + case 'B': /* Down */ + linenoiseEditHistoryNext(&l, LINENOISE_HISTORY_NEXT); + break; + case 'C': /* Right */ + linenoiseEditMoveRight(&l); + break; + case 'D': /* Left */ + linenoiseEditMoveLeft(&l); + break; + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; + } + } + } + + /* ESC O sequences. */ + else if (seq[0] == 'O') { + switch(seq[1]) { + case 'H': /* Home */ + linenoiseEditMoveHome(&l); + break; + case 'F': /* End*/ + linenoiseEditMoveEnd(&l); + break; + } + } + break; + default: + if (linenoiseEditInsert(&l,c)) return -1; + break; + case CTRL_U: /* Ctrl+u, delete the whole line. */ + buf[0] = '\0'; + l.pos = l.len = 0; + refreshLine(&l); + break; + case CTRL_K: /* Ctrl+k, delete from current to end of line. */ + buf[l.pos] = '\0'; + l.len = l.pos; + refreshLine(&l); + break; + case CTRL_A: /* Ctrl+a, go to the start of the line */ + linenoiseEditMoveHome(&l); + break; + case CTRL_E: /* ctrl+e, go to the end of the line */ + linenoiseEditMoveEnd(&l); + break; + case CTRL_L: /* ctrl+l, clear screen */ + linenoiseClearScreen(); + refreshLine(&l); + break; + case CTRL_W: /* ctrl+w, delete previous word */ + linenoiseEditDeletePrevWord(&l); + break; + } + } + return l.len; +} + +/* This special mode is used by linenoise in order to print scan codes + * on screen for debugging / development purposes. It is implemented + * by the linenoise_example program using the --keycodes option. */ +void linenoisePrintKeyCodes(void) { + char quit[4]; + + printf("Linenoise key codes debugging mode.\n" + "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); + if (enableRawMode(STDIN_FILENO) == -1) return; + memset(quit,' ',4); + while(1) { + char c; + int nread; + + nread = read(STDIN_FILENO,&c,1); + if (nread <= 0) continue; + memmove(quit,quit+1,sizeof(quit)-1); /* shift string to left. */ + quit[sizeof(quit)-1] = c; /* Insert current char on the right. */ + if (memcmp(quit,"quit",sizeof(quit)) == 0) break; + + printf("'%c' %02x (%d) (type quit to exit)\n", + isprint(c) ? c : '?', (int)c, (int)c); + printf("\r"); /* Go left edge manually, we are in raw mode. */ + fflush(stdout); + } + disableRawMode(STDIN_FILENO); +} + +/* This function calls the line editing function linenoiseEdit() using + * the STDIN file descriptor set in raw mode. */ +static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { + int count; + + if (buflen == 0) { + errno = EINVAL; + return -1; + } + + if (enableRawMode(STDIN_FILENO) == -1) return -1; + count = linenoiseEdit(STDIN_FILENO, STDOUT_FILENO, buf, buflen, prompt); + disableRawMode(STDIN_FILENO); + printf("\n"); + return count; +} + +/* This function is called when linenoise() is called with the standard + * input file descriptor not attached to a TTY. So for example when the + * program using linenoise is called in pipe or with a file redirected + * to its standard input. In this case, we want to be able to return the + * line regardless of its length (by default we are limited to 4k). */ +static char *linenoiseNoTTY(void) { + char *line = NULL; + size_t len = 0, maxlen = 0; + + while(1) { + if (len == maxlen) { + if (maxlen == 0) maxlen = 16; + maxlen *= 2; + char *oldval = line; + line = realloc(line,maxlen); + if (line == NULL) { + if (oldval) free(oldval); + return NULL; + } + } + int c = fgetc(stdin); + if (c == EOF || c == '\n') { + if (c == EOF && len == 0) { + free(line); + return NULL; + } else { + line[len] = '\0'; + return line; + } + } else { + line[len] = c; + len++; + } + } +} + +/* The high level function that is the main API of the linenoise library. + * This function checks if the terminal has basic capabilities, just checking + * for a blacklist of stupid terminals, and later either calls the line + * editing function or uses dummy fgets() so that you will be able to type + * something even in the most desperate of the conditions. */ +char *linenoise(const char *prompt) { + char buf[LINENOISE_MAX_LINE]; + int count; + + if (!isatty(STDIN_FILENO)) { + /* Not a tty: read from file / pipe. In this mode we don't want any + * limit to the line size, so we call a function to handle that. */ + return linenoiseNoTTY(); + } else if (isUnsupportedTerm()) { + size_t len; + + printf("%s",prompt); + fflush(stdout); + if (fgets(buf,LINENOISE_MAX_LINE,stdin) == NULL) return NULL; + len = strlen(buf); + while(len && (buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + buf[len] = '\0'; + } + return strdup(buf); + } else { + count = linenoiseRaw(buf,LINENOISE_MAX_LINE,prompt); + if (count == -1) return NULL; + return strdup(buf); + } +} + +/* This is just a wrapper the user may want to call in order to make sure + * the linenoise returned buffer is freed with the same allocator it was + * created with. Useful when the main program is using an alternative + * allocator. */ +void linenoiseFree(void *ptr) { + free(ptr); +} + +/* ================================ History ================================= */ + +/* Free the history, but does not reset it. Only used when we have to + * exit() to avoid memory leaks are reported by valgrind & co. */ +static void freeHistory(void) { + if (history) { + int j; + + for (j = 0; j < history_len; j++) + free(history[j]); + free(history); + } +} + +/* At exit we'll try to fix the terminal to the initial conditions. */ +static void linenoiseAtExit(void) { + disableRawMode(STDIN_FILENO); + freeHistory(); +} + +/* This is the API call to add a new entry in the linenoise history. + * It uses a fixed array of char pointers that are shifted (memmoved) + * when the history max length is reached in order to remove the older + * entry and make room for the new one, so it is not exactly suitable for huge + * histories, but will work well for a few hundred of entries. + * + * Using a circular buffer is smarter, but a bit more complex to handle. */ +int linenoiseHistoryAdd(const char *line) { + char *linecopy; + + if (history_max_len == 0) return 0; + + /* Initialization on first call. */ + if (history == NULL) { + history = malloc(sizeof(char*)*history_max_len); + if (history == NULL) return 0; + memset(history,0,(sizeof(char*)*history_max_len)); + } + + /* Don't add duplicated lines. */ + if (history_len && !strcmp(history[history_len-1], line)) return 0; + + /* Add an heap allocated copy of the line in the history. + * If we reached the max length, remove the older line. */ + linecopy = strdup(line); + if (!linecopy) return 0; + if (history_len == history_max_len) { + free(history[0]); + memmove(history,history+1,sizeof(char*)*(history_max_len-1)); + history_len--; + } + history[history_len] = linecopy; + history_len++; + return 1; +} + +/* Set the maximum length for the history. This function can be called even + * if there is already some history, the function will make sure to retain + * just the latest 'len' elements if the new history length value is smaller + * than the amount of items already inside the history. */ +int linenoiseHistorySetMaxLen(int len) { + char **new; + + if (len < 1) return 0; + if (history) { + int tocopy = history_len; + + new = malloc(sizeof(char*)*len); + if (new == NULL) return 0; + + /* If we can't copy everything, free the elements we'll not use. */ + if (len < tocopy) { + int j; + + for (j = 0; j < tocopy-len; j++) free(history[j]); + tocopy = len; + } + memset(new,0,sizeof(char*)*len); + memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy); + free(history); + history = new; + } + history_max_len = len; + if (history_len > history_max_len) + history_len = history_max_len; + return 1; +} + +/* Save the history in the specified file. On success 0 is returned + * otherwise -1 is returned. */ +int linenoiseHistorySave(const char *filename) { + mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO); + FILE *fp; + int j; + + fp = fopen(filename,"w"); + umask(old_umask); + if (fp == NULL) return -1; + chmod(filename,S_IRUSR|S_IWUSR); + for (j = 0; j < history_len; j++) + fprintf(fp,"%s\n",history[j]); + fclose(fp); + return 0; +} + +/* Load the history from the specified file. If the file does not exist + * zero is returned and no operation is performed. + * + * If the file exists and the operation succeeded 0 is returned, otherwise + * on error -1 is returned. */ +int linenoiseHistoryLoad(const char *filename) { + FILE *fp = fopen(filename,"r"); + char buf[LINENOISE_MAX_LINE]; + + if (fp == NULL) return -1; + + while (fgets(buf,LINENOISE_MAX_LINE,fp) != NULL) { + char *p; + + p = strchr(buf,'\r'); + if (!p) p = strchr(buf,'\n'); + if (p) *p = '\0'; + linenoiseHistoryAdd(buf); + } + fclose(fp); + return 0; +} diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h new file mode 100644 index 000000000000..ed20232c576e --- /dev/null +++ b/src/linenoise/linenoise.h @@ -0,0 +1,73 @@ +/* linenoise.h -- VERSION 1.0 + * + * Guerrilla line editing library against the idea that a line editing lib + * needs to be 20,000 lines of C code. + * + * See linenoise.c for more information. + * + * ------------------------------------------------------------------------ + * + * Copyright (c) 2010-2014, Salvatore Sanfilippo <antirez at gmail dot com> + * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __LINENOISE_H +#define __LINENOISE_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct linenoiseCompletions { + size_t len; + char **cvec; +} linenoiseCompletions; + +typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); +typedef void(linenoiseFreeHintsCallback)(void *); +void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); +void linenoiseSetHintsCallback(linenoiseHintsCallback *); +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); +void linenoiseAddCompletion(linenoiseCompletions *, const char *); + +char *linenoise(const char *prompt); +void linenoiseFree(void *ptr); +int linenoiseHistoryAdd(const char *line); +int linenoiseHistorySetMaxLen(int len); +int linenoiseHistorySave(const char *filename); +int linenoiseHistoryLoad(const char *filename); +void linenoiseClearScreen(void); +void linenoiseSetMultiLine(int ml); +void linenoisePrintKeyCodes(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __LINENOISE_H */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 065447684aaf..7167e96f11d3 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -15,6 +15,7 @@ #include "shared.hh" using namespace nix; +using namespace std::string_literals; extern char * * environ; @@ -325,7 +326,7 @@ int main(int argc, char ** argv) if (packages) { instArgs.push_back("--expr"); std::ostringstream joined; - joined << "with import <nixpkgs> { }; runCommand \"shell\" { buildInputs = [ "; + joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; for (const auto & i : exprs) joined << '(' << i << ") "; joined << "]; } \"\""; @@ -408,8 +409,20 @@ int main(int argc, char ** argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = settings.buildCores; + auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", "")); + + bool keepTmp = false; + int fileNr = 0; + for (auto & var : drv.env) - env[var.first] = var.second; + if (passAsFile.count(var.first)) { + keepTmp = true; + string fn = ".attr-" + std::to_string(fileNr++); + Path p = (Path) tmpDir + "/" + fn; + writeFile(p, var.second); + env[var.first + "Path"] = p; + } else + env[var.first] = var.second; restoreAffinity(); @@ -419,14 +432,14 @@ int main(int argc, char ** argv) // the current $PATH directories. auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( - "rm -rf '%1%'; " + (keepTmp ? "" : "rm -rf '%1%'; "s) + "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" "set +e; " - "[ -n \"$PS1\" ] && PS1=\"\\n\\[\\033[1;32m\\][nix-shell:\\w]$\\[\\033[0m\\] \"; " + R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "unset NIX_INDENT_MAKE; " diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 361627823126..f2742bc3bbda 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -103,19 +103,15 @@ static void update(const StringSet & channelNames) auto unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - try { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " - "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); - unpacked = true; - } catch (ExecError & e) { - } + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import <nix/unpack-channel.nix> " + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); + unpacked = true; } if (!unpacked) { // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it // Check if the channel advertises a binary cache. DownloadRequest request(url + "/binary-cache-url"); - request.showProgress = DownloadRequest::no; try { auto dlRes = dl->download(request); extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; @@ -172,9 +168,7 @@ int main(int argc, char ** argv) setenv("NIX_DOWNLOAD_CACHE", channelCache.c_str(), 1); // Figure out the name of the `.nix-channels' file to use - auto home = getEnv("HOME"); - if (home.empty()) - throw Error("$HOME not set"); + auto home = getHome(); channelsList = home + "/.nix-channels"; nixDefExpr = home + "/.nix-defexpr"; diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index ed43bffbc8c8..dc324abcb3ba 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -58,6 +58,6 @@ int main(int argc, char ** argv) PathSet closure; from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, closure, useSubstitutes); + copyPaths(from, to, closure, useSubstitutes, true); }); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8786e2561b9c..44127635ded8 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -82,12 +82,7 @@ class TunnelLogger : public Logger defaultLogger->log(lvl, fs); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - log(lvl, fs); - } - - void stopActivity(Activity & activity) override + void event(const Event & ev) override { } }; @@ -436,31 +431,70 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe } case wopSetOptions: { - from >> settings.keepFailed; - from >> settings.keepGoing; - settings.set("build-fallback", readInt(from) ? "true" : "false"); + settings.keepFailed = readInt(from); + settings.keepGoing = readInt(from); + settings.tryFallback = readInt(from); verbosity = (Verbosity) readInt(from); - settings.set("build-max-jobs", std::to_string(readInt(from))); - settings.set("build-max-silent-time", std::to_string(readInt(from))); + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); settings.useBuildHook = readInt(from) != 0; settings.verboseBuild = lvlError == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace - settings.set("build-cores", std::to_string(readInt(from))); - settings.set("build-use-substitutes", readInt(from) ? "true" : "false"); + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from); + + StringMap overrides; if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { unsigned int n = readInt(from); for (unsigned int i = 0; i < n; i++) { string name = readString(from); string value = readString(from); - if (name == "build-timeout" || name == "use-ssh-substituter") + overrides.emplace(name, value); + } + } + + startWork(); + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting<Strings> & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString<Strings>(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == settings.connectTimeout.name) settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; else - settings.set(trusted ? name : "untrusted-" + name, value); + debug("ignoring untrusted setting '%s'", name); + } catch (UsageError & e) { + warn(e.what()); } } - settings.update(); - startWork(); + stopWork(); break; } @@ -582,6 +616,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe from >> info.ca >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; + if (!trusted) + info.ultimate = false; TeeSink tee(from); parseDump(tee, tee.source); @@ -878,8 +914,8 @@ static void daemonLoop(char * * argv) struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.get("trusted-users", Strings({"root"})); - Strings allowedUsers = settings.get("allowed-users", Strings({"*"})); + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = true; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 908c09bc8c8a..464bcee4a848 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -192,17 +192,9 @@ static void loadDerivations(EvalState & state, Path nixExprPath, } -static Path getHomeDir() -{ - Path homeDir(getEnv("HOME", "")); - if (homeDir == "") throw Error("HOME environment variable not set"); - return homeDir; -} - - static Path getDefNixExprPath() { - return getHomeDir() + "/.nix-defexpr"; + return getHome() + "/.nix-defexpr"; } @@ -997,7 +989,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); + //Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); if (globals.prebuiltOnly && validPaths.find(i.queryOutPath()) == validPaths.end() && @@ -1188,7 +1180,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError(format("exactly one argument expected")); Path profile = absPath(opArgs.front()); - Path profileLink = getHomeDir() + "/.nix-profile"; + Path profileLink = getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1413,7 +1405,7 @@ int main(int argc, char * * argv) globals.profile = getEnv("NIX_PROFILE", ""); if (globals.profile == "") { - Path profileLink = getHomeDir() + "/.nix-profile"; + Path profileLink = getHome() + "/.nix-profile"; globals.profile = pathExists(profileLink) ? absPath(readLink(profileLink), dirOf(profileLink)) : canonPath(settings.nixStateDir + "/profiles/default"); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c1b0b0ea092d..25f0b1bd692e 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -19,7 +19,7 @@ using namespace nix; static Expr * parseStdin(EvalState & state) { - Activity act(*logger, lvlTalkative, format("parsing standard input")); + //Activity act(*logger, lvlTalkative, format("parsing standard input")); return state.parseExprFromString(drainFD(0), absPath(".")); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3dc167191c83..950222812e2b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -145,7 +145,7 @@ static void opRealise(Strings opFlags, Strings opArgs) unknown = PathSet(); } - if (settings.get("print-missing", true)) + if (settings.printMissing) printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (dryRun) return; @@ -795,11 +795,11 @@ static void opServe(Strings opFlags, Strings opArgs) settings.maxSilentTime = readInt(in); settings.buildTimeout = readInt(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - in >> settings.maxLogSize; + settings.maxLogSize = readNum<unsigned long>(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.set("build-repeat", std::to_string(readInt(in))); - settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); - settings.set("run-diff-hook", "true"); + settings.buildRepeat = readInt(in); + settings.enforceDeterminism = readInt(in); + settings.runDiffHook = true; } settings.printRepeatedBuilds = false; }; diff --git a/src/nix/build.cc b/src/nix/build.cc index 812464d7582b..00bda1fd1045 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,12 +1,11 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" using namespace nix; -struct CmdBuild : StoreCommand, MixDryRun, MixInstallables +struct CmdBuild : MixDryRun, InstallablesCommand { CmdBuild() { @@ -24,22 +23,9 @@ struct CmdBuild : StoreCommand, MixDryRun, MixInstallables void run(ref<Store> store) override { - auto elems = evalInstallables(store); + auto paths = buildInstallables(store, dryRun); - PathSet pathsToBuild; - - for (auto & elem : elems) { - if (elem.isDrv) - pathsToBuild.insert(elem.drvPath); - else - pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - - printMissing(store, pathsToBuild); - - if (dryRun) return; - - store->buildPaths(pathsToBuild); + printInfo("build result: %s", showPaths(paths)); } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index a1b2c120a5d9..3c82e0df57f7 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "derivations.hh" namespace nix { @@ -79,6 +80,13 @@ StoreCommand::StoreCommand() mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); } +ref<Store> StoreCommand::getStore() +{ + if (!_store) + _store = createStore(); + return ref<Store>(_store); +} + ref<Store> StoreCommand::createStore() { return openStore(storeUri); @@ -91,23 +99,24 @@ void StoreCommand::run() StorePathsCommand::StorePathsCommand() { - expectArgs("paths", &storePaths); mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); mkFlag(0, "all", "apply operation to the entire store", &all); } void StorePathsCommand::run(ref<Store> store) { + Paths storePaths; + if (all) { - if (storePaths.size()) + if (installables.size()) throw UsageError("‘--all’ does not expect arguments"); for (auto & p : store->queryAllValidPaths()) storePaths.push_back(p); } else { - for (auto & storePath : storePaths) - storePath = store->followLinksToStorePath(storePath); + for (auto & p : buildInstallables(store, false)) + storePaths.push_back(p); if (recursive) { PathSet closure; @@ -120,4 +129,14 @@ void StorePathsCommand::run(ref<Store> store) run(store, storePaths); } +void StorePathCommand::run(ref<Store> store) +{ + auto storePaths = buildInstallables(store, false); + + if (storePaths.size() != 1) + throw UsageError("this command requires exactly one store path"); + + run(store, *storePaths.begin()); +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index fa6c21abf8ad..4800b5c912e4 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -4,6 +4,9 @@ namespace nix { +struct Value; +class EvalState; + /* A command is an argument parser that can be executed by calling its run() method. */ struct Command : virtual Args @@ -33,16 +36,72 @@ struct StoreCommand : virtual Command std::string storeUri; StoreCommand(); void run() override; + ref<Store> getStore(); virtual ref<Store> createStore(); virtual void run(ref<Store>) = 0; + +private: + std::shared_ptr<Store> _store; +}; + +struct Installable +{ + virtual std::string what() = 0; + + virtual PathSet toBuildable() + { + throw Error("argument ‘%s’ cannot be built", what()); + } + + virtual Value * toValue(EvalState & state) + { + throw Error("argument ‘%s’ cannot be evaluated", what()); + } +}; + +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : virtual Args, StoreCommand +{ + std::vector<std::shared_ptr<Installable>> installables; + Path file; + + InstallablesCommand() + { + mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); + expectArgs("installables", &_installables); + } + + /* Return a value representing the Nix expression from which we + are installing. This is either the file specified by ‘--file’, + or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs + = import ...; bla = import ...; }’. */ + Value * getSourceExpr(EvalState & state); + + std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss); + + PathSet buildInstallables(ref<Store> store, bool dryRun); + + ref<EvalState> getEvalState(); + + void prepare() override; + + virtual bool useDefaultInstallables() { return true; } + +private: + + Strings _installables; + + std::shared_ptr<EvalState> evalState; + + Value * vSourceExpr = 0; }; /* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public StoreCommand +struct StorePathsCommand : public InstallablesCommand { private: - Paths storePaths; bool recursive = false; bool all = false; @@ -55,6 +114,18 @@ public: virtual void run(ref<Store> store, Paths storePaths) = 0; void run(ref<Store> store) override; + + bool useDefaultInstallables() override { return !all; } +}; + +/* A command that operates on exactly one store path. */ +struct StorePathCommand : public InstallablesCommand +{ + using StoreCommand::run; + + virtual void run(ref<Store> store, const Path & storePath) = 0; + + void run(ref<Store> store) override; }; typedef std::map<std::string, ref<Command>> Commands; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc new file mode 100644 index 000000000000..1a1866437b07 --- /dev/null +++ b/src/nix/dump-path.cc @@ -0,0 +1,35 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdDumpPath : StorePathCommand +{ + std::string name() override + { + return "dump-path"; + } + + std::string description() override + { + return "dump a store path to stdout (in NAR format)"; + } + + Examples examples() override + { + return { + Example{ + "To get a NAR from the binary cache https://cache.nixos.org/:", + "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" + }, + }; + } + + void run(ref<Store> store, const Path & storePath) override + { + FdSink sink(STDOUT_FILENO); + store->narFromPath(storePath, sink); + } +}; + +static RegisterCommand r1(make_ref<CmdDumpPath>()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc new file mode 100644 index 000000000000..632b555771c2 --- /dev/null +++ b/src/nix/edit.cc @@ -0,0 +1,75 @@ +#include "command.hh" +#include "shared.hh" +#include "eval.hh" +#include "attr-path.hh" + +#include <unistd.h> + +using namespace nix; + +struct CmdEdit : InstallablesCommand +{ + std::string name() override + { + return "edit"; + } + + std::string description() override + { + return "open the Nix expression of a Nix package in $EDITOR"; + } + + Examples examples() override + { + return { + Example{ + "To open the Nix expression of the GNU Hello package:", + "nix edit nixpkgs.hello" + }, + }; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + + for (auto & i : installables) { + auto v = i->toValue(*state); + + Value * v2; + try { + auto dummyArgs = state->allocBindings(0); + v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v); + } catch (Error &) { + throw Error("package ‘%s’ has no source location information", i->what()); + } + + auto pos = state->forceString(*v2); + debug("position is %s", pos); + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute ‘%s’", pos); + + std::string filename(pos, 0, colon); + int lineno = std::stoi(std::string(pos, colon + 1)); + + auto editor = getEnv("EDITOR", "cat"); + + Strings args{editor}; + + if (editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos) + args.push_back(fmt("+%d", lineno)); + + args.push_back(filename); + + execvp(editor.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("cannot run editor ‘%s’", editor); + } + } +}; + +static RegisterCommand r1(make_ref<CmdEdit>()); diff --git a/src/nix/eval.cc b/src/nix/eval.cc new file mode 100644 index 000000000000..981cefa800c4 --- /dev/null +++ b/src/nix/eval.cc @@ -0,0 +1,55 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "eval.hh" +#include "json.hh" +#include "value-to-json.hh" + +using namespace nix; + +struct CmdEval : MixJSON, InstallablesCommand +{ + bool raw = false; + + CmdEval() + { + mkFlag(0, "raw", "print strings unquoted", &raw); + } + + std::string name() override + { + return "eval"; + } + + std::string description() override + { + return "evaluate a Nix expression"; + } + + void run(ref<Store> store) override + { + if (raw && json) + throw UsageError("--raw and --json are mutually exclusive"); + + auto state = getEvalState(); + + auto jsonOut = json ? std::make_unique<JSONList>(std::cout) : nullptr; + + for (auto & i : installables) { + auto v = i->toValue(*state); + if (raw) { + std::cout << state->forceString(*v); + } else if (json) { + PathSet context; + auto jsonElem = jsonOut->placeholder(); + printValueAsJSON(*state, true, *v, jsonElem, context); + } else { + state->forceValueDeep(*v); + std::cout << *v << "\n"; + } + } + } +}; + +static RegisterCommand r1(make_ref<CmdEval>()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 8341bbc5a3a4..f23308b9bc30 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,21 +1,26 @@ +#include "command.hh" #include "attr-path.hh" #include "common-opts.hh" #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" #include "get-drvs.hh" -#include "installables.hh" #include "store-api.hh" +#include "shared.hh" + +#include <regex> namespace nix { -Value * MixInstallables::buildSourceExpr(EvalState & state) +Value * InstallablesCommand::getSourceExpr(EvalState & state) { - Value * vRoot = state.allocValue(); + if (vSourceExpr) return vSourceExpr; + + vSourceExpr = state.allocValue(); if (file != "") { Expr * e = state.parseExprFromFile(resolveExprPath(lookupFileArg(state, file))); - state.eval(e, *vRoot); + state.eval(e, *vSourceExpr); } else { @@ -24,7 +29,7 @@ Value * MixInstallables::buildSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vRoot, searchPath.size()); + state.mkAttrs(*vSourceExpr, searchPath.size()); std::unordered_set<std::string> seen; @@ -32,76 +37,219 @@ Value * MixInstallables::buildSourceExpr(EvalState & state) if (i.first == "") continue; if (seen.count(i.first)) continue; seen.insert(i.first); - if (!pathExists(i.second)) continue; - mkApp(*state.allocAttr(*vRoot, state.symbols.create(i.first)), +#if 0 + auto res = state.resolveSearchPathElem(i); + if (!res.first) continue; + if (!pathExists(res.second)) continue; + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), state.getBuiltin("import"), - mkString(*state.allocValue(), i.second)); + mkString(*state.allocValue(), res.second)); +#endif + Value * v1 = state.allocValue(); + mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); + Value * v2 = state.allocValue(); + mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), + state.getBuiltin("import"), *v2); } - vRoot->attrs->sort(); + vSourceExpr->attrs->sort(); } - return vRoot; + return vSourceExpr; } -UserEnvElems MixInstallables::evalInstallables(ref<Store> store) +struct InstallableStoreDrv : Installable { - UserEnvElems res; - - for (auto & installable : installables) { - - if (std::string(installable, 0, 1) == "/") { - - if (store->isStorePath(installable)) { - - if (isDerivation(installable)) { - UserEnvElem elem; - // FIXME: handle empty case, drop version - elem.attrPath = {storePathToName(installable)}; - elem.isDrv = true; - elem.drvPath = installable; - res.push_back(elem); - } - - else { - UserEnvElem elem; - // FIXME: handle empty case, drop version - elem.attrPath = {storePathToName(installable)}; - elem.isDrv = false; - elem.outPaths = {installable}; - res.push_back(elem); - } - } + Path storePath; - else - throw UsageError(format("don't know what to do with ‘%1%’") % installable); - } + InstallableStoreDrv(const Path & storePath) : storePath(storePath) { } + + std::string what() override { return storePath; } + + PathSet toBuildable() override + { + return {storePath}; + } +}; + +struct InstallableStorePath : Installable +{ + Path storePath; + + InstallableStorePath(const Path & storePath) : storePath(storePath) { } + + std::string what() override { return storePath; } + + PathSet toBuildable() override + { + return {storePath}; + } +}; + +struct InstallableExpr : Installable +{ + InstallablesCommand & installables; + std::string text; + + InstallableExpr(InstallablesCommand & installables, const std::string & text) + : installables(installables), text(text) { } + + std::string what() override { return text; } + + PathSet toBuildable() override + { + auto state = installables.getEvalState(); + + auto v = toValue(*state); + + // FIXME + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_)); + + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); + + PathSet res; + + for (auto & i : drvs) + res.insert(i.queryDrvPath()); + + return res; + } + + Value * toValue(EvalState & state) override + { + auto v = state.allocValue(); + state.eval(state.parseExprFromString(text, absPath(".")), *v); + return v; + } +}; + +struct InstallableAttrPath : Installable +{ + InstallablesCommand & installables; + std::string attrPath; + + InstallableAttrPath(InstallablesCommand & installables, const std::string & attrPath) + : installables(installables), attrPath(attrPath) + { } + + std::string what() override { return attrPath; } - else { + PathSet toBuildable() override + { + auto state = installables.getEvalState(); - EvalState state({}, store); + auto v = toValue(*state); - auto vRoot = buildSourceExpr(state); + // FIXME + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(*state, autoArgs_)); - std::map<string, string> autoArgs_; - Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); - Value & v(*findAlongAttrPath(state, installable, autoArgs, *vRoot)); - state.forceValue(v); + PathSet res; - DrvInfos drvs; - getDerivations(state, v, "", autoArgs, drvs, false); + for (auto & i : drvs) + res.insert(i.queryDrvPath()); - for (auto & i : drvs) { - UserEnvElem elem; - elem.isDrv = true; - elem.drvPath = i.queryDrvPath(); - res.push_back(elem); + return res; + } + + Value * toValue(EvalState & state) override + { + auto source = installables.getSourceExpr(state); + + // FIXME + std::map<string, string> autoArgs_; + Bindings & autoArgs(*evalAutoArgs(state, autoArgs_)); + + Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); + state.forceValue(*v); + + return v; + } +}; + +// FIXME: extend +std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; +static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); + +std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings ss) +{ + std::vector<std::shared_ptr<Installable>> result; + + if (ss.empty() && useDefaultInstallables()) { + if (file == "") + file = "."; + ss = Strings{""}; + } + + for (auto & s : ss) { + + if (s.find("/") != std::string::npos) { + + auto path = store->toStorePath(store->followLinksToStore(s)); + + if (store->isStorePath(path)) { + if (isDerivation(path)) + result.push_back(std::make_shared<InstallableStoreDrv>(path)); + else + result.push_back(std::make_shared<InstallableStorePath>(path)); } } + + else if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(*this, s)); + + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared<InstallableAttrPath>(*this, s)); + + else + throw UsageError("don't know what to do with argument ‘%s’", s); } - return res; + return result; +} + +PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun) +{ + PathSet buildables; + + for (auto & i : installables) { + auto b = i->toBuildable(); + buildables.insert(b.begin(), b.end()); + } + + if (dryRun) + printMissing(store, buildables); + else + store->buildPaths(buildables); + + PathSet outPaths; + for (auto & path : buildables) + if (isDerivation(path)) { + Derivation drv = store->derivationFromPath(path); + for (auto & output : drv.outputs) + outPaths.insert(output.second.path); + } else + outPaths.insert(path); + + return outPaths; +} + +ref<EvalState> InstallablesCommand::getEvalState() +{ + if (!evalState) + evalState = std::make_shared<EvalState>(Strings{}, getStore()); + return ref<EvalState>(evalState); +} + +void InstallablesCommand::prepare() +{ + installables = parseInstallables(getStore(), _installables); } } diff --git a/src/nix/installables.hh b/src/nix/installables.hh deleted file mode 100644 index a58f7dc59bb4..000000000000 --- a/src/nix/installables.hh +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "args.hh" - -namespace nix { - -struct UserEnvElem -{ - Strings attrPath; - - // FIXME: should use boost::variant or so. - bool isDrv; - - // Derivation case: - Path drvPath; - StringSet outputNames; - - // Non-derivation case: - PathSet outPaths; -}; - -typedef std::vector<UserEnvElem> UserEnvElems; - -struct Value; -class EvalState; - -struct MixInstallables : virtual Args -{ - Strings installables; - Path file; - - MixInstallables() - { - mkFlag('f', "file", "file", "evaluate FILE rather than the default", &file); - expectArgs("installables", &installables); - } - - UserEnvElems evalInstallables(ref<Store> store); - - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * buildSourceExpr(EvalState & state); - -}; - -} diff --git a/src/nix/local.mk b/src/nix/local.mk index f6e7073b6e7d..c7d2d328aab5 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -2,7 +2,7 @@ programs += nix nix_DIR := $(d) -nix_SOURCES := $(wildcard $(d)/*.cc) +nix_SOURCES := $(wildcard $(d)/*.cc) src/linenoise/linenoise.c nix_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix/log.cc b/src/nix/log.cc index d8a3830e91c8..ed610261d1ca 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,12 +1,11 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" using namespace nix; -struct CmdLog : StoreCommand, MixInstallables +struct CmdLog : InstallablesCommand { CmdLog() { @@ -24,32 +23,23 @@ struct CmdLog : StoreCommand, MixInstallables void run(ref<Store> store) override { - auto elems = evalInstallables(store); - - PathSet paths; - - for (auto & elem : elems) { - if (elem.isDrv) - paths.insert(elem.drvPath); - else - paths.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - auto subs = getDefaultSubstituters(); subs.push_front(store); - for (auto & path : paths) { - bool found = false; - for (auto & sub : subs) { - auto log = sub->getBuildLog(path); - if (!log) continue; - std::cout << *log; - found = true; - break; + for (auto & inst : installables) { + for (auto & path : inst->toBuildable()) { + bool found = false; + for (auto & sub : subs) { + auto log = sub->getBuildLog(path); + if (!log) continue; + std::cout << *log; + found = true; + break; + } + if (!found) + throw Error("build log of path ‘%s’ is not available", path); } - if (!found) - throw Error("build log of path ‘%s’ is not available", path); } } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3476dfb05287..417b7b421b1c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -61,6 +61,10 @@ struct MixLs : virtual Args showFile(curPath, relPath); }; + if (path == "/") { + path = ""; + } + auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) throw Error(format("path ‘%1%’ does not exist") % path); diff --git a/src/nix/main.cc b/src/nix/main.cc index 440ced97dfcc..216f0bccef11 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -27,6 +27,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { + verbosity = lvlError; settings.verboseBuild = false; initNix(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 0f9a1125f2e9..f16209238610 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,25 +2,24 @@ #include "shared.hh" #include "store-api.hh" #include "json.hh" +#include "common-args.hh" #include <iomanip> #include <algorithm> using namespace nix; -struct CmdPathInfo : StorePathsCommand +struct CmdPathInfo : StorePathsCommand, MixJSON { bool showSize = false; bool showClosureSize = false; bool showSigs = false; - bool json = false; CmdPathInfo() { mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); mkFlag(0, "sigs", "show signatures", &showSigs); - mkFlag(0, "json", "produce JSON output", &json); } std::string name() override @@ -100,7 +99,6 @@ struct CmdPathInfo : StorePathsCommand } } - } }; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 69811b282804..24e435f81e8b 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -1,8 +1,12 @@ #include "progress-bar.hh" #include "util.hh" #include "sync.hh" +#include "store-api.hh" #include <map> +#include <atomic> + +#include <sys/ioctl.h> namespace nix { @@ -12,31 +16,47 @@ private: struct ActInfo { - Activity * activity; - Verbosity lvl; - std::string s; + std::string s, s2; }; - struct Progress + struct DownloadInfo { - uint64_t expected = 0, progress = 0; + std::string uri; + uint64_t current = 0; + uint64_t expected = 0; }; struct State { + std::map<Activity::t, Path> builds; + std::set<Activity::t> runningBuilds; + uint64_t succeededBuilds = 0; + uint64_t failedBuilds = 0; + std::map<Activity::t, Path> substitutions; + std::set<Activity::t> runningSubstitutions; + uint64_t succeededSubstitutions = 0; + uint64_t downloadedBytes = 0; // finished downloads + std::map<Activity::t, DownloadInfo> downloads; std::list<ActInfo> activities; - std::map<Activity *, std::list<ActInfo>::iterator> its; - std::map<std::string, Progress> progress; + std::map<Activity::t, std::list<ActInfo>::iterator> its; }; Sync<State> state_; + int width = 0; + public: + ProgressBar() + { + struct winsize ws; + if (ioctl(1, TIOCGWINSZ, &ws) == 0) + width = ws.ws_col; + } + ~ProgressBar() { auto state(state_.lock()); - assert(state->activities.empty()); writeToStderr("\r\e[K"); } @@ -52,52 +72,36 @@ public: update(state); } - void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override - { - if (lvl > verbosity) return; - auto state(state_.lock()); - state->activities.emplace_back(ActInfo{&activity, lvl, fs.s}); - state->its.emplace(&activity, std::prev(state->activities.end())); - update(*state); - } - - void stopActivity(Activity & activity) override - { - auto state(state_.lock()); - auto i = state->its.find(&activity); - if (i == state->its.end()) return; - state->activities.erase(i->second); - state->its.erase(i); - update(*state); - } - - void setExpected(const std::string & label, uint64_t value) override + void createActivity(State & state, Activity::t activity, const std::string & s) { - auto state(state_.lock()); - state->progress[label].expected = value; - } - - void setProgress(const std::string & label, uint64_t value) override - { - auto state(state_.lock()); - state->progress[label].progress = value; + state.activities.emplace_back(ActInfo{s}); + state.its.emplace(activity, std::prev(state.activities.end())); } - void incExpected(const std::string & label, uint64_t value) override + void deleteActivity(State & state, Activity::t activity) { - auto state(state_.lock()); - state->progress[label].expected += value; + auto i = state.its.find(activity); + if (i != state.its.end()) { + state.activities.erase(i->second); + state.its.erase(i); + } } - void incProgress(const std::string & label, uint64_t value) override + void updateActivity(State & state, Activity::t activity, const std::string & s2) { - auto state(state_.lock()); - state->progress[label].progress += value; + auto i = state.its.find(activity); + assert(i != state.its.end()); + ActInfo info = *i->second; + state.activities.erase(i->second); + info.s2 = s2; + state.activities.emplace_back(info); + i->second = std::prev(state.activities.end()); } void update() { auto state(state_.lock()); + update(*state); } void update(State & state) @@ -113,28 +117,169 @@ public: if (!state.activities.empty()) { if (!status.empty()) line += " "; - line += state.activities.rbegin()->s; + auto i = state.activities.rbegin(); + line += i->s; + if (!i->s2.empty()) { + line += ": "; + line += i->s2; + } } line += "\e[K"; - writeToStderr(line); + writeToStderr(std::string(line, 0, width - 1)); } std::string getStatus(State & state) { std::string res; - for (auto & p : state.progress) - if (p.second.expected || p.second.progress) { - if (!res.empty()) res += ", "; - res += std::to_string(p.second.progress); - if (p.second.expected) { - res += "/"; - res += std::to_string(p.second.expected); - } - res += " "; res += p.first; + + if (state.failedBuilds) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds); + } + + if (!state.builds.empty() || state.succeededBuilds) + { + if (!res.empty()) res += ", "; + if (!state.runningBuilds.empty()) + res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningBuilds.size()); + res += fmt(ANSI_GREEN "%d/%d built" ANSI_NORMAL, + state.succeededBuilds, state.succeededBuilds + state.builds.size()); + } + + if (!state.substitutions.empty() || state.succeededSubstitutions) { + if (!res.empty()) res += ", "; + if (!state.runningSubstitutions.empty()) + res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningSubstitutions.size()); + res += fmt(ANSI_GREEN "%d/%d fetched" ANSI_NORMAL, + state.succeededSubstitutions, + state.succeededSubstitutions + state.substitutions.size()); + } + + if (!state.downloads.empty() || state.downloadedBytes) { + if (!res.empty()) res += ", "; + uint64_t expected = state.downloadedBytes, current = state.downloadedBytes; + for (auto & i : state.downloads) { + expected += i.second.expected; + current += i.second.current; } + res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0); + } + return res; } + + void event(const Event & ev) override + { + if (ev.type == evBuildCreated) { + auto state(state_.lock()); + state->builds[ev.getI(0)] = ev.getS(1); + update(*state); + } + + if (ev.type == evBuildStarted) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + state->runningBuilds.insert(act); + auto name = storePathToName(state->builds[act]); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name)); + update(*state); + } + + if (ev.type == evBuildFinished) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + if (ev.getI(1)) { + if (state->runningBuilds.count(act)) + state->succeededBuilds++; + } else + state->failedBuilds++; + state->runningBuilds.erase(act); + state->builds.erase(act); + deleteActivity(*state, act); + update(*state); + } + + if (ev.type == evBuildOutput) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + assert(state->runningBuilds.count(act)); + updateActivity(*state, act, ev.getS(1)); + update(*state); + } + + if (ev.type == evSubstitutionCreated) { + auto state(state_.lock()); + state->substitutions[ev.getI(0)] = ev.getS(1); + update(*state); + } + + if (ev.type == evSubstitutionStarted) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + state->runningSubstitutions.insert(act); + auto name = storePathToName(state->substitutions[act]); + createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name)); + update(*state); + } + + if (ev.type == evSubstitutionFinished) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + if (ev.getI(1)) { + if (state->runningSubstitutions.count(act)) + state->succeededSubstitutions++; + } + state->runningSubstitutions.erase(act); + state->substitutions.erase(act); + deleteActivity(*state, act); + update(*state); + } + + if (ev.type == evDownloadCreated) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + std::string uri = ev.getS(1); + state->downloads.emplace(act, DownloadInfo{uri}); + if (state->runningSubstitutions.empty()) // FIXME: hack + createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri)); + update(*state); + } + + if (ev.type == evDownloadProgress) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto i = state->downloads.find(act); + assert(i != state->downloads.end()); + i->second.expected = ev.getI(1); + i->second.current = ev.getI(2); + update(*state); + } + + if (ev.type == evDownloadSucceeded) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto i = state->downloads.find(act); + assert(i != state->downloads.end()); + state->downloadedBytes += ev.getI(1); + state->downloads.erase(i); + deleteActivity(*state, act); + update(*state); + } + + if (ev.type == evDownloadDestroyed) { + auto state(state_.lock()); + Activity::t act = ev.getI(0); + auto i = state->downloads.find(act); + if (i != state->downloads.end()) { + state->downloads.erase(i); + deleteActivity(*state, act); + update(*state); + } + } + } }; StartProgressBar::StartProgressBar() diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 000000000000..437c7903ed40 --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,689 @@ +#include <iostream> +#include <cstdlib> + +#include <setjmp.h> + +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "common-opts.hh" +#include "get-drvs.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "globals.hh" +#include "command.hh" +#include "finally.hh" + +#include "src/linenoise/linenoise.h" + +namespace nix { + +#define ESC_RED "\033[31m" +#define ESC_GRE "\033[32m" +#define ESC_YEL "\033[33m" +#define ESC_BLU "\033[34;1m" +#define ESC_MAG "\033[35m" +#define ESC_CYA "\033[36m" +#define ESC_END "\033[0m" + +struct NixRepl +{ + string curDir; + EvalState state; + + Strings loadedFiles; + + const static int envSize = 32768; + StaticEnv staticEnv; + Env * env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings & searchPath, nix::ref<Store> store); + ~NixRepl(); + void mainLoop(const Strings & files); + StringSet completePrefix(string prefix); + bool getLine(string & input, const std::string &prompt); + Path getDerivationPath(Value & v); + bool processLine(string line); + void loadFile(const Path & path); + void initEnv(); + void reloadFiles(); + void addAttrsToScope(Value & attrs); + void addVarToScope(const Symbol & name, Value & v); + Expr * parseString(string s); + void evalString(string s, Value & v); + + typedef set<Value *> ValuesSeen; + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); +}; + + +void printHelp() +{ + std::cout + << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" + << "\n" + << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" + << "\n" + << "Options:\n" + << " --help\n" + << " Prints out a summary of the command syntax and exits.\n" + << "\n" + << " --version\n" + << " Prints out the Nix version number on standard output and exits.\n" + << "\n" + << " -I path\n" + << " Add a path to the Nix expression search path. This option may be given\n" + << " multiple times. See the NIX_PATH environment variable for information on\n" + << " the semantics of the Nix search path. Paths added through -I take\n" + << " precedence over NIX_PATH.\n" + << "\n" + << " paths...\n" + << " A list of paths to files containing Nix expressions which nix-repl will\n" + << " load and add to its scope.\n" + << "\n" + << " A path surrounded in < and > will be looked up in the Nix expression search\n" + << " path, as in the Nix language itself.\n" + << "\n" + << " If an element of paths starts with http:// or https://, it is interpreted\n" + << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" + << " location. The tarball must include a single top-level directory containing\n" + << " at least a file named default.nix.\n"; +} + + +string removeWhitespace(string s) +{ + s = chomp(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != string::npos) s = string(s, n); + return s; +} + + +NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store) + : state(searchPath, store) + , staticEnv(false, &state.staticBaseEnv) + , historyFile(getDataDir() + "/nix/repl-history") +{ + curDir = absPath("."); +} + + +NixRepl::~NixRepl() +{ + linenoiseHistorySave(historyFile.c_str()); +} + + +static NixRepl * curRepl; // ugly + +static void completionCallback(const char * s, linenoiseCompletions *lc) +{ + /* Otherwise, return all symbols that start with the prefix. */ + for (auto & c : curRepl->completePrefix(s)) + linenoiseAddCompletion(lc, c.c_str()); +} + + +void NixRepl::mainLoop(const Strings & files) +{ + string error = ANSI_RED "error:" ANSI_NORMAL " "; + std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; + + for (auto & i : files) + loadedFiles.push_back(i); + + reloadFiles(); + if (!loadedFiles.empty()) std::cout << std::endl; + + createDirs(dirOf(historyFile)); + linenoiseHistorySetMaxLen(1000); + linenoiseHistoryLoad(historyFile.c_str()); + + curRepl = this; + linenoiseSetCompletionCallback(completionCallback); + + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just align to the same + // number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) + break; + + try { + if (!removeWhitespace(input).empty() && !processLine(input)) return; + } catch (ParseError & e) { + if (e.msg().find("unexpected $end") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the next line of + // input without clearing the input so far. + continue; + } else { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } + } catch (Error & e) { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } catch (Interrupted & e) { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } + + // We handled the current input fully, so we should clear it + // and read brand new input. + linenoiseHistoryAdd(input.c_str()); + input.clear(); + std::cout << std::endl; + } +} + + +bool NixRepl::getLine(string & input, const std::string &prompt) +{ + char * s = linenoise(prompt.c_str()); + Finally doFree([&]() { linenoiseFree(s); }); + if (!s) return false; + input += s; + return true; +} + + +StringSet NixRepl::completePrefix(string prefix) +{ + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev, cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } + + size_t slash, dot; + + if ((slash = cur.rfind('/')) != string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { + if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) + completions.insert(prev + dir + "/" + entry.name); + } + } catch (Error &) { + } + } else if ((dot = cur.rfind('.')) == string::npos) { + /* This is a variable name; look it up in the current scope. */ + StringSet::iterator i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (string(*i, 0, cur.size()) != cur) break; + completions.insert(prev + *i); + i++; + } + } else { + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + string expr(cur, 0, dot); + string cur2 = string(cur, dot + 1); + + Expr * e = parseString(expr); + Value v; + e->eval(state, *env, v); + state.forceAttrs(v); + + for (auto & i : *v.attrs) { + string name = i.name; + if (string(name, 0, cur2.size()) != cur2) continue; + completions.insert(prev + expr + "." + name); + } + + } catch (ParseError & e) { + // Quietly ignore parse errors. + } catch (EvalError & e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError & e) { + // Quietly ignore undefined variable errors. + } + } + + return completions; +} + + +static int runProgram(const string & program, const Strings & args) +{ + Strings args2(args); + args2.push_front(program); + + Pid pid; + pid = fork(); + if (pid == -1) throw SysError("forking"); + if (pid == 0) { + restoreAffinity(); + execvp(program.c_str(), stringsToCharPtrs(args2).data()); + _exit(1); + } + + return pid.wait(); +} + + +bool isVarName(const string & s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + + +Path NixRepl::getDerivationPath(Value & v) { + DrvInfo drvInfo(state); + if (!getDerivation(state, v, drvInfo, false)) + throw Error("expression does not evaluate to a derivation, so I can't build it"); + Path drvPath = drvInfo.queryDrvPath(); + if (drvPath == "" || !state.store->isValidPath(drvPath)) + throw Error("expression did not evaluate to a valid derivation"); + return drvPath; +} + + +bool NixRepl::processLine(string line) +{ + if (line == "") return true; + + string command, arg; + + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = string(line, 0, p); + if (p != string::npos) arg = removeWhitespace(string(line, p)); + } else { + arg = line; + } + + if (command == ":?" || command == ":help") { + std::cout + << "The following commands are available:\n" + << "\n" + << " <expr> Evaluate and print expression\n" + << " <x> = <expr> Bind expression to variable\n" + << " :a <expr> Add attributes from resulting set to scope\n" + << " :b <expr> Build derivation\n" + << " :i <expr> Build derivation, then install result into current profile\n" + << " :l <path> Load Nix expression and add it to scope\n" + << " :p <expr> Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :s <expr> Build dependencies of derivation, then start nix-shell\n" + << " :t <expr> Describe result of evaluation\n" + << " :u <expr> Build derivation, then start nix-shell\n"; + } + + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } + + else if (command == ":l" || command == ":load") { + state.resetFileCache(); + loadFile(arg); + } + + else if (command == ":r" || command == ":reload") { + state.resetFileCache(); + reloadFiles(); + } + + else if (command == ":t") { + Value v; + evalString(arg, v); + std::cout << showType(v) << std::endl; + + } else if (command == ":u") { + Value v, f, result; + evalString(arg, v); + evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); + state.callFunction(f, v, result, Pos()); + + Path drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + + else if (command == ":b" || command == ":i" || command == ":s") { + Value v; + evalString(arg, v); + Path drvPath = getDerivationPath(v); + + if (command == ":b") { + /* We could do the build in this process using buildPaths(), + but doing it in a child makes it easier to recover from + problems / SIGINT. */ + if (runProgram(settings.nixBinDir + "/nix-store", Strings{"-r", drvPath}) == 0) { + Derivation drv = readDerivation(drvPath); + std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; + for (auto & i : drv.outputs) + std::cout << format(" %1% -> %2%") % i.first % i.second.path << std::endl; + } + } else if (command == ":i") { + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + } else { + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + } + + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } + + else if (command == ":q" || command == ":quit") + return false; + + else if (command != "") + throw Error(format("unknown command ‘%1%’") % command); + + else { + size_t p = line.find('='); + string name; + if (p != string::npos && + p < line.size() && + line[p + 1] != '=' && + isVarName(name = removeWhitespace(string(line, 0, p)))) + { + Expr * e = parseString(string(line, p + 1)); + Value & v(*state.allocValue()); + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = e; + addVarToScope(state.symbols.create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; + } + } + + return true; +} + + +void NixRepl::loadFile(const Path & path) +{ + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v, v2; + state.evalFile(lookupFileArg(state, path), v); + Bindings & bindings(*state.allocBindings(0)); + state.autoCallFunction(bindings, v, v2); + addAttrsToScope(v2); +} + + +void NixRepl::initEnv() +{ + env = &state.allocEnv(envSize); + env->up = &state.baseEnv; + displ = 0; + staticEnv.vars.clear(); + + varNames.clear(); + for (auto & i : state.staticBaseEnv.vars) + varNames.insert(i.first); +} + + +void NixRepl::reloadFiles() +{ + initEnv(); + + Strings old = loadedFiles; + loadedFiles.clear(); + + bool first = true; + for (auto & i : old) { + if (!first) std::cout << std::endl; + first = false; + std::cout << format("Loading ‘%1%’...") % i << std::endl; + loadFile(i); + } +} + + +void NixRepl::addAttrsToScope(Value & attrs) +{ + state.forceAttrs(attrs); + for (auto & i : *attrs.attrs) + addVarToScope(i.name, *i.value); + std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl; +} + + +void NixRepl::addVarToScope(const Symbol & name, Value & v) +{ + if (displ >= envSize) + throw Error("environment full; cannot add more variables"); + staticEnv.vars[name] = displ; + env->values[displ++] = &v; + varNames.insert((string) name); +} + + +Expr * NixRepl::parseString(string s) +{ + Expr * e = state.parseExprFromString(s, curDir, staticEnv); + return e; +} + + +void NixRepl::evalString(string s, Value & v) +{ + Expr * e = parseString(s); + e->eval(state, *env, v); + state.forceValue(v); +} + + +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) +{ + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); +} + + +std::ostream & printStringValue(std::ostream & str, const char * string) { + str << "\""; + for (const char * i = string; *i; i++) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; + return str; +} + + +// FIXME: lot of cut&paste from Nix's eval.cc. +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) +{ + str.flush(); + checkInterrupt(); + + state.forceValue(v); + + switch (v.type) { + + case tInt: + str << ESC_CYA << v.integer << ESC_END; + break; + + case tBool: + str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + break; + + case tString: + str << ESC_YEL; + printStringValue(str, v.string.s); + str << ESC_END; + break; + + case tPath: + str << ESC_GRE << v.path << ESC_END; // !!! escaping? + break; + + case tNull: + str << ESC_CYA "null" ESC_END; + break; + + case tAttrs: { + seen.insert(&v); + + bool isDrv = state.isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state.sDrvPath); + PathSet context; + Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->pos, *i->value, context) : "???"; + str << drvPath << "»"; + } + + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map<string, Value *> Sorted; + Sorted sorted; + for (auto & i : *v.attrs) + sorted[i.name] = i.value; + + /* If this is a derivation, then don't show the + self-references ("all", "out", etc.). */ + StringSet hidden; + if (isDrv) { + hidden.insert("all"); + Bindings::iterator i = v.attrs->find(state.sOutputs); + if (i == v.attrs->end()) + hidden.insert("out"); + else { + state.forceList(*i->value); + for (unsigned int j = 0; j < i->value->listSize(); ++j) + hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j])); + } + } + + for (auto & i : sorted) { + if (isVarName(i.first)) + str << i.first; + else + printStringValue(str, i.first.c_str()); + str << " = "; + if (hidden.find(i.first) != hidden.end()) + str << "«...»"; + else if (seen.find(i.second) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << "; "; + } + + str << "}"; + } else + str << "{ ... }"; + + break; + } + + case tList1: + case tList2: + case tListN: + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (seen.find(v.listElems()[n]) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *v.listElems()[n], maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << " "; + } + else + str << "... "; + str << "]"; + break; + + case tLambda: { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + break; + } + + case tPrimOp: + str << ESC_MAG "«primop»" ESC_END; + break; + + case tPrimOpApp: + str << ESC_BLU "«primop-app»" ESC_END; + break; + + default: + str << ESC_RED "«unknown»" ESC_END; + break; + } + + return str; +} + +struct CmdRepl : StoreCommand +{ + Strings files; + + CmdRepl() + { + expectArgs("files", &files); + } + + std::string name() override { return "repl"; } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + void run(ref<Store> store) override + { + // FIXME: pass searchPath + NixRepl repl({}, openStore()); + repl.mainLoop(files); + } +}; + +static RegisterCommand r1(make_ref<CmdRepl>()); + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index a30031ad07b4..bcfa74eb5f5f 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "common-args.hh" -#include "installables.hh" #include "shared.hh" #include "store-api.hh" #include "derivations.hh" @@ -13,7 +12,7 @@ using namespace nix; -struct CmdRun : StoreCommand, MixInstallables +struct CmdRun : InstallablesCommand { CmdRun() { @@ -31,20 +30,7 @@ struct CmdRun : StoreCommand, MixInstallables void run(ref<Store> store) override { - auto elems = evalInstallables(store); - - PathSet pathsToBuild; - - for (auto & elem : elems) { - if (elem.isDrv) - pathsToBuild.insert(elem.drvPath); - else - pathsToBuild.insert(elem.outPaths.begin(), elem.outPaths.end()); - } - - printMissing(store, pathsToBuild); - - store->buildPaths(pathsToBuild); + auto outPaths = buildInstallables(store, false); auto store2 = store.dynamic_pointer_cast<LocalStore>(); @@ -103,14 +89,6 @@ struct CmdRun : StoreCommand, MixInstallables #endif } - PathSet outPaths; - for (auto & path : pathsToBuild) - if (isDerivation(path)) { - Derivation drv = store->derivationFromPath(path); - for (auto & output : drv.outputs) - outPaths.insert(output.second.path); - } else - outPaths.insert(path); auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":"); for (auto & path : outPaths) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc new file mode 100644 index 000000000000..c628c2898d73 --- /dev/null +++ b/src/nix/show-config.cc @@ -0,0 +1,38 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "json.hh" + +using namespace nix; + +struct CmdShowConfig : Command, MixJSON +{ + CmdShowConfig() + { + } + + std::string name() override + { + return "show-config"; + } + + std::string description() override + { + return "show the Nix configuration"; + } + + void run() override + { + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + JSONObject jsonObj(std::cout, true); + settings.toJSON(jsonObj); + } else { + for (auto & s : settings.getSettings()) + std::cout << s.first + " = " + s.second + "\n"; + } + } +}; + +static RegisterCommand r1(make_ref<CmdShowConfig>()); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index d8d8c0f53df0..3dd03771619f 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -42,10 +42,10 @@ struct CmdCopySigs : StorePathsCommand std::string doneLabel = "done"; std::atomic<size_t> added{0}; - logger->setExpected(doneLabel, storePaths.size()); + //logger->setExpected(doneLabel, storePaths.size()); auto doPath = [&](const Path & storePath) { - Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); + //Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); checkInterrupt(); @@ -76,7 +76,7 @@ struct CmdCopySigs : StorePathsCommand added += newSigs.size(); } - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); }; for (auto & storePath : storePaths) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2f8d02fa060e..8facb4bef8a2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -65,7 +65,7 @@ struct CmdVerify : StorePathsCommand std::string untrustedLabel("untrusted"); std::string corruptedLabel("corrupted"); std::string failedLabel("failed"); - logger->setExpected(doneLabel, storePaths.size()); + //logger->setExpected(doneLabel, storePaths.size()); ThreadPool pool; @@ -73,7 +73,7 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); + //Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); auto info = store->queryPathInfo(storePath); @@ -85,7 +85,7 @@ struct CmdVerify : StorePathsCommand auto hash = sink.finish(); if (hash.first != info->narHash) { - logger->incProgress(corruptedLabel); + //logger->incProgress(corruptedLabel); corrupted = 1; printError( format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") @@ -137,19 +137,19 @@ struct CmdVerify : StorePathsCommand } if (!good) { - logger->incProgress(untrustedLabel); + //logger->incProgress(untrustedLabel); untrusted++; printError(format("path ‘%s’ is untrusted") % info->path); } } - logger->incProgress(doneLabel); + //logger->incProgress(doneLabel); done++; } catch (Error & e) { printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); - logger->incProgress(failedLabel); + //logger->incProgress(failedLabel); failed++; } }; |