diff options
Diffstat (limited to 'src')
42 files changed, 371 insertions, 107 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index c6e75e8cc704..dbf8fe1b8f8a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -64,6 +64,8 @@ int main (int argc, char * * argv) settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + initPlugins(); + auto store = openStore().cast<LocalStore>(); /* It would be more appropriate to use $XDG_RUNTIME_DIR, since diff --git a/src/buildenv/buildenv.cc b/src/buildenv/buildenv.cc index eddb9fdaa8d2..2afad913ac6b 100644 --- a/src/buildenv/buildenv.cc +++ b/src/buildenv/buildenv.cc @@ -74,10 +74,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto prevPriority = priorities[dstFile]; if (prevPriority == priority) throw Error(format( - "Packages '%1%' and '%2%' have the same priority '%3%'" + "packages '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " "to change the priority of one of the conflicting packages" - " ('0' being the highest priority)" + " (0 being the highest priority)" ) % srcFile % target % priority); if (prevPriority < priority) continue; diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 910428c02686..b284daa3c2f7 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -7,13 +7,14 @@ namespace nix { +/* Note: Various places expect the allocated memory to be zeroed. */ static void * allocBytes(size_t n) { void * p; #if HAVE_BOEHMGC p = GC_malloc(n); #else - p = malloc(n); + p = calloc(n, 1); #endif if (!p) throw std::bad_alloc(); return p; diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index e1fc2bf6d796..3119a1848af2 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -83,7 +83,7 @@ public: for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]); std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { - return (string) a->name < (string) b->name; + return (const string &) a->name < (const string &) b->name; }); return res; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 11195af77939..f94c23ea72bb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -43,13 +43,14 @@ static char * dupString(const char * s) } +/* Note: Various places expect the allocated memory to be zeroed. */ static void * allocBytes(size_t n) { void * p; #if HAVE_BOEHMGC p = GC_malloc(n); #else - p = malloc(n); + p = calloc(n, 1); #endif if (!p) throw std::bad_alloc(); return p; @@ -293,6 +294,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) + , sArgs(symbols.create("args")) + , sOutputHash(symbols.create("outputHash")) + , sOutputHashAlgo(symbols.create("outputHashAlgo")) + , sOutputHashMode(symbols.create("outputHashMode")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -308,7 +313,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); } - addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); if (settings.restrictEval || settings.pureEval) { allowedPaths = PathSet(); @@ -404,7 +409,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context) }; -void EvalState::addConstant(const string & name, Value & v) +Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; @@ -412,12 +417,18 @@ void EvalState::addConstant(const string & name, Value & v) baseEnv.values[baseEnvDispl++] = v2; string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + return v2; } Value * EvalState::addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } Value * v = allocValue(); string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; Symbol sym = symbols.create(name2); @@ -576,9 +587,7 @@ Env & EvalState::allocEnv(unsigned int size) Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); env->size = size; - /* Clear the values because maybeThunk() and lookupVar fromWith expect this. */ - for (unsigned i = 0; i < size; ++i) - env->values[i] = 0; + /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9e3d30d95f49..9d8799b7906b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -69,7 +69,8 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder; + sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sOutputHash, sOutputHashAlgo, sOutputHashMode; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -210,7 +211,7 @@ private: void createBaseEnv(); - void addConstant(const string & name, Value & v); + Value * addConstant(const string & name, Value & v); Value * addPrimOp(const string & name, unsigned int arity, PrimOpFun primOp); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 28a0a6a87896..e5e01fb5831a 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -49,9 +49,10 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } -static Expr * unescapeStr(SymbolTable & symbols, const char * s) +static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; + t.reserve(length); char c; while ((c = *s++)) { if (c == '\\') { @@ -150,7 +151,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); + yylval->e = unescapeStr(data->symbols, yytext, yyleng); return STR; } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } @@ -178,7 +179,7 @@ or { return OR_KW; } return IND_STR; } <IND_STRING>\'\'\\. { - yylval->e = unescapeStr(data->symbols, yytext + 2); + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } <IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc index 6d78d2116121..382088c78872 100644 --- a/src/libexpr/names.cc +++ b/src/libexpr/names.cc @@ -41,7 +41,7 @@ bool DrvName::matches(DrvName & n) } -static string nextComponent(string::const_iterator & p, +string nextComponent(string::const_iterator & p, const string::const_iterator end) { /* Skip any dots and dashes (component separators). */ diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index 9667fc96fd0f..13c3093e77b0 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -24,6 +24,8 @@ private: typedef list<DrvName> DrvNames; +string nextComponent(string::const_iterator & p, + const string::const_iterator end); int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 466fd13e8698..6778023f506d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -553,7 +553,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : args[0]->attrs->lexicographicOrder()) { if (i->name == state.sIgnoreNulls) continue; - string key = i->name; + const string & key = i->name; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { @@ -589,7 +589,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (key == "args") { + if (i->name == state.sArgs) { state.forceList(*i->value, pos); for (unsigned int n = 0; n < i->value->listSize(); ++n) { string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); @@ -612,15 +612,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.builder = state.forceString(*i->value, context, posDrvName); else if (i->name == state.sSystem) drv.platform = state.forceStringNoCtx(*i->value, posDrvName); - else if (i->name == state.sName) - drvName = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHash") + else if (i->name == state.sOutputHash) outputHash = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHashAlgo") + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); - else if (key == "outputHashMode") + else if (i->name == state.sOutputHashMode) handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); - else if (key == "outputs") { + else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; @@ -634,14 +632,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = s; else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is '%1%'") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") handleHashMode(s); - else if (key == "outputs") + else if (i->name == state.sOutputHash) outputHash = s; + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; + else if (i->name == state.sOutputHashMode) handleHashMode(s); + else if (i->name == state.sOutputs) handleOutputs(tokenizeString<Strings>(s)); } @@ -1138,8 +1132,11 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V state.mkList(v, args[0]->attrs->size()); size_t n = 0; - for (auto & i : args[0]->attrs->lexicographicOrder()) - mkString(*(v.listElems()[n++] = state.allocValue()), i->name); + for (auto & i : *args[0]->attrs) + mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + + std::sort(v.listElems(), v.listElems() + n, + [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } @@ -1916,21 +1913,32 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar auto s = state.forceString(*args[2], context, pos); string res; - for (size_t p = 0; p < s.size(); ) { + // Loops one past last character to handle the case where 'from' contains an empty string. + for (size_t p = 0; p <= s.size(); ) { bool found = false; auto i = from.begin(); auto j = to.begin(); for (; i != from.end(); ++i, ++j) if (s.compare(p, i->size(), *i) == 0) { found = true; - p += i->size(); res += j->first; + if (i->empty()) { + if (p < s.size()) + res += s[p]; + p++; + } else { + p += i->size(); + } for (auto& path : j->second) context.insert(path); j->second.clear(); break; } - if (!found) res += s[p++]; + if (!found) { + if (p < s.size()) + res += s[p]; + p++; + } } mkString(v, res, context); @@ -1961,6 +1969,26 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a } +static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) + break; + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto & component : components) { + auto listElem = v.listElems()[n++] = state.allocValue(); + mkString(*listElem, std::move(component)); + } +} + + /************************************************************* * Networking *************************************************************/ @@ -2196,6 +2224,7 @@ void EvalState::createBaseEnv() // Versions addPrimOp("__parseDrvName", 1, prim_parseDrvName); addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); // Derivations addPrimOp("derivationStrict", 1, prim_derivationStrict); @@ -2207,7 +2236,7 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix"; + string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); sDerivationNix = symbols.create(path); evalFile(path, v); addConstant("derivation", v); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 39d23b04a5ce..31bf3f84f6c7 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -9,6 +9,9 @@ struct RegisterPrimOp { typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; static PrimOps * primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); }; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 2e3e2634db8f..9fc0d46626dd 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -79,7 +79,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, ref = "HEAD"s; } - if (!ref) ref = "master"s; + if (!ref) ref = "HEAD"s; if (rev != "" && !std::regex_match(rev, revRegex)) throw Error("invalid Git revision '%s'", rev); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 90a4867163df..7d888202bbf1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -262,6 +262,7 @@ void printVersion(const string & programName) void showManPage(const string & name) { restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), NULL); throw SysError(format("command 'man %1%' failed") % name.c_str()); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 1dcc4f0ac942..8e4861232db5 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -22,6 +22,7 @@ public: int handleExceptions(const string & programName, std::function<void()> fun); +/* Don't forget to call initPlugins() after settings are initialized! */ void initNix(); void parseCmdLine(int argc, char * * argv, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cc69ff1c74bf..1d611ffbaba5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -49,7 +49,9 @@ #include <sys/param.h> #include <sys/mount.h> #include <sys/syscall.h> +#if HAVE_SECCOMP #include <seccomp.h> +#endif #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif @@ -2471,7 +2473,7 @@ void setupSeccomp() { #if __linux__ if (!settings.filterSyscalls) return; - +#if HAVE_SECCOMP scmp_filter_ctx ctx; if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) @@ -2517,6 +2519,11 @@ void setupSeccomp() if (seccomp_load(ctx) != 0) throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif #endif } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 258d7937cc39..5ab625f42288 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -717,7 +717,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}); + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); @@ -727,7 +727,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (expectedStorePath != "" && storePath != expectedStorePath) throw nix::Error("store path mismatch in file downloaded from '%s'", url); - return storePath; + return store->toRealPath(storePath); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d3c96ddd6e61..f46e8326235f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -6,6 +6,7 @@ #include <algorithm> #include <map> #include <thread> +#include <dlfcn.h> namespace nix { @@ -37,6 +38,7 @@ Settings::Settings() , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; @@ -137,4 +139,46 @@ void MaxBuildJobsSetting::set(const std::string & str) throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } + +void initPlugins() +{ + for (const auto & pluginFile : settings.pluginFiles.get()) { + Paths pluginFiles; + try { + auto ents = readDirectory(pluginFile); + for (const auto & ent : ents) + pluginFiles.emplace_back(pluginFile + "/" + ent.name); + } catch (SysError & e) { + if (e.errNo != ENOTDIR) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ + void *handle = + dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror()); + } + } + /* We handle settings registrations here, since plugins can add settings */ + if (RegisterSetting::settingRegistrations) { + for (auto & registration : *RegisterSetting::settingRegistrations) + settings.addSetting(registration); + delete RegisterSetting::settingRegistrations; + } + settings.handleUnknownSettings(); +} + +RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations; + +RegisterSetting::RegisterSetting(AbstractSetting * s) +{ + if (!settingRegistrations) + settingRegistrations = new SettingRegistrations; + settingRegistrations->emplace_back(s); +} + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 20ac8fe4e9ae..dd01f832df0c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -82,6 +82,9 @@ public: /* The directory where the main programs are stored. */ Path nixBinDir; + /* The directory where the man pages are stored. */ + Path nixManDir; + /* File name of the socket the daemon listens to. */ Path nixDaemonSocketFile; @@ -367,14 +370,28 @@ public: Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<Paths> pluginFiles{this, {}, "plugin-files", + "Plugins to dynamically load at nix initialization time."}; }; // FIXME: don't use a global variable. extern Settings settings; +/* This should be called after settings are initialized, but before + anything else */ +void initPlugins(); + extern const string nixVersion; +struct RegisterSetting +{ + typedef std::vector<AbstractSetting *> SettingRegistrations; + static SettingRegistrations * settingRegistrations; + RegisterSetting(AbstractSetting * s); +}; + } diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 50c46ce6fe99..e11efa5c2b54 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +ifneq ($(OS), FreeBSD) + libstore_LDFLAGS += -ldl +endif libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb @@ -22,7 +25,7 @@ ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket endif -ifeq ($(OS), Linux) +ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif @@ -35,6 +38,7 @@ libstore_CXXFLAGS = \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ + -DNIX_MAN_DIR=\"$(mandir)\" \ -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ -DLSOF=\"$(lsof)\" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4d43ef082d53..8830edcc3449 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -839,7 +839,7 @@ ref<Store> openStore(const std::string & uri_, for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { - store->warnUnknownSettings(); + store->handleUnknownSettings(); return ref<Store>(store); } } diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ed15761b32a2..470c925ed7a6 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,7 @@ #include "compression.hh" #include "util.hh" #include "finally.hh" +#include "logging.hh" #include <lzma.h> #include <bzlib.h> @@ -189,28 +190,9 @@ struct XzSink : CompressionSink lzma_stream strm = LZMA_STREAM_INIT; bool finished = false; - XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink) - { - lzma_ret ret; - if (parallel) { - lzma_mt mt_options = {}; - mt_options.flags = 0; - mt_options.timeout = 300; // Using the same setting as the xz cmd line - mt_options.preset = LZMA_PRESET_DEFAULT; - mt_options.filters = NULL; - mt_options.check = LZMA_CHECK_CRC64; - mt_options.threads = lzma_cputhreads(); - mt_options.block_size = 0; - if (mt_options.threads == 0) - mt_options.threads = 1; - // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the - // number of threads. - ret = lzma_stream_encoder_mt( - &strm, &mt_options); - } else - ret = lzma_easy_encoder( - &strm, 6, LZMA_CHECK_CRC64); - + template <typename F> + XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) { + lzma_ret ret = initEncoder(); if (ret != LZMA_OK) throw CompressionError("unable to initialise lzma encoder"); // FIXME: apply the x86 BCJ filter? @@ -218,6 +200,9 @@ struct XzSink : CompressionSink strm.next_out = outbuf; strm.avail_out = sizeof(outbuf); } + XzSink(Sink & nextSink) : XzSink(nextSink, [this]() { + return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + }) {} ~XzSink() { @@ -271,6 +256,27 @@ struct XzSink : CompressionSink } }; +#ifdef HAVE_LZMA_MT +struct ParallelXzSink : public XzSink +{ + ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() { + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + return lzma_stream_encoder_mt(&strm, &mt_options); + }) {} +}; +#endif + struct BzipSink : CompressionSink { Sink & nextSink; @@ -469,10 +475,18 @@ struct BrotliSink : CompressionSink ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) { + if (parallel) { +#ifdef HAVE_LZMA_MT + if (method == "xz") + return make_ref<ParallelXzSink>(nextSink); +#endif + printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method); + } + if (method == "none") return make_ref<NoneSink>(nextSink); else if (method == "xz") - return make_ref<XzSink>(nextSink, parallel); + return make_ref<XzSink>(nextSink); else if (method == "bzip2") return make_ref<BzipSink>(nextSink); else if (method == "br") diff --git a/src/libutil/config.cc b/src/libutil/config.cc index d46ca65a3863..ce6858f0d65a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -7,10 +7,12 @@ namespace nix { void Config::set(const std::string & name, const std::string & value) { auto i = _settings.find(name); - if (i == _settings.end()) - throw UsageError("unknown setting '%s'", name); - i->second.setting->set(value); - i->second.setting->overriden = true; + if (i == _settings.end()) { + extras.emplace(name, value); + } else { + i->second.setting->set(value); + i->second.setting->overriden = true; + } } void Config::addSetting(AbstractSetting * setting) @@ -21,34 +23,34 @@ void Config::addSetting(AbstractSetting * setting) bool set = false; - auto i = initials.find(setting->name); - if (i != initials.end()) { + auto i = extras.find(setting->name); + if (i != extras.end()) { setting->set(i->second); setting->overriden = true; - initials.erase(i); + extras.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = initials.find(alias); - if (i != initials.end()) { + auto i = extras.find(alias); + if (i != extras.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { setting->set(i->second); setting->overriden = true; - initials.erase(i); + extras.erase(i); set = true; } } } } -void Config::warnUnknownSettings() +void Config::handleUnknownSettings() { - for (auto & i : initials) - warn("unknown setting '%s'", i.first); + for (auto & s : extras) + warn("unknown setting '%s'", s.first); } StringMap Config::getSettings(bool overridenOnly) @@ -60,7 +62,7 @@ StringMap Config::getSettings(bool overridenOnly) return res; } -void Config::applyConfigFile(const Path & path, bool fatal) +void Config::applyConfigFile(const Path & path) { try { string contents = readFile(path); @@ -80,7 +82,31 @@ void Config::applyConfigFile(const Path & path, bool fatal) vector<string> tokens = tokenizeString<vector<string> >(line); if (tokens.empty()) continue; - if (tokens.size() < 2 || tokens[1] != "=") + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); string name = tokens[0]; @@ -88,12 +114,7 @@ void Config::applyConfigFile(const Path & path, bool fatal) vector<string>::iterator i = tokens.begin(); advance(i, 2); - try { - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - } catch (UsageError & e) { - if (fatal) throw; - warn("in configuration file '%s': %s", path, e.what()); - } + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow }; } catch (SysError &) { } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 9a32af528ec7..d2e7faf17434 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -48,25 +48,25 @@ private: Settings _settings; - StringMap initials; + StringMap extras; public: Config(const StringMap & initials) - : initials(initials) + : extras(initials) { } void set(const std::string & name, const std::string & value); void addSetting(AbstractSetting * setting); - void warnUnknownSettings(); + void handleUnknownSettings(); StringMap getSettings(bool overridenOnly = false); const Settings & _getSettings() { return _settings; } - void applyConfigFile(const Path & path, bool fatal = false); + void applyConfigFile(const Path & path); void resetOverriden(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 11e3c9dca58a..75e4767550f7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -189,7 +189,8 @@ Hash::Hash(const std::string & s, HashType type) else if (size == base64Len()) { auto d = base64Decode(std::string(s, pos)); - assert(d.size() == hashSize); + if (d.size() != hashSize) + throw BadHash("invalid base-64 hash '%s'", s); memcpy(hash, d.data(), hashSize); } diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index e0ec66c01803..5ee0b88ef50f 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -21,13 +21,29 @@ public: MonitorFdHup(int fd) { thread = std::thread([fd]() { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - fds[0].events = 0; - if (poll(fds, 1, -1) == -1) abort(); // can't happen - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } }); }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f7a12d21b244..2391e14a94bd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -73,6 +73,13 @@ std::map<std::string, std::string> getEnv() } +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + + Path absPath(Path path, Path dir) { if (path[0] != '/') { @@ -1216,7 +1223,7 @@ std::string filterANSIEscapes(const std::string & s, unsigned int width) else if (*i == '\r') // do nothing for now - ; + i++; else { t += *i++; w++; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 47e02bc898a6..c5c537ee63d8 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -32,6 +32,9 @@ string getEnv(const string & key, const string & def = ""); /* Get the entire environment. */ std::map<std::string, std::string> getEnv(); +/* Clear the environment. */ +void clearEnv(); + /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 1581c282c75c..99f773451ffe 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -232,6 +232,8 @@ void mainWrapped(int argc, char * * argv) myArgs.parseCmdline(args); + initPlugins(); + if (packages && fromArgs) throw UsageError("'-p' and '-E' are mutually exclusive"); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 370f216abccd..ec9a7174ecb9 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -213,6 +213,9 @@ int main(int argc, char ** argv) } return true; }); + + initPlugins(); + switch (cmd) { case cNone: throw UsageError("no command specified"); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index cc663a96924d..37fe22f48134 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -77,6 +77,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + auto profilesDir = settings.nixStateDir + "/profiles"; if (removeOld) removeOldGenerations(profilesDir); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 861fc2e5cd64..dfb1b8fc5dc4 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -44,6 +44,8 @@ int main(int argc, char ** argv) return true; }); + initPlugins(); + if (sshHost.empty()) throw UsageError("no host name specified"); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index d3a8ebbdda50..890bffa19aa5 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -1060,6 +1060,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (stdio) { if (getStoreType() == tDaemon) { /* Forward on this connection to the real daemon */ diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 016caf6d2346..97e66cbd937e 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1393,6 +1393,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index e05040a42deb..dd262bea0918 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -151,6 +151,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (evalOnly && !wantsReadWrite) settings.readOnlyMode = true; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index fef3eaa45538..fa7ee254500c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -89,6 +89,8 @@ int main(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (args.size() > 2) throw UsageError("too many arguments"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4bea1316134e..e1e27ceef94d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1054,6 +1054,8 @@ int main(int argc, char * * argv) return true; }); + initPlugins(); + if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/build.cc b/src/nix/build.cc index b4f21b32d78f..b329ac38ac2b 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,6 +52,8 @@ struct CmdBuild : MixDryRun, InstallablesCommand { auto buildables = build(store, dryRun ? DryRun : Build, installables); + if (dryRun) return; + for (size_t i = 0; i < buildables.size(); ++i) { auto & b(buildables[i]); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 2ddea9e70a6a..f29429c1ac49 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -57,15 +57,15 @@ struct CmdCopy : StorePathsCommand return { Example{ "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache -r $(type -p firefox)" + "nix copy --to file:///tmp/cache $(type -p firefox)" }, Example{ "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server -r /run/current-system" + "nix copy --to ssh://server /run/current-system" }, Example{ "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server -r /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" + "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" }, }; } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 69620595d8ca..e99622faf472 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -90,6 +90,16 @@ struct CmdLsStore : StoreCommand, MixLs expectArg("path", &path); } + Examples examples() override + { + return { + Example{ + "To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + }, + }; + } + std::string name() override { return "ls-store"; @@ -116,6 +126,16 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } + Examples examples() override + { + return { + Example{ + "To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello" + }, + }; + } + std::string name() override { return "ls-nar"; diff --git a/src/nix/main.cc b/src/nix/main.cc index 8f6bbe8f51ae..bb107ec7d3f6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -92,6 +92,8 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); + initPlugins(); + if (!args.command) args.showHelpAndExit(); Finally f([]() { stopProgressBar(); }); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc new file mode 100644 index 000000000000..310942574a2a --- /dev/null +++ b/src/nix/ping-store.cc @@ -0,0 +1,35 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdPingStore : StoreCommand +{ + std::string name() override + { + return "ping-store"; + } + + std::string description() override + { + return "test whether a store can be opened"; + } + + Examples examples() override + { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1" + }, + }; + } + + void run(ref<Store> store) override + { + store->connect(); + } +}; + +static RegisterCommand r1(make_ref<CmdPingStore>()); diff --git a/src/nix/run.cc b/src/nix/run.cc index ade87e63a49c..d04e106e037b 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -16,8 +16,6 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -extern char * * environ; - struct CmdRun : InstallablesCommand { std::vector<std::string> command = { "bash" }; @@ -85,6 +83,10 @@ struct CmdRun : InstallablesCommand "To run GNU Hello:", "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" }, + Example{ + "To run GNU Hello in a chroot store:", + "nix run --store ~/my-nix nixpkgs.hello -c hello" + }, }; } @@ -105,7 +107,7 @@ struct CmdRun : InstallablesCommand if (s) kept[var] = s; } - environ = nullptr; + clearEnv(); for (auto & var : kept) setenv(var.first.c_str(), var.second.c_str(), 1); |