diff options
Diffstat (limited to 'src')
30 files changed, 931 insertions, 626 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1ac9711a1103..2ce20882da17 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -5,6 +5,9 @@ #include <memory> #include <tuple> #include <iomanip> +#if __APPLE__ +#include <sys/time.h> +#endif #include "shared.hh" #include "pathlocks.hh" @@ -64,7 +67,8 @@ public: enabled{true} {}; };; -static std::vector<machine> read_conf() { +static std::vector<machine> read_conf() +{ auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines"); auto machines = std::vector<machine>{}; @@ -104,8 +108,9 @@ static std::vector<machine> read_conf() { static string currentLoad; -static int openSlotLock(const machine & m, unsigned long long slot) { - auto fn_stream = std::stringstream(currentLoad, std::ios_base::ate | std::ios_base::out); +static AutoCloseFD openSlotLock(const machine & m, unsigned long long slot) +{ + std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out); fn_stream << "/"; for (auto t : m.systemTypes) { fn_stream << t << "-"; @@ -181,7 +186,7 @@ int main (int argc, char * * argv) AutoCloseFD free; unsigned long long load = 0; for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { - AutoCloseFD slotLock = openSlotLock(m, slot); + auto slotLock = openSlotLock(m, slot); if (lockFile(slotLock.get(), ltWrite, false)) { if (!free) { free = std::move(slotLock); @@ -224,7 +229,11 @@ int main (int argc, char * * argv) break; } +#if __APPLE__ + futimes(bestSlotLock.get(), NULL); +#else futimens(bestSlotLock.get(), NULL); +#endif lock = -1; diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc deleted file mode 100644 index ff28a60ff4b0..000000000000 --- a/src/download-via-ssh/download-via-ssh.cc +++ /dev/null @@ -1,141 +0,0 @@ -#include "shared.hh" -#include "util.hh" -#include "serialise.hh" -#include "archive.hh" -#include "affinity.hh" -#include "globals.hh" -#include "serve-protocol.hh" -#include "worker-protocol.hh" -#include "store-api.hh" - -#include <iostream> -#include <cstdlib> -#include <unistd.h> - -using namespace nix; - -// !!! TODO: -// * Respect more than the first host -// * use a database -// * show progress - - -static std::pair<FdSink, FdSource> connect(const string & conn) -{ - Pipe to, from; - to.create(); - from.create(); - startProcess([&]() { - if (dup2(to.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(from.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - execlp("ssh", "ssh", "-x", "-T", conn.c_str(), "nix-store --serve", NULL); - throw SysError("executing ssh"); - }); - // If child exits unexpectedly, we'll EPIPE or EOF early. - // If we exit unexpectedly, child will EPIPE or EOF early. - // So no need to keep track of it. - - return std::pair<FdSink, FdSource>(to.writeSide.borrow(), from.readSide.borrow()); -} - - -static void substitute(std::pair<FdSink, FdSource> & pipes, Path storePath, Path destPath) -{ - pipes.first << cmdDumpStorePath << storePath; - pipes.first.flush(); - restorePath(destPath, pipes.second); - std::cout << std::endl; -} - - -static void query(std::pair<FdSink, FdSource> & pipes) -{ - for (string line; getline(std::cin, line);) { - Strings tokenized = tokenizeString<Strings>(line); - string cmd = tokenized.front(); - tokenized.pop_front(); - if (cmd == "have") { - pipes.first - << cmdQueryValidPaths - << 0 // don't lock - << 0 // don't substitute - << tokenized; - pipes.first.flush(); - PathSet paths = readStrings<PathSet>(pipes.second); - for (auto & i : paths) - std::cout << i << std::endl; - } else if (cmd == "info") { - pipes.first << cmdQueryPathInfos << tokenized; - pipes.first.flush(); - while (1) { - Path path = readString(pipes.second); - if (path.empty()) break; - assertStorePath(path); - std::cout << path << std::endl; - string deriver = readString(pipes.second); - if (!deriver.empty()) assertStorePath(deriver); - std::cout << deriver << std::endl; - PathSet references = readStorePaths<PathSet>(pipes.second); - std::cout << references.size() << std::endl; - for (auto & i : references) - std::cout << i << std::endl; - std::cout << readLongLong(pipes.second) << std::endl; - std::cout << readLongLong(pipes.second) << std::endl; - } - } else - throw Error(format("unknown substituter query ‘%1%’") % cmd); - std::cout << std::endl; - } -} - - -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - if (argc < 2) - throw UsageError("download-via-ssh requires an argument"); - - initNix(); - - settings.update(); - - if (settings.sshSubstituterHosts.empty()) - return; - - std::cout << std::endl; - - /* Pass on the location of the daemon client's SSH - authentication socket. */ - string sshAuthSock = settings.get("ssh-auth-sock", string("")); - if (sshAuthSock != "") setenv("SSH_AUTH_SOCK", sshAuthSock.c_str(), 1); - - string host = settings.sshSubstituterHosts.front(); - std::pair<FdSink, FdSource> pipes = connect(host); - - /* Exchange the greeting */ - pipes.first << SERVE_MAGIC_1; - pipes.first.flush(); - unsigned int magic = readInt(pipes.second); - if (magic != SERVE_MAGIC_2) - throw Error("protocol mismatch"); - readInt(pipes.second); // Server version, unused for now - pipes.first << SERVE_PROTOCOL_VERSION; - pipes.first.flush(); - - string arg = argv[1]; - if (arg == "--query") - query(pipes); - else if (arg == "--substitute") { - if (argc != 4) - throw UsageError("download-via-ssh: --substitute takes exactly two arguments"); - Path storePath = argv[2]; - Path destPath = argv[3]; - printError(format("downloading ‘%1%’ via SSH from ‘%2%’...") % storePath % host); - substitute(pipes, storePath, destPath); - } - else - throw UsageError(format("download-via-ssh: unknown command ‘%1%’") % arg); - }); -} diff --git a/src/download-via-ssh/local.mk b/src/download-via-ssh/local.mk deleted file mode 100644 index 80f4c385acb3..000000000000 --- a/src/download-via-ssh/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += download-via-ssh - -download-via-ssh_DIR := $(d) - -download-via-ssh_SOURCES := $(d)/download-via-ssh.cc - -download-via-ssh_INSTALL_DIR := $(libexecdir)/nix/substituters - -download-via-ssh_CXXFLAGS = -Isrc/nix-store - -download-via-ssh_LIBS = libmain libstore libutil libformat diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7cf6a9c58086..e1fc2bf6d796 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -75,6 +75,19 @@ public: size_t capacity() { return capacity_; } + /* Returns the attributes in lexicographically sorted order. */ + std::vector<const Attr *> lexicographicOrder() const + { + std::vector<const Attr *> res; + res.reserve(size_); + for (size_t n = 0; n < size_; n++) + res.emplace_back(&attrs[n]); + std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { + return (string) a->name < (string) b->name; + }); + return res; + } + friend class EvalState; }; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 64f3874db614..d418ab4e43aa 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -91,13 +91,9 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con break; case tAttrs: { str << "{ "; - typedef std::map<string, Value *> Sorted; - Sorted sorted; - for (auto & i : *v.attrs) - sorted[i.name] = i.value; - for (auto & i : sorted) { - str << i.first << " = "; - printValue(str, active, *i.second); + for (auto & i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, active, *i->value); str << "; "; } str << "}"; @@ -295,6 +291,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sToString(symbols.create("__toString")) , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) + , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sBuilder(symbols.create("builder")) , store(store) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 195cb0db3acc..46d5a1cc866c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -68,7 +68,7 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong; + sRight, sWrong, sStructuredAttrs, sBuilder; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index dc5def911ca0..5342739c53c4 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -284,25 +284,19 @@ static void getDerivations(EvalState & state, Value & vIn, there are names clashes between derivations, the derivation bound to the attribute with the "lower" name should take precedence). */ - typedef std::map<string, Symbol> SortedSymbols; - SortedSymbols attrs; - for (auto & i : *v.attrs) - attrs.insert(std::pair<string, Symbol>(i.name, i.name)); - - for (auto & i : attrs) { - Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); - string pathPrefix2 = addToPath(pathPrefix, i.first); - Value & v2(*v.attrs->find(i.second)->value); + for (auto & i : v.attrs->lexicographicOrder()) { + Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i->name); + string pathPrefix2 = addToPath(pathPrefix, i->name); if (combineChannels) - getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, v2, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (v2.type == tAttrs) { - Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations")); - if (j != v2.attrs->end() && state.forceBool(*j->value, *j->pos)) - getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + if (i->value->type == tAttrs) { + Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); + if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5be61c647496..5a570cefb2fa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -8,6 +8,7 @@ #include "names.hh" #include "store-api.hh" #include "util.hh" +#include "json.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" @@ -474,6 +475,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * throw; } + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr<JSONObject> jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + jsonObject = std::make_unique<JSONObject>(jsonBuf); + /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; attr = args[0]->attrs->find(state.sIgnoreNulls); @@ -491,24 +499,48 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * StringSet outputs; outputs.insert("out"); - for (auto & i : *args[0]->attrs) { - if (i.name == state.sIgnoreNulls) continue; - string key = i.name; + for (auto & i : args[0]->attrs->lexicographicOrder()) { + if (i->name == state.sIgnoreNulls) continue; + string key = i->name; Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); + auto handleHashMode = [&](const std::string & s) { + if (s == "recursive") outputHashRecursive = true; + else if (s == "flat") outputHashRecursive = false; + else throw EvalError("invalid value ‘%s’ for ‘outputHashMode’ attribute, at %s", s, posDrvName); + }; + + auto handleOutputs = [&](const Strings & ss) { + outputs.clear(); + for (auto & j : ss) { + if (outputs.find(j) != outputs.end()) + throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName); + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") + throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName); + outputs.insert(j); + } + if (outputs.empty()) + throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + }; + try { if (ignoreNulls) { - state.forceValue(*i.value); - if (i.value->type == tNull) continue; + state.forceValue(*i->value); + if (i->value->type == tNull) continue; } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ if (key == "args") { - state.forceList(*i.value, pos); - for (unsigned int n = 0; n < i.value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i.value->listElems()[n], context, true); + state.forceList(*i->value, pos); + for (unsigned int n = 0; n < i->value->listSize(); ++n) { + string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); drv.args.push_back(s); } } @@ -516,39 +548,51 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* All other attributes are passed to the builder through the environment. */ else { - string s = state.coerceToString(posDrvName, *i.value, context, true); - drv.env[key] = s; - if (key == "builder") drv.builder = s; - else if (i.name == state.sSystem) drv.platform = s; - else if (i.name == state.sName) { - drvName = s; - printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName); - } - else if (key == "outputHash") outputHash = s; - else if (key == "outputHashAlgo") outputHashAlgo = s; - else if (key == "outputHashMode") { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; - else throw EvalError(format("invalid value ‘%1%’ for ‘outputHashMode’ attribute, at %2%") % s % posDrvName); - } - else if (key == "outputs") { - Strings tmp = tokenizeString<Strings>(s); - outputs.clear(); - for (auto & j : tmp) { - if (outputs.find(j) != outputs.end()) - throw EvalError(format("duplicate derivation output ‘%1%’, at %2%") % j % posDrvName); - /* !!! Check whether j is a valid attribute - name. */ - /* Derivations cannot be named ‘drv’, because - then we'd have an attribute ‘drvPath’ in - the resulting set. */ - if (j == "drv") - throw EvalError(format("invalid derivation output name ‘drv’, at %1%") % posDrvName); - outputs.insert(j); + + if (jsonObject) { + + if (i->name == state.sStructuredAttrs) continue; + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i->value, placeholder, context); + + if (i->name == state.sBuilder) + drv.builder = state.forceString(*i->value, context, posDrvName); + else if (i->name == state.sSystem) + drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sName) + drvName = state.forceStringNoCtx(*i->value, posDrvName); + else if (key == "outputHash") + outputHash = state.forceStringNoCtx(*i->value, posDrvName); + else if (key == "outputHashAlgo") + outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + else if (key == "outputHashMode") + handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + else if (key == "outputs") { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i->value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i->value->listSize(); ++n) + ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + handleOutputs(ss); + } + + } else { + auto s = state.coerceToString(posDrvName, *i->value, context, true); + drv.env.emplace(key, s); + if (i->name == state.sBuilder) drv.builder = s; + else if (i->name == state.sSystem) drv.platform = s; + else if (i->name == state.sName) { + drvName = s; + printMsg(lvlVomit, format("derivation name is ‘%1%’") % drvName); } - if (outputs.empty()) - throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + else if (key == "outputHash") outputHash = s; + else if (key == "outputHashAlgo") outputHashAlgo = s; + else if (key == "outputHashMode") handleHashMode(s); + else if (key == "outputs") + handleOutputs(tokenizeString<Strings>(s)); } + } } catch (Error & e) { @@ -558,6 +602,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ @@ -779,10 +828,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va string s = readFile(state.checkSourcePath(path)); if (s.find((char) 0) != string::npos) throw Error(format("the contents of the file ‘%1%’ cannot be represented as a Nix string") % path); - context = state.store->isInStore(path) ? - state.store->queryPathInfo(state.store->toStorePath(path))->references : - PathSet{}; - mkString(v, s.c_str(), context); + mkString(v, s.c_str()); } @@ -1001,12 +1047,9 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V state.mkList(v, args[0]->attrs->size()); - unsigned int n = 0; - for (auto & i : *args[0]->attrs) - mkString(*(v.listElems()[n++] = state.allocValue()), i.name); - - std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + size_t n = 0; + for (auto & i : args[0]->attrs->lexicographicOrder()) + mkString(*(v.listElems()[n++] = state.allocValue()), i->name); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 12f083c7f794..52cb2312826b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -97,6 +97,9 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) } +static void sigHandler(int signo) { } + + void initNix() { /* Turn on buffering for cerr. */ @@ -116,20 +119,18 @@ void initNix() startSignalHandlerThread(); - /* Ignore SIGPIPE. */ + /* Reset SIGCHLD to its default. */ struct sigaction act; sigemptyset(&act.sa_mask); - act.sa_handler = SIG_IGN; - act.sa_flags = 0; - if (sigaction(SIGPIPE, &act, 0)) - throw SysError("ignoring SIGPIPE"); - - /* Reset SIGCHLD to its default. */ act.sa_handler = SIG_DFL; act.sa_flags = 0; if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); + /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + act.sa_handler = sigHandler; + if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); @@ -245,7 +246,7 @@ void printVersion(const string & programName) void showManPage(const string & name) { - restoreSIGPIPE(); + restoreSignals(); execlp("man", "man", name.c_str(), NULL); throw SysError(format("command ‘man %1%’ failed") % name.c_str()); } @@ -253,6 +254,8 @@ void showManPage(const string & name) int handleExceptions(const string & programName, std::function<void()> fun) { + ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + string error = ANSI_RED "error:" ANSI_NORMAL " "; try { try { @@ -296,16 +299,6 @@ RunPager::RunPager() if (!pager) pager = getenv("PAGER"); if (pager && ((string) pager == "" || (string) pager == "cat")) return; - /* Ignore SIGINT. The pager will handle it (and we'll get - SIGPIPE). */ - struct sigaction act; - act.sa_handler = SIG_IGN; - act.sa_flags = 0; - sigemptyset(&act.sa_mask); - if (sigaction(SIGINT, &act, 0)) throw SysError("ignoring SIGINT"); - - restoreSIGPIPE(); - Pipe toPager; toPager.create(); @@ -314,6 +307,7 @@ RunPager::RunPager() throw SysError("dupping stdin"); if (!getenv("LESS")) setenv("LESS", "FRSXMK", 1); + restoreSignals(); if (pager) execl("/bin/sh", "sh", "-c", pager, NULL); execlp("pager", "pager", NULL); @@ -322,6 +316,8 @@ RunPager::RunPager() throw SysError(format("executing ‘%1%’") % pager); }); + pid.setKillSignal(SIGINT); + if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); } @@ -336,7 +332,11 @@ RunPager::~RunPager() pid.wait(); } } catch (...) { - ignoreException(); + try { + pid.kill(true); + } catch (...) { + ignoreException(); + } } } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 31878bbb2476..a70d50d4949c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -71,9 +71,6 @@ public: PathSet & referrers) override { notImpl(); } - PathSet queryValidDerivers(const Path & path) override - { return {}; } - PathSet queryDerivationOutputs(const Path & path) override { notImpl(); } @@ -83,13 +80,6 @@ public: Path queryPathFromHashPart(const string & hashPart) override { notImpl(); } - PathSet querySubstitutablePaths(const PathSet & paths) override - { return {}; } - - void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) override - { } - bool wantMassQuery() override { return wantMassQuery_; } void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, @@ -121,25 +111,14 @@ public: void addIndirectRoot(const Path & path) override { notImpl(); } - void syncWithGC() override - { } - Roots findRoots() override { notImpl(); } void collectGarbage(const GCOptions & options, GCResults & results) override { notImpl(); } - void optimiseStore() override - { } - - bool verifyStore(bool checkContents, bool repair) override - { return true; } - ref<FSAccessor> getFSAccessor() override; -public: - void addSignatures(const Path & storePath, const StringSet & sigs) override { notImpl(); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 7fc6ff0df0f8..5d6fff4e349f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -10,6 +10,7 @@ #include "builtins.hh" #include "finally.hh" #include "compression.hh" +#include "json.hh" #include <algorithm> #include <iostream> @@ -399,6 +400,8 @@ void Goal::trace(const format & f) /* Common initialisation performed in child processes. */ static void commonChildInit(Pipe & logPipe) { + restoreSignals(); + /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning that e.g. ssh cannot open /dev/tty) and it doesn't receive @@ -434,22 +437,20 @@ private: close that file again (without closing the original file descriptor), we lose the lock. So we have to be *very* careful not to open a lock file on which we are holding a lock. */ - static PathSet lockedPaths; /* !!! not thread-safe */ + static Sync<PathSet> lockedPaths_; Path fnUserLock; AutoCloseFD fdUserLock; string user; - uid_t uid = 0; - gid_t gid = 0; + uid_t uid; + gid_t gid; std::vector<gid_t> supplementaryGIDs; public: + UserLock(); ~UserLock(); - void acquire(); - void release(); - void kill(); string getUser() { return user; } @@ -462,19 +463,11 @@ public: }; -PathSet UserLock::lockedPaths; +Sync<PathSet> UserLock::lockedPaths_; -UserLock::~UserLock() +UserLock::UserLock() { - release(); -} - - -void UserLock::acquire() -{ - assert(uid == 0); - assert(settings.buildUsersGroup != ""); /* Get the members of the build-users-group. */ @@ -509,39 +502,48 @@ void UserLock::acquire() fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - if (lockedPaths.find(fnUserLock) != lockedPaths.end()) - /* We already have a lock on this one. */ - continue; + { + auto lockedPaths(lockedPaths_.lock()); + if (lockedPaths->count(fnUserLock)) + /* We already have a lock on this one. */ + continue; + lockedPaths->insert(fnUserLock); + } - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError(format("opening user lock ‘%1%’") % fnUserLock); + try { - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - lockedPaths.insert(fnUserLock); - user = i; - uid = pw->pw_uid; + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError(format("opening user lock ‘%1%’") % fnUserLock); - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of ‘%1%’") - % settings.buildUsersGroup); + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + user = i; + uid = pw->pw_uid; + + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of ‘%1%’") + % settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name); - - supplementaryGIDs.resize(ngroups); + /* Get the list of supplementary groups of this build user. This + is usually either empty or contains a group such as "kvm". */ + supplementaryGIDs.resize(10); + int ngroups = supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name); + + supplementaryGIDs.resize(ngroups); #endif - return; + return; + } + + } catch (...) { + lockedPaths_.lock()->erase(fnUserLock); } } @@ -551,20 +553,16 @@ void UserLock::acquire() } -void UserLock::release() +UserLock::~UserLock() { - if (uid == 0) return; - fdUserLock = -1; /* releases lock */ - assert(lockedPaths.find(fnUserLock) != lockedPaths.end()); - lockedPaths.erase(fnUserLock); - fnUserLock = ""; - uid = 0; + auto lockedPaths(lockedPaths_.lock()); + assert(lockedPaths->count(fnUserLock)); + lockedPaths->erase(fnUserLock); } void UserLock::kill() { - assert(enabled()); killUser(uid); } @@ -720,7 +718,7 @@ private: PathSet missingPaths; /* User selected for running the builder. */ - UserLock buildUser; + std::unique_ptr<UserLock> buildUser; /* The process ID of the builder. */ Pid pid; @@ -780,6 +778,7 @@ private: }; typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path DirsInChroot dirsInChroot; + typedef map<string, string> Environment; Environment env; @@ -817,6 +816,8 @@ private: const uid_t sandboxUid = 1000; const gid_t sandboxGid = 100; + const static Path homeDir; + public: DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -864,6 +865,18 @@ private: /* Start building a derivation. */ void startBuilder(); + /* Fill in the environment for the builder. */ + void initEnv(); + + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); + + /* Make a file owned by the builder. */ + void chownToBuilder(const Path & path); + + /* Handle the exportReferencesGraph attribute. */ + void doExportReferencesGraph(); + /* Run the builder's process. */ void runChild(); @@ -904,6 +917,9 @@ private: }; +const Path DerivationGoal::homeDir = "/homeless-shelter"; + + DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) @@ -951,7 +967,7 @@ void DerivationGoal::killChild() if (pid != -1) { worker.childTerminated(this); - if (buildUser.enabled()) { + if (buildUser) { /* If we're using a build user, then there is a tricky race condition: if we kill the build user before the child has done its setuid() to the build user uid, then @@ -959,7 +975,7 @@ void DerivationGoal::killChild() pid.wait(). So also send a conventional kill to the child. */ ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser.kill(); + buildUser->kill(); pid.wait(); } else pid.kill(); @@ -1380,7 +1396,7 @@ void DerivationGoal::tryToBuild() } catch (BuildError & e) { printError(e.msg()); outputLocks.unlock(); - buildUser.release(); + buildUser.reset(); worker.permanentFailure = true; done(BuildResult::InputRejected, e.msg()); return; @@ -1414,6 +1430,11 @@ void DerivationGoal::buildDone() { trace("build done"); + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, @@ -1443,7 +1464,7 @@ void DerivationGoal::buildDone() malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - if (buildUser.enabled()) buildUser.kill(); + if (buildUser) buildUser->kill(); bool diskFull = false; @@ -1513,7 +1534,6 @@ void DerivationGoal::buildDone() /* Repeat the build if necessary. */ if (curRound++ < nrRounds) { outputLocks.unlock(); - buildUser.release(); state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); return; @@ -1530,7 +1550,6 @@ void DerivationGoal::buildDone() if (!hook) printError(e.msg()); outputLocks.unlock(); - buildUser.release(); BuildResult::Status st = BuildResult::MiscFailure; @@ -1552,9 +1571,6 @@ void DerivationGoal::buildDone() return; } - /* Release the build user, if applicable. */ - buildUser.release(); - done(BuildResult::Built); } @@ -1669,11 +1685,7 @@ void DerivationGoal::startBuilder() additionalSandboxProfile = get(drv->env, "__sandboxProfile"); #endif - /* Are we doing a chroot build? Note that fixed-output - derivations are never done in a chroot, mainly so that - functions like fetchurl (which needs a proper /etc/resolv.conf) - work properly. Purity checking for fixed-output derivations - is somewhat pointless anyway. */ + /* Are we doing a chroot build? */ { string x = settings.get("build-use-sandbox", /* deprecated alias */ @@ -1700,31 +1712,15 @@ void DerivationGoal::startBuilder() if (worker.store.storeDir != worker.store.realStoreDir) useChroot = true; - /* Construct the environment passed to the builder. */ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - Path homeDir = "/homeless-shelter"; - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { + buildUser = std::make_unique<UserLock>(); - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } /* Create a temporary directory where the build will take place. */ @@ -1734,127 +1730,19 @@ void DerivationGoal::startBuilder() /* In a sandbox, for determinism, always use the same temporary directory. */ tmpDirInSandbox = useChroot ? canonPath("/tmp", true) + "/nix-build-" + drvName + "-0" : tmpDir; - - /* Add all bindings specified in the derivation via the - environments, except those listed in the passAsFile - attribute. Those are passed as file names pointing to - temporary files containing the contents. */ - PathSet filesToChown; - StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); - int fileNr = 0; - for (auto & i : drv->env) { - if (passAsFile.find(i.first) == passAsFile.end()) { - env[i.first] = i.second; - } else { - string fn = ".attr-" + std::to_string(fileNr++); - Path p = tmpDir + "/" + fn; - writeFile(p, i.second); - filesToChown.insert(p); - env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; - } - } - - /* For convenience, set an environment pointing to the top build - directory. */ - env["NIX_BUILD_TOP"] = tmpDirInSandbox; - - /* Also set TMPDIR and variants to point to this directory. */ - env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; - - /* Explicitly set PWD to prevent problems with chroot builds. In - particular, dietlibc cannot figure out the cwd because the - inode of the current directory doesn't appear in .. (because - getdents returns the inode of the mount point). */ - env["PWD"] = tmpDirInSandbox; - - /* Compatibility hack with Nix <= 0.7: if this is a fixed-output - derivation, tell the builder, so that for instance `fetchurl' - can skip checking the output. On older Nixes, this environment - variable won't be set, so `fetchurl' will do the check. */ - if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; - - /* *Only* if this is a fixed-output derivation, propagate the - values of the environment variables specified in the - `impureEnvVars' attribute to the builder. This allows for - instance environment variables for proxy configuration such as - `http_proxy' to be easily passed to downloaders like - `fetchurl'. Passing such environment variables from the caller - to the builder is generally impure, but the output of - fixed-output derivations is by definition pure (since we - already know the cryptographic hash of the output). */ - if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); - for (auto & i : varNames) env[i] = getEnv(i); - } + chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) inputRewrites[hashPlaceholder(output.first)] = output.second.path; - /* The `exportReferencesGraph' feature allows the references graph - to be passed to a builder. This attribute should be a list of - pairs [name1 path1 name2 path2 ...]. The references graph of - each `pathN' will be stored in a text file `nameN' in the - temporary build directory. The text files have the format used - by `nix-store --register-validity'. However, the deriver - fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); - Strings ss = tokenizeString<Strings>(s); - if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s); - for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ - - /* Check that the store path is valid. */ - Path storePath = *i++; - if (!worker.store.isInStore(storePath)) - throw BuildError(format("‘exportReferencesGraph’ contains a non-store path ‘%1%’") - % storePath); - storePath = worker.store.toStorePath(storePath); - if (!worker.store.isValidPath(storePath)) - throw BuildError(format("‘exportReferencesGraph’ contains an invalid path ‘%1%’") - % storePath); - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - PathSet paths, paths2; - worker.store.computeFSClosure(storePath, paths); - paths2 = paths; - - for (auto & j : paths2) { - if (isDerivation(j)) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputs) - worker.store.computeFSClosure(k.second.path, paths); - } - } - - /* Write closure info to `fileName'. */ - writeFile(tmpDir + "/" + fileName, - worker.store.makeValidityRegistration(paths, false, false)); - } - - - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { - buildUser.acquire(); - - /* Make sure that no other processes are executing under this - uid. */ - buildUser.kill(); - - /* Change ownership of the temporary build directory. */ - filesToChown.insert(tmpDir); + /* Construct the environment passed to the builder. */ + initEnv(); - for (auto & p : filesToChown) - if (chown(p.c_str(), buildUser.getUID(), buildUser.getGID()) == -1) - throw SysError(format("cannot change ownership of ‘%1%’") % p); - } + writeStructuredAttrs(); + /* Handle exportReferencesGraph(), if set. */ + doExportReferencesGraph(); if (useChroot) { @@ -1946,7 +1834,7 @@ void DerivationGoal::startBuilder() if (mkdir(chrootRootDir.c_str(), 0750) == -1) throw SysError(format("cannot create ‘%1%’") % chrootRootDir); - if (buildUser.enabled() && chown(chrootRootDir.c_str(), 0, buildUser.getGID()) == -1) + if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) throw SysError(format("cannot change ownership of ‘%1%’") % chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need @@ -1990,7 +1878,7 @@ void DerivationGoal::startBuilder() createDirs(chrootStoreDir); chmod_(chrootStoreDir, 01775); - if (buildUser.enabled() && chown(chrootStoreDir.c_str(), 0, buildUser.getGID()) == -1) + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) throw SysError(format("cannot change ownership of ‘%1%’") % chrootStoreDir); for (auto & i : inputPaths) { @@ -2200,8 +2088,8 @@ void DerivationGoal::startBuilder() /* Set the UID/GID mapping of the builder's user namespace such that the sandbox user maps to the build user, or to the calling user (if build users are disabled). */ - uid_t hostUid = buildUser.enabled() ? buildUser.getUID() : getuid(); - uid_t hostGid = buildUser.enabled() ? buildUser.getGID() : getgid(); + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); writeFile("/proc/" + std::to_string(pid) + "/uid_map", (format("%d %d 1") % sandboxUid % hostUid).str()); @@ -2219,7 +2107,7 @@ void DerivationGoal::startBuilder() } else #endif { - options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); + options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); }, options); @@ -2242,6 +2130,174 @@ void DerivationGoal::startBuilder() } +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + /* In non-structured mode, add all bindings specified in the + derivation via the environments, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!drv->env.count("__json")) { + + StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); + int fileNr = 0; + for (auto & i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; + writeFile(p, i.second); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); + for (auto & i : varNames) env[i] = getEnv(i); + } +} + + +void DerivationGoal::writeStructuredAttrs() +{ + auto json = drv->env.find("__json"); + if (json == drv->env.end()) return; + + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json->second, inputRewrites)); +} + + +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of ‘%1%’") % path); +} + + +void DerivationGoal::doExportReferencesGraph() +{ + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph"); + Strings ss = tokenizeString<Strings>(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in ‘exportReferencesGraph’: ‘%1%’") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + + /* Check that the store path is valid. */ + Path storePath = *i++; + if (!worker.store.isInStore(storePath)) + throw BuildError(format("‘exportReferencesGraph’ contains a non-store path ‘%1%’") + % storePath); + storePath = worker.store.toStorePath(storePath); + if (!worker.store.isValidPath(storePath)) + throw BuildError(format("‘exportReferencesGraph’ contains an invalid path ‘%1%’") + % storePath); + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths, paths2; + worker.store.computeFSClosure(storePath, paths); + paths2 = paths; + + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputs) + worker.store.computeFSClosure(k.second.path, paths); + } + } + + if (!drv->env.count("__json")) { + + /* Write closure info to <fileName>. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration(paths, false, false)); + + } else { + + /* Write a more comprehensive JSON serialisation to + <fileName>. */ + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + worker.store.pathInfoToJSON(jsonRoot, paths, false, true); + } + writeFile(tmpDir + "/" + fileName, str.str()); + + } + } +} + + void DerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite @@ -2475,22 +2531,22 @@ void DerivationGoal::runChild() descriptors except std*, so that's safe. Also note that setuid() when run as root sets the real, effective and saved UIDs. */ - if (setUser && buildUser.enabled()) { + if (setUser && buildUser) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (!buildUser.getSupplementaryGIDs().empty() && - setgroups(buildUser.getSupplementaryGIDs().size(), - buildUser.getSupplementaryGIDs().data()) == -1) + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) throw SysError("cannot set supplementary groups of build user"); - if (setgid(buildUser.getGID()) == -1 || - getgid() != buildUser.getGID() || - getegid() != buildUser.getGID()) + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) throw SysError("setgid failed"); - if (setuid(buildUser.getUID()) == -1 || - getuid() != buildUser.getUID() || - geteuid() != buildUser.getUID()) + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) throw SysError("setuid failed"); } @@ -2614,8 +2670,6 @@ void DerivationGoal::runChild() for (auto & i : drv->args) args.push_back(rewriteStrings(i, inputRewrites)); - restoreSIGPIPE(); - /* Indicate that we managed to set up the build environment. */ writeFull(STDERR_FILENO, string("\1\n")); @@ -2730,7 +2784,7 @@ void DerivationGoal::registerOutputs() build. Also, the output should be owned by the build user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || - (buildUser.enabled() && st.st_uid != buildUser.getUID())) + (buildUser && st.st_uid != buildUser->getUID())) throw BuildError(format("suspicious ownership or permission on ‘%1%’; rejecting this build output") % path); #endif @@ -2742,7 +2796,7 @@ void DerivationGoal::registerOutputs() /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or something like that. */ - canonicalisePathMetaData(actualPath, buildUser.enabled() ? buildUser.getUID() : -1, inodesSeen); + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); /* FIXME: this is in-memory. */ StringSink sink; @@ -2800,7 +2854,7 @@ void DerivationGoal::registerOutputs() /* Get rid of all weird permissions. This also checks that all files are owned by the build user, if applicable. */ canonicalisePathMetaData(actualPath, - buildUser.enabled() && !rewritten ? buildUser.getUID() : -1, inodesSeen); + buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); /* For this output path, find the references to other paths contained in it. Compute the SHA-256 NAR hash at the same diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 42873d9e8a10..074e0ca6642a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -172,6 +172,13 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); } + static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) + { + if (type == CURLINFO_TEXT) + vomit("curl: %s", chomp(std::string(data, size))); + return 0; + } + void init() { // FIXME: handle parallel downloads. @@ -184,6 +191,12 @@ struct CurlDownloader : public Downloader if (!req) req = curl_easy_init(); curl_easy_reset(req); + + if (verbosity >= lvlVomit) { + curl_easy_setopt(req, CURLOPT_VERBOSE, 1); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + } + curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); @@ -263,7 +276,7 @@ struct CurlDownloader : public Downloader code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, format("download of ‘%s’ was interrupted") % request.uri) : httpStatus != 0 - ? DownloadError(err, format("unable to download ‘%s’: HTTP error %d") % request.uri % httpStatus) + ? DownloadError(err, format("unable to download ‘%s’: HTTP error %d (curl error: %s)") % request.uri % httpStatus % curl_easy_strerror(code)) : DownloadError(err, format("unable to download ‘%s’: %s (%d)") % request.uri % curl_easy_strerror(code) % code); /* If this is a transient error, then maybe retry the @@ -387,7 +400,7 @@ struct CurlDownloader : public Downloader nextWakeup != std::chrono::steady_clock::time_point() ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count()) : 1000000000; - //printMsg(lvlVomit, format("download thread waiting for %d ms") % sleepTimeMs); + vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc new file mode 100644 index 000000000000..5d9e5aad6e0a --- /dev/null +++ b/src/libstore/legacy-ssh-store.cc @@ -0,0 +1,247 @@ +#include "archive.hh" +#include "pool.hh" +#include "remote-store.hh" +#include "serve-protocol.hh" +#include "store-api.hh" +#include "worker-protocol.hh" + +namespace nix { + +static std::string uriScheme = "legacy-ssh://"; + +struct LegacySSHStore : public Store +{ + string host; + + struct Connection + { + Pid sshPid; + AutoCloseFD out; + AutoCloseFD in; + FdSink to; + FdSource from; + }; + + AutoDelete tmpDir; + + Path socketPath; + + Pid sshMaster; + + ref<Pool<Connection>> connections; + + Path key; + + LegacySSHStore(const string & host, const Params & params, + size_t maxConnections = std::numeric_limits<size_t>::max()) + : Store(params) + , host(host) + , tmpDir(createTempDir("", "nix", true, true, 0700)) + , socketPath((Path) tmpDir + "/ssh.sock") + , connections(make_ref<Pool<Connection>>( + maxConnections, + [this]() { return openConnection(); }, + [](const ref<Connection> & r) { return true; } + )) + , key(get(params, "ssh-key", "")) + { + } + + ref<Connection> openConnection() + { + if ((pid_t) sshMaster == -1) { + sshMaster = startProcess([&]() { + restoreSignals(); + Strings args{ "ssh", "-M", "-S", socketPath, "-N", "-x", "-a", host }; + if (!key.empty()) + args.insert(args.end(), {"-i", key}); + execvp("ssh", stringsToCharPtrs(args).data()); + throw SysError("starting SSH master connection to host ‘%s’", host); + }); + } + + auto conn = make_ref<Connection>(); + Pipe in, out; + in.create(); + out.create(); + conn->sshPid = startProcess([&]() { + if (dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("duping over STDIN"); + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("duping over STDOUT"); + execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-store", "--serve", "--write", nullptr); + throw SysError("executing ‘nix-store --serve’ on remote host ‘%s’", host); + }); + in.readSide = -1; + out.writeSide = -1; + conn->out = std::move(out.readSide); + conn->in = std::move(in.writeSide); + conn->to = FdSink(conn->in.get()); + conn->from = FdSource(conn->out.get()); + + int remoteVersion; + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + unsigned int magic = readInt(conn->from); + if (magic != SERVE_MAGIC_2) + throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host); + remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host); + + } catch (EndOfFile & e) { + throw Error("cannot connect to ‘%1%’", host); + } + + return conn; + }; + + string getUri() override + { + return uriScheme + host; + } + + void queryPathInfoUncached(const Path & path, + std::function<void(std::shared_ptr<ValidPathInfo>)> success, + std::function<void(std::exception_ptr exc)> failure) override + { + sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> { + auto conn(connections->get()); + + debug("querying remote host ‘%s’ for info on ‘%s’", host, path); + + conn->to << cmdQueryPathInfos << PathSet{path}; + conn->to.flush(); + + auto info = std::make_shared<ValidPathInfo>(); + conn->from >> info->path; + if (info->path.empty()) return nullptr; + assert(path == info->path); + + PathSet references; + conn->from >> info->deriver; + info->references = readStorePaths<PathSet>(*this, conn->from); + readLongLong(conn->from); // download size + info->narSize = readLongLong(conn->from); + + auto s = readString(conn->from); + assert(s == ""); + + return info; + }); + } + + void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + bool repair, bool dontCheckSigs, + std::shared_ptr<FSAccessor> accessor) override + { + debug("adding path ‘%s’ to remote host ‘%s’", info.path, host); + + auto conn(connections->get()); + + conn->to + << cmdImportPaths + << 1; + conn->to(*nar); + conn->to + << exportMagic + << info.path + << info.references + << info.deriver + << 0 + << 0; + conn->to.flush(); + + if (readInt(conn->from) != 1) + throw Error("failed to add path ‘%s’ to remote host ‘%s’, info.path, host"); + + } + + void narFromPath(const Path & path, Sink & sink) override + { + auto conn(connections->get()); + + conn->to << cmdDumpStorePath << path; + conn->to.flush(); + + /* FIXME: inefficient. */ + ParseSink parseSink; /* null sink; just parse the NAR */ + SavingSourceAdapter savedNAR(conn->from); + parseDump(parseSink, savedNAR); + sink(savedNAR.s); + } + + /* Unsupported methods. */ + [[noreturn]] void unsupported() + { + throw Error("operation not supported on SSH stores"); + } + + PathSet queryAllValidPaths() override { unsupported(); } + + void queryReferrers(const Path & path, PathSet & referrers) override + { unsupported(); } + + PathSet queryDerivationOutputs(const Path & path) override + { unsupported(); } + + StringSet queryDerivationOutputNames(const Path & path) override + { unsupported(); } + + Path queryPathFromHashPart(const string & hashPart) override + { unsupported(); } + + Path addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, + PathFilter & filter, bool repair) override + { unsupported(); } + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, bool repair) override + { unsupported(); } + + void buildPaths(const PathSet & paths, BuildMode buildMode) override + { unsupported(); } + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { unsupported(); } + + void ensurePath(const Path & path) override + { unsupported(); } + + void addTempRoot(const Path & path) override + { unsupported(); } + + void addIndirectRoot(const Path & path) override + { unsupported(); } + + Roots findRoots() override + { unsupported(); } + + void collectGarbage(const GCOptions & options, GCResults & results) override + { unsupported(); } + + ref<FSAccessor> getFSAccessor() + { unsupported(); } + + void addSignatures(const Path & storePath, const StringSet & sigs) override + { unsupported(); } + + bool isTrusted() override + { return true; } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); +}); + +} diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index d28ff42c7f23..13b67b81f35e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -36,13 +36,9 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); -create table if not exists NARExistence ( - cache integer not null, - storePath text not null, - exist integer not null, - timestamp integer not null, - primary key (cache, storePath), - foreign key (cache) references BinaryCaches(id) on delete cascade +create table if not exists LastPurge ( + dummy text primary key, + value integer ); )sql"; @@ -51,8 +47,12 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { public: - /* How long negative lookups are valid. */ + /* How long negative and positive lookups are valid. */ const int ttlNegative = 3600; + const int ttlPositive = 30 * 24 * 3600; + + /* How often to purge expired entries from the cache. */ + const int purgeInterval = 24 * 3600; struct Cache { @@ -65,7 +65,7 @@ public: struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; std::map<std::string, Cache> caches; }; @@ -103,7 +103,28 @@ public: "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->queryNAR.create(state->db, - "select * from NARs where cache = ? and hashPart = ?"); + "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + + /* Periodically purge expired entries from the database. */ + auto now = time(0); + + SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); + auto queryLastPurge_(queryLastPurge.use()); + + if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) { + SQLiteStmt(state->db, + "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") + .use() + (now - ttlNegative) + (now - ttlPositive) + .exec(); + + debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); + + SQLiteStmt(state->db, + "insert or replace into LastPurge(dummy, value) values ('', ?)") + .use()(now).exec(); + } } Cache & getCache(State & state, const std::string & uri) @@ -152,10 +173,15 @@ public: auto & cache(getCache(*state, uri)); - auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)); + auto now = time(0); + + auto queryNAR(state->queryNAR.use() + (cache.id) + (hashPart) + (now - ttlNegative) + (now - ttlPositive)); if (!queryNAR.next()) - // FIXME: check NARExistence return {oUnknown, 0}; if (!queryNAR.getInt(13)) @@ -163,8 +189,6 @@ public: auto narInfo = make_ref<NarInfo>(); - // FIXME: implement TTL. - auto namePart = queryNAR.getStr(2); narInfo->path = cache.storeDir + "/" + hashPart + (namePart.empty() ? "" : "-" + namePart); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 620c9a6b752d..bf7ad3d21851 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -13,7 +13,7 @@ namespace nix { -int openLockFile(const Path & path, bool create) +AutoCloseFD openLockFile(const Path & path, bool create) { AutoCloseFD fd; @@ -21,7 +21,7 @@ int openLockFile(const Path & path, bool create) if (!fd && (create || errno != ENOENT)) throw SysError(format("opening lock file ‘%1%’") % path); - return fd.release(); + return fd; } @@ -136,6 +136,7 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Failed to lock this path; release all other locks. */ unlock(); + lockedPaths_.lock()->erase(lockPath); return false; } } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 40103c393f64..2a7de611446e 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,6 +1,6 @@ #pragma once -#include "types.hh" +#include "util.hh" namespace nix { @@ -9,7 +9,7 @@ namespace nix { /* Open (possibly create) a lock file and return the file descriptor. -1 is returned if create is false and the lock could not be opened because it doesn't exist. Any other error throws an exception. */ -int openLockFile(const Path & path, bool create); +AutoCloseFD openLockFile(const Path & path, bool create); /* Delete an open lock file. */ void deleteLockFile(const Path & path, int fd); @@ -19,7 +19,7 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -class PathLocks +class PathLocks { private: typedef std::pair<int, Path> FDPair; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 816d95ba6075..42c09ec7e0b6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -37,6 +37,7 @@ template<class T> T readStorePaths(Store & store, Source & from) } template PathSet readStorePaths(Store & store, Source & from); +template Paths readStorePaths(Store & store, Source & from); /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params, size_t maxConnections) diff --git a/src/nix-store/serve-protocol.hh b/src/libstore/serve-protocol.hh index f8cc9a4b6ebe..f8cc9a4b6ebe 100644 --- a/src/nix-store/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 7c1ed538215c..4d347a2e56ab 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -31,6 +31,7 @@ struct SQLiteStmt sqlite3 * db = 0; sqlite3_stmt * stmt = 0; SQLiteStmt() { } + SQLiteStmt(sqlite3 * db, const std::string & s) { create(db, s); } void create(sqlite3 * db, const std::string & s); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index cce0458c69f2..6f1862afa899 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -7,11 +7,13 @@ namespace nix { +static std::string uriScheme = "ssh://"; + class SSHStore : public RemoteStore { public: - SSHStore(string uri, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max()); + SSHStore(string host, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max()); std::string getUri() override; @@ -36,19 +38,19 @@ private: Pid sshMaster; - string uri; + string host; Path key; bool compress; }; -SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) +SSHStore::SSHStore(string host, const Params & params, size_t maxConnections) : Store(params) , RemoteStore(params, maxConnections) , tmpDir(createTempDir("", "nix", true, true, 0700)) , socketPath((Path) tmpDir + "/ssh.sock") - , uri(std::move(uri)) + , host(std::move(host)) , key(get(params, "ssh-key", "")) , compress(get(params, "compress", "") == "true") { @@ -58,7 +60,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) string SSHStore::getUri() { - return "ssh://" + uri; + return uriScheme + host; } class ForwardSource : public Source @@ -93,12 +95,12 @@ ref<FSAccessor> SSHStore::getFSAccessor() ref<RemoteStore::Connection> SSHStore::openConnection() { if ((pid_t) sshMaster == -1) { - auto flags = compress ? "-NMCS" : "-NMS"; sshMaster = startProcess([&]() { + restoreSignals(); if (key.empty()) - execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL); + execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), host.c_str(), NULL); else - execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); + execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), host.c_str(), NULL); throw SysError("starting ssh master"); }); } @@ -112,7 +114,7 @@ ref<RemoteStore::Connection> SSHStore::openConnection() throw SysError("duping over STDIN"); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over STDOUT"); - execlp("ssh", "ssh", "-S", socketPath.c_str(), uri.c_str(), "nix-daemon", "--stdio", NULL); + execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-daemon", "--stdio", NULL); throw SysError("executing nix-daemon --stdio over ssh"); }); in.readSide = -1; @@ -129,8 +131,8 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> { - if (std::string(uri, 0, 6) != "ssh://") return 0; - return std::make_shared<SSHStore>(uri.substr(6), params); + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); }); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c8ca00f00694..b5934a0d1232 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -4,6 +4,7 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" +#include "json.hh" #include "derivations.hh" #include <future> @@ -285,6 +286,19 @@ bool Store::isValidPath(const Path & storePath) } +/* Default implementation for stores that only implement + queryPathInfoUncached(). */ +bool Store::isValidPathUncached(const Path & path) +{ + try { + queryPathInfo(path); + return true; + } catch (InvalidPath &) { + return false; + } +} + + ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) { std::promise<ref<ValidPathInfo>> promise; @@ -440,6 +454,64 @@ string Store::makeValidityRegistration(const PathSet & paths, } +void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, + bool includeImpureInfo, bool showClosureSize) +{ + auto jsonList = jsonOut.list(); + + for (auto storePath : storePaths) { + auto info = queryPathInfo(storePath); + storePath = info->path; + + auto jsonPath = jsonList.object(); + jsonPath + .attr("path", storePath) + .attr("narHash", info->narHash.to_string()) + .attr("narSize", info->narSize); + + { + auto jsonRefs = jsonPath.list("references"); + for (auto & ref : info->references) + jsonRefs.elem(ref); + } + + if (info->ca != "") + jsonPath.attr("ca", info->ca); + + if (showClosureSize) + jsonPath.attr("closureSize", getClosureSize(storePath)); + + if (!includeImpureInfo) continue; + + if (info->deriver != "") + jsonPath.attr("deriver", info->deriver); + + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); + + if (info->ultimate) + jsonPath.attr("ultimate", info->ultimate); + + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto & sig : info->sigs) + jsonSigs.elem(sig); + } + } +} + + +unsigned long long Store::getClosureSize(const Path & storePath) +{ + unsigned long long totalSize = 0; + PathSet closure; + computeFSClosure(storePath, closure, false, false); + for (auto & p : closure) + totalSize += queryPathInfo(p)->narSize; + return totalSize; +} + + const Store::Stats & Store::getStats() { { @@ -458,6 +530,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); + if (srcStore->isTrusted()) + dontCheckSigs = true; + + if (!info->narHash && dontCheckSigs) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->narHash = hashString(htSHA256, *sink.s); + info = info2; + } + dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 30ee433bf074..d03e70849f93 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -22,6 +22,7 @@ struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; +class JSONPlaceholder; /* Size of the hash part of store paths, in base-32 characters. */ @@ -319,7 +320,7 @@ public: protected: - virtual bool isValidPathUncached(const Path & path) = 0; + virtual bool isValidPathUncached(const Path & path); public: @@ -359,7 +360,7 @@ public: output. (Note that the result of `queryDeriver()' is the derivation that was actually used to produce `path', which may not exist anymore.) */ - virtual PathSet queryValidDerivers(const Path & path) = 0; + virtual PathSet queryValidDerivers(const Path & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ virtual PathSet queryDerivationOutputs(const Path & path) = 0; @@ -372,13 +373,13 @@ public: virtual Path queryPathFromHashPart(const string & hashPart) = 0; /* Query which of the given paths have substitutes. */ - virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; + virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const PathSet & paths, - SubstitutablePathInfos & infos) = 0; + SubstitutablePathInfos & infos) { return; }; virtual bool wantMassQuery() { return false; } @@ -453,7 +454,7 @@ public: permanent root and sees our's. In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() = 0; + virtual void syncWithGC() { }; /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink @@ -469,13 +470,26 @@ public: string makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash); + /* Write a JSON representation of store path metadata, such as the + hash and the references. If ‘includeImpureInfo’ is true, + variable elements such as the registration time are + included. If ‘showClosureSize’ is true, the closure size of + each path is included. */ + void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, + bool includeImpureInfo, bool showClosureSize); + + /* Return the size of the closure of the specified path, that is, + the sum of the size of the NAR serialisation of each path in + the closure. */ + unsigned long long getClosureSize(const Path & storePath); + /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ - virtual void optimiseStore() = 0; + virtual void optimiseStore() { }; /* Check the integrity of the Nix store. Returns true if errors remain. */ - virtual bool verifyStore(bool checkContents, bool repair) = 0; + virtual bool verifyStore(bool checkContents, bool repair) { return false; }; /* Return an object to access files in the Nix store. */ virtual ref<FSAccessor> getFSAccessor() = 0; @@ -548,6 +562,10 @@ public: const Stats & getStats(); + /* Whether this store paths from this store can be imported even + if they lack a signature. */ + virtual bool isTrusted() { return false; } + protected: Stats stats; diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 6f01ccd91a43..e0ec66c01803 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -27,8 +27,7 @@ public: fds[0].events = 0; if (poll(fds, 1, -1) == -1) abort(); // can't happen assert(fds[0].revents & POLLHUP); - /* We got POLLHUP, so send an INT signal to the main thread. */ - kill(getpid(), SIGINT); + triggerInterrupt(); }); }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index f12f02543bc0..5646d08c1314 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -139,6 +139,21 @@ struct StringSource : Source }; +/* Adapter class of a Source that saves all data read to `s'. */ +struct SavingSourceAdapter : Source +{ + Source & orig; + string s; + SavingSourceAdapter(Source & orig) : orig(orig) { } + size_t read(unsigned char * data, size_t len) + { + size_t n = orig.read(data, len); + s.append((const char *) data, n); + return n; + } +}; + + void writePadding(size_t len, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e9457582810a..6c4c5c969d86 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -860,6 +860,8 @@ string runProgram(Path program, bool searchPath, const Strings & args, Strings args_(args); args_.push_front(program); + restoreSignals(); + if (searchPath) execvp(program.c_str(), stringsToCharPtrs(args_).data()); else @@ -909,16 +911,6 @@ void closeOnExec(int fd) } -void restoreSIGPIPE() -{ - struct sigaction act; - act.sa_handler = SIG_DFL; - act.sa_flags = 0; - sigemptyset(&act.sa_mask); - if (sigaction(SIGPIPE, &act, 0)) throw SysError("resetting SIGPIPE"); -} - - ////////////////////////////////////////////////////////////////////// @@ -1197,36 +1189,52 @@ static void signalHandlerThread(sigset_t set) int signal = 0; sigwait(&set, &signal); - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) { - _isInterrupted = 1; - - { - auto interruptCallbacks(_interruptCallbacks.lock()); - for (auto & callback : *interruptCallbacks) { - try { - callback(); - } catch (...) { - ignoreException(); - } - } + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + } +} + +void triggerInterrupt() +{ + _isInterrupted = 1; + + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto & callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); } } } } +static sigset_t savedSignalMask; + void startSignalHandlerThread() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("quering signal mask"); + sigset_t set; sigemptyset(&set); sigaddset(&set, SIGINT); sigaddset(&set, SIGTERM); sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) throw SysError("blocking signals"); std::thread(signalHandlerThread, set).detach(); } +void restoreSignals() +{ + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + /* RAII helper to automatically deregister a callback. */ struct InterruptCallbackImpl : InterruptCallback { @@ -1246,7 +1254,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> res->it = interruptCallbacks->end(); res->it--; - return res; + return std::unique_ptr<InterruptCallback>(res.release()); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b68d48582b34..cfaaf1486e9e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,10 +256,6 @@ void closeMostFDs(const set<int> & exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); -/* Restore default handling of SIGPIPE, otherwise some programs will - randomly say "Broken pipe". */ -void restoreSIGPIPE(); - /* User interruption. */ @@ -423,6 +419,9 @@ void callSuccess( on the current thread (and thus any threads created by it). */ void startSignalHandlerThread(); +/* Restore default signal handling. */ +void restoreSignals(); + struct InterruptCallback { virtual ~InterruptCallback() { }; @@ -433,5 +432,21 @@ struct InterruptCallback std::unique_ptr<InterruptCallback> createInterruptCallback( std::function<void()> callback); +void triggerInterrupt(); + +/* A RAII class that causes the current thread to receive SIGUSR1 when + the signal handler thread receives SIGINT. That is, this allows + SIGINT to be multiplexed to multiple threads. */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr<InterruptCallback> callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3eb2d2c0b7a9..ee030c57b6b3 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -452,6 +452,8 @@ int main(int argc, char ** argv) auto argPtrs = stringsToCharPtrs(args); + restoreSignals(); + execvp(getEnv("NIX_BUILD_SHELL", "bash").c_str(), argPtrs.data()); throw SysError("executing shell"); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 90a7301873c4..f3ee0afc11e7 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -168,21 +168,6 @@ struct RetrieveRegularNARSink : ParseSink }; -/* Adapter class of a Source that saves all data read to `s'. */ -struct SavingSourceAdapter : Source -{ - Source & orig; - string s; - SavingSourceAdapter(Source & orig) : orig(orig) { } - size_t read(unsigned char * data, size_t len) - { - size_t n = orig.read(data, len); - s.append((const char *) data, n); - return n; - } -}; - - static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVersion, Source & from, Sink & to, unsigned int op) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c1e6afef0e50..0aabe66c5626 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -922,9 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - Paths sorted = store->topoSortPaths(readStorePaths<PathSet>(*store, in)); - reverse(sorted.begin(), sorted.end()); - store->exportPaths(sorted, out); + store->exportPaths(readStorePaths<Paths>(*store, in), out); break; } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index a9b33e1877dd..0f9a1125f2e9 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -65,55 +65,12 @@ struct CmdPathInfo : StorePathsCommand for (auto & storePath : storePaths) pathLen = std::max(pathLen, storePath.size()); - auto getClosureSize = [&](const Path & storePath) -> unsigned long long { - unsigned long long totalSize = 0; - PathSet closure; - store->computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) - totalSize += store->queryPathInfo(p)->narSize; - return totalSize; - }; - if (json) { - JSONList jsonRoot(std::cout, true); - - for (auto storePath : storePaths) { - auto info = store->queryPathInfo(storePath); - storePath = info->path; - - auto jsonPath = jsonRoot.object(); - jsonPath - .attr("path", storePath) - .attr("narHash", info->narHash.to_string()) - .attr("narSize", info->narSize); - - if (showClosureSize) - jsonPath.attr("closureSize", getClosureSize(storePath)); - - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); - - { - auto jsonRefs = jsonPath.list("references"); - for (auto & ref : info->references) - jsonRefs.elem(ref); - } - - if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); - - if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); - - if (info->ca != "") - jsonPath.attr("ca", info->ca); - - if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); - for (auto & sig : info->sigs) - jsonSigs.elem(sig); - } - } + JSONPlaceholder jsonRoot(std::cout, true); + store->pathInfoToJSON(jsonRoot, + // FIXME: preserve order? + PathSet(storePaths.begin(), storePaths.end()), + true, showClosureSize); } else { @@ -128,7 +85,7 @@ struct CmdPathInfo : StorePathsCommand std::cout << '\t' << std::setw(11) << info->narSize; if (showClosureSize) - std::cout << '\t' << std::setw(11) << getClosureSize(storePath); + std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath); if (showSigs) { std::cout << '\t'; |