diff options
Diffstat (limited to 'src')
29 files changed, 406 insertions, 245 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index d7aee288670a..8876da6c063c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -9,6 +9,7 @@ #include <sys/time.h> #endif +#include "machines.hh" #include "shared.hh" #include "pathlocks.hh" #include "globals.hh" @@ -22,131 +23,56 @@ using std::cin; static void handleAlarm(int sig) { } -class Machine { - const std::set<string> supportedFeatures; - const std::set<string> mandatoryFeatures; - -public: - const string hostName; - const std::vector<string> systemTypes; - const string sshKey; - const unsigned int maxJobs; - const unsigned int speedFactor; - bool enabled; - - bool allSupported(const std::set<string> & features) const { - return std::all_of(features.begin(), features.end(), - [&](const string & feature) { - return supportedFeatures.count(feature) || - mandatoryFeatures.count(feature); - }); - } - - bool mandatoryMet(const std::set<string> & features) const { - return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { - return features.count(feature); - }); - } - - Machine(decltype(hostName) hostName, - decltype(systemTypes) systemTypes, - decltype(sshKey) sshKey, - decltype(maxJobs) maxJobs, - decltype(speedFactor) speedFactor, - decltype(supportedFeatures) supportedFeatures, - decltype(mandatoryFeatures) mandatoryFeatures) : - supportedFeatures(supportedFeatures), - mandatoryFeatures(mandatoryFeatures), - hostName(hostName), - systemTypes(systemTypes), - sshKey(sshKey), - maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), - enabled(true) - {}; -};; - -static std::vector<Machine> readConf() +std::string escapeUri(std::string uri) { - auto conf = getEnv("NIX_REMOTE_SYSTEMS", SYSCONFDIR "/nix/machines"); - - auto machines = std::vector<Machine>{}; - auto lines = std::vector<string>{}; - try { - lines = tokenizeString<std::vector<string>>(readFile(conf), "\n"); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - } - for (auto line : lines) { - chomp(line); - line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) { - continue; - } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 4) - throw FormatError("bad machines.conf file ‘%1%’", conf); - machines.emplace_back(tokens[0], - tokenizeString<std::vector<string>>(tokens[1], ","), - tokens[2], - stoull(tokens[3]), - sz >= 5 ? stoull(tokens[4]) : 1LL, - sz >= 6 ? - tokenizeString<std::set<string>>(tokens[5], ",") : - std::set<string>{}, - sz >= 7 ? - tokenizeString<std::set<string>>(tokens[6], ",") : - std::set<string>{}); - } - return machines; + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; } static string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) { - std::ostringstream fn_stream(currentLoad, std::ios_base::ate | std::ios_base::out); - fn_stream << "/"; - for (auto t : m.systemTypes) { - fn_stream << t << "-"; - } - fn_stream << m.hostName << "-" << slot; - return openLockFile(fn_stream.str(), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -static char display_env[] = "DISPLAY="; -static char ssh_env[] = "SSH_ASKPASS="; - int main (int argc, char * * argv) { return handleExceptions(argv[0], [&]() { initNix(); /* Ensure we don't get any SSH passphrase or host key popups. */ - if (putenv(display_env) == -1 || - putenv(ssh_env) == -1) - throw SysError("setting SSH env vars"); + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); - if (argc != 4) + if (argc != 6) throw UsageError("called without required arguments"); auto store = openStore(); auto localSystem = argv[1]; - settings.maxSilentTime = stoull(string(argv[2])); - settings.buildTimeout = stoull(string(argv[3])); + settings.maxSilentTime = std::stoll(argv[2]); + settings.buildTimeout = std::stoll(argv[3]); + verbosity = (Verbosity) std::stoll(argv[4]); + settings.builders = argv[5]; - currentLoad = getEnv("NIX_CURRENT_LOAD", "/run/nix/current-load"); + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on OS X. */ + currentLoad = settings.nixStateDir + "/current-load"; std::shared_ptr<Store> sshStore; AutoCloseFD bestSlotLock; - auto machines = readConf(); + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return; + } + string drvPath; - string hostName; + string storeUri; for (string line; getline(cin, line);) { auto tokens = tokenizeString<std::vector<string>>(line); auto sz = tokens.size(); @@ -173,6 +99,8 @@ int main (int argc, char * * argv) Machine * bestMachine = nullptr; unsigned long long bestLoad = 0; for (auto & m : machines) { + debug("considering building on ‘%s’", m.storeUri); + if (m.enabled && std::find(m.systemTypes.begin(), m.systemTypes.end(), neededSystem) != m.systemTypes.end() && @@ -233,16 +161,22 @@ int main (int argc, char * * argv) lock = -1; try { - sshStore = openStore("ssh-ng://" + bestMachine->hostName, - { {"ssh-key", bestMachine->sshKey }, - {"max-connections", "1" } }); - hostName = bestMachine->hostName; + + Store::Params storeParams{{"max-connections", "1"}, {"log-fd", "4"}}; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + } catch (std::exception & e) { printError("unable to open SSH connection to ‘%s’: %s; trying other available machines...", - bestMachine->hostName, e.what()); + bestMachine->storeUri, e.what()); bestMachine->enabled = false; continue; } + goto connected; } } @@ -252,22 +186,29 @@ connected: string line; if (!getline(cin, line)) throw Error("hook caller didn't send inputs"); + auto inputs = tokenizeString<PathSet>(line); if (!getline(cin, line)) throw Error("hook caller didn't send outputs"); + auto outputs = tokenizeString<PathSet>(line); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + hostName + ".upload-lock", true); + + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); if (!lockFile(uploadLock.get(), ltWrite, true)) printError("somebody is hogging the upload lock for ‘%s’, continuing..."); alarm(0); signal(SIGALRM, old); - copyPaths(store, ref<Store>(sshStore), inputs); + copyPaths(store, ref<Store>(sshStore), inputs, false, true); uploadLock = -1; - printError("building ‘%s’ on ‘%s’", drvPath, hostName); - sshStore->buildDerivation(drvPath, readDerivation(drvPath)); + BasicDerivation drv(readDerivation(drvPath)); + drv.inputSrcs = inputs; + + printError("building ‘%s’ on ‘%s’", drvPath, storeUri); + sshStore->buildDerivation(drvPath, drv); PathSet missing; for (auto & path : outputs) @@ -275,7 +216,7 @@ connected: if (!missing.empty()) { setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missing); + copyPaths(ref<Store>(sshStore), store, missing, false, true); } return; diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk index 62d5a010c247..64368a43ff73 100644 --- a/src/build-remote/local.mk +++ b/src/build-remote/local.mk @@ -7,5 +7,3 @@ build-remote_INSTALL_DIR := $(libexecdir)/nix build-remote_LIBS = libmain libutil libformat libstore build-remote_SOURCES := $(d)/build-remote.cc - -build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5b1ff0350cd1..40ca77258037 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -142,25 +142,34 @@ or { return OR_KW; } \{ { return '{'; } <INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { PUSH_STATE(STRING); return '"'; } +<INITIAL,INSIDE_DOLLAR_CURLY>\" { + PUSH_STATE(STRING); return '"'; + } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\.|\$\\.)+ { - /* It is impossible to match strings ending with '$' with one - regex because trailing contexts are only valid at the end - of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext); - return STR; - } + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext); + return STR; + } <STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -<STRING>\" { POP_STATE(); return '"'; } -<STRING>. return yytext[0]; /* just in case: shouldn't be reached */ +<STRING>\" { POP_STATE(); return '"'; } +<STRING>\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } <INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; } -<IND_STRING>\'\'\$ { +<IND_STRING>\'\'\$ | +<IND_STRING>\$ { yylval->e = new ExprIndStr("$"); return IND_STR; } @@ -178,7 +187,6 @@ or { return OR_KW; } yylval->e = new ExprIndStr("'"); return IND_STR; } -<IND_STRING>. return yytext[0]; /* just in case: shouldn't be reached */ <INITIAL,INSIDE_DOLLAR_CURLY>{ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index b536c6c00044..46c5aa21b2eb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -114,11 +114,6 @@ void BinaryCacheStore::init() } } -void BinaryCacheStore::notImpl() -{ - throw Error("operation not implemented for binary cache stores"); -} - std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) { std::promise<std::shared_ptr<std::string>> promise; diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5c2d0acfdbb7..87d4aa43838e 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -27,8 +27,6 @@ protected: BinaryCacheStore(const Params & params); - [[noreturn]] void notImpl(); - public: virtual bool fileExists(const std::string & path) = 0; @@ -65,7 +63,7 @@ public: bool isValidPathUncached(const Path & path) override; PathSet queryAllValidPaths() override - { notImpl(); } + { unsupported(); } void queryPathInfoUncached(const Path & path, std::function<void(std::shared_ptr<ValidPathInfo>)> success, @@ -73,16 +71,16 @@ public: void queryReferrers(const Path & path, PathSet & referrers) override - { notImpl(); } + { unsupported(); } PathSet queryDerivationOutputs(const Path & path) override - { notImpl(); } + { unsupported(); } StringSet queryDerivationOutputNames(const Path & path) override - { notImpl(); } + { unsupported(); } Path queryPathFromHashPart(const string & hashPart) override - { notImpl(); } + { unsupported(); } bool wantMassQuery() override { return wantMassQuery_; } @@ -99,32 +97,29 @@ public: void narFromPath(const Path & path, Sink & sink) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { notImpl(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { notImpl(); } + { unsupported(); } void ensurePath(const Path & path) override - { notImpl(); } + { unsupported(); } void addTempRoot(const Path & path) override - { notImpl(); } + { unsupported(); } void addIndirectRoot(const Path & path) override - { notImpl(); } + { unsupported(); } Roots findRoots() override - { notImpl(); } + { unsupported(); } void collectGarbage(const GCOptions & options, GCResults & results) override - { notImpl(); } + { unsupported(); } ref<FSAccessor> getFSAccessor() override; void addSignatures(const Path & storePath, const StringSet & sigs) override - { notImpl(); } + { unsupported(); } std::shared_ptr<std::string> getBuildLog(const Path & path) override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9bf1ab5aa581..8c2602a701bd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -583,11 +583,7 @@ struct HookInstance HookInstance::HookInstance() { - debug("starting build hook"); - - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; - buildHook = canonPath(buildHook); + debug("starting build hook ‘%s’", settings.buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -614,15 +610,17 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(buildHook), + baseNameOf(settings.buildHook), settings.thisSystem, - (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.buildTimeout).str() + std::to_string(settings.maxSilentTime), + std::to_string(settings.buildTimeout), + std::to_string(verbosity), + settings.builders }; - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); - throw SysError(format("executing ‘%1%’") % buildHook); + throw SysError("executing ‘%s’", settings.buildHook); }); pid.setSeparatePG(true); @@ -1568,7 +1566,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; + if (!settings.useBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique<HookInstance>(); @@ -1601,8 +1599,15 @@ HookReply DerivationGoal::tryBuildHook() debug(format("hook reply is ‘%1%’") % reply); - if (reply == "decline" || reply == "postpone") - return reply == "decline" ? rpDecline : rpPostpone; + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + settings.useBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; else if (reply != "accept") throw Error(format("bad hook reply ‘%1%’") % reply); @@ -1621,23 +1626,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the - remote system. This unfortunately has to contain the entire - derivation closure to ensure that the validity invariant holds - on the remote system. (I.e., it's unfortunate that we have to - list it since the remote system *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - worker.store.computeFSClosure(drvPath, allInputs); - - string s; - for (auto & i : allInputs) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + remote system. */ + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", inputPaths)); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - s = ""; - for (auto & i : missingPaths) { s += i; s += ' '; } - writeLine(hook->toHook.writeSide.get(), s); + writeLine(hook->toHook.writeSide.get(), concatStringsSep(" ", missingPaths)); hook->toHook.writeSide = -1; @@ -1876,6 +1870,7 @@ void DerivationGoal::startBuilder() dirsInChroot[i] = r; else { Path p = chrootRootDir + i; + debug("linking ‘%1%’ to ‘%2%’", p, r); if (link(r.c_str(), p.c_str()) == -1) { /* Hard-linking fails if we exceed the maximum link count on a file (e.g. 32000 of ext3), @@ -3105,7 +3100,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide.get()) - printError(data); // FIXME? + printError(chomp(data)); } @@ -3379,7 +3374,7 @@ void SubstitutionGoal::tryToRun() if maxBuildJobs == 0 (no local builds allowed), we still allow a substituter to run. This is because substitutions cannot be distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::min(1U, (unsigned int) settings.maxBuildJobs)) { + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { worker.waitForBuildSlot(shared_from_this()); return; } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 953bf6aaaa0a..4bdbde989ab2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -43,6 +43,10 @@ Settings::Settings() lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt")); + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s != "") builderFiles = tokenizeString<Strings>(s, ":"); + #if __linux__ sandboxPaths = tokenizeString<StringSet>("/bin/sh=" BASH_PATH); #endif diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b4f44de2e65d..ac6f6a2cfa36 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -127,6 +127,16 @@ public: Setting<bool> useBuildHook{this, true, "remote-builds", "Whether to use build hooks (for distributed builds)."}; + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + "The path of the helper program that executes builds to remote machines."}; + + Setting<std::string> builders{this, "", "builders", + "A semicolon-separated list of build machines, in the format of nix.machines."}; + + Setting<Strings> builderFiles{this, + {nixConfDir + "/machines"}, "builder-files", + "A list of files specifying build machines."}; + Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index befc560bfcec..e09932e3d182 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "worker-protocol.hh" #include "ssh.hh" +#include "derivations.hh" namespace nix { @@ -16,11 +17,15 @@ struct LegacySSHStore : public Store const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; + // Hack for getting remote build log output. + const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + struct Connection { std::unique_ptr<SSHMaster::Connection> sshConn; FdSink to; FdSource from; + int remoteVersion; }; std::string host; @@ -42,7 +47,8 @@ struct LegacySSHStore : public Store sshKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - compress) + compress, + logFD) { } @@ -53,8 +59,6 @@ struct LegacySSHStore : public Store conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - int remoteVersion; - try { conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; conn->to.flush(); @@ -62,8 +66,8 @@ struct LegacySSHStore : public Store unsigned int magic = readInt(conn->from); if (magic != SERVE_MAGIC_2) throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host); - remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host); } catch (EndOfFile & e) { @@ -148,12 +152,6 @@ struct LegacySSHStore : public Store sink(*savedNAR.data); } - /* Unsupported methods. */ - [[noreturn]] void unsupported() - { - throw Error("operation not supported on SSH stores"); - } - PathSet queryAllValidPaths() override { unsupported(); } void queryReferrers(const Path & path, PathSet & referrers) override @@ -177,12 +175,36 @@ struct LegacySSHStore : public Store const PathSet & references, bool repair) override { unsupported(); } - void buildPaths(const PathSet & paths, BuildMode buildMode) override - { unsupported(); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { unsupported(); } + { + auto conn(connections->get()); + + conn->to + << cmdBuildDerivation + << drvPath + << drv + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to + << settings.buildRepeat + << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status) readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + + return status; + } void ensurePath(const Path & path) override { unsupported(); } @@ -205,9 +227,6 @@ struct LegacySSHStore : public Store void addSignatures(const Path & storePath, const StringSet & sigs) override { unsupported(); } - bool isTrusted() override - { return true; } - void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -243,6 +262,11 @@ struct LegacySSHStore : public Store return readStorePaths<PathSet>(*this, conn->from); } + + void connect() override + { + auto conn(connections->get()); + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bf247903c9db..bf28a1c70c62 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -31,7 +31,7 @@ struct LocalStoreAccessor : public FSAccessor auto realPath = toRealPath(path); struct stat st; - if (lstat(path.c_str(), &st)) { + if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; throw SysError(format("getting status of ‘%1%’") % path); } @@ -51,7 +51,7 @@ struct LocalStoreAccessor : public FSAccessor { auto realPath = toRealPath(path); - auto entries = nix::readDirectory(path); + auto entries = nix::readDirectory(realPath); StringSet res; for (auto & entry : entries) @@ -73,7 +73,8 @@ struct LocalStoreAccessor : public FSAccessor ref<FSAccessor> LocalFSStore::getFSAccessor() { - return make_ref<LocalStoreAccessor>(ref<LocalFSStore>(std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); + return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( + std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } void LocalFSStore::narFromPath(const Path & path, Sink & sink) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5a98454ab38e..c8e61126c1b8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -915,6 +915,8 @@ void LocalStore::invalidatePath(State & state, const Path & path) void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { + assert(info.narHash); + Hash h = hashString(htSHA256, *nar); if (h != info.narHash) throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") % diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc new file mode 100644 index 000000000000..c1d9047537d3 --- /dev/null +++ b/src/libstore/machines.cc @@ -0,0 +1,91 @@ +#include "machines.hh" +#include "util.hh" +#include "globals.hh" + +#include <algorithm> + +namespace nix { + +Machine::Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) : + storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos || hasPrefix(storeUri, "local") || hasPrefix(storeUri, "remote") || hasPrefix(storeUri, "auto") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) +{} + +bool Machine::allSupported(const std::set<string> & features) const { + return std::all_of(features.begin(), features.end(), + [&](const string & feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); +} + +bool Machine::mandatoryMet(const std::set<string> & features) const { + return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string & feature) { + return features.count(feature); + }); +} + +void parseMachines(const std::string & s, Machines & machines) +{ + for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + chomp(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; + auto tokens = tokenizeString<std::vector<string>>(line); + auto sz = tokens.size(); + if (sz < 1) + throw FormatError("bad machine specification ‘%s’", line); + + auto isSet = [&](int n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; + + machines.emplace_back(tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() +{ + Machines machines; + + for (auto & file : settings.builderFiles.get()) { + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + } + } + + parseMachines(settings.builders, machines); + + return machines; +} + +} diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh new file mode 100644 index 000000000000..de92eb924e4a --- /dev/null +++ b/src/libstore/machines.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct Machine { + + const string storeUri; + const std::vector<string> systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set<string> supportedFeatures; + const std::set<string> mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set<string> & features) const; + + bool mandatoryMet(const std::set<string> & features) const; + + Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); +}; + +typedef std::vector<Machine> Machines; + +void parseMachines(const std::string & s, Machines & machines); + +Machines getMachines(); + +} diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cf234e35d373..d354812e3da4 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -220,8 +220,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - if (st.st_size) - printInfo(format("‘%1%’ has maximum number of links") % linkPath); + debug("‘%s’ has reached maximum number of links", linkPath); return; } throw SysError(format("cannot rename ‘%1%’ to ‘%2%’") % tempLink % path); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index af59d51106fc..be8819bbc004 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -100,7 +100,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() throw Error(format("socket path ‘%1%’ is too long") % socketPath); strcpy(addr.sun_path, socketPath.c_str()); - if (connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); conn->from.fd = conn->fd.get(); @@ -613,6 +613,12 @@ void RemoteStore::queryMissing(const PathSet & targets, } +void RemoteStore::connect() +{ + auto conn(connections->get()); +} + + RemoteStore::Connection::~Connection() { try { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 479cf3a7909d..ed430e4cabb6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -92,6 +92,8 @@ public: PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) override; + void connect() override; + protected: struct Connection diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index e54f3f4ba284..6edabaa3a1d9 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -31,6 +31,8 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string throw SysError("duping over stdin"); if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) + throw SysError("duping over stderr"); Strings args = { "ssh", host.c_str(), "-x", "-a" }; addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index b4396467e54e..18dea227ad1f 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -13,6 +13,7 @@ private: const std::string keyFile; const bool useMaster; const bool compress; + const int logFD; struct State { @@ -27,11 +28,12 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress) + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1) : host(host) , keyFile(keyFile) , useMaster(useMaster) , compress(compress) + , logFD(logFD) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 835bbb90e0bb..b5a91e53672f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -523,6 +523,17 @@ const Store::Stats & Store::getStats() } +void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & path : paths) + if (isDerivation(path)) + unsupported(); + + if (queryValidPaths(paths).size() != paths.size()) + unsupported(); +} + + void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const Path & storePath, bool repair, bool dontCheckSigs) { @@ -531,15 +542,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); - if (srcStore->isTrusted()) - dontCheckSigs = true; - if (!info->narHash && dontCheckSigs) { auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); info = info2; } + assert(info->narHash); + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + assert(info->narHash); + dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); } @@ -698,10 +716,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStore(const std::string & uri_) +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) { auto uri(uri_); - Store::Params params; + Store::Params params(extraParams); auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { @@ -711,11 +730,7 @@ ref<Store> openStore(const std::string & uri_) } uri = uri_.substr(0, q); } - return openStore(uri, params); -} -ref<Store> openStore(const std::string & uri, const Store::Params & params) -{ for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); if (store) { @@ -724,7 +739,7 @@ ref<Store> openStore(const std::string & uri, const Store::Params & params) } } - throw Error(format("don't know how to open Nix store ‘%s’") % uri); + throw Error("don't know how to open Nix store ‘%s’", uri); } @@ -794,7 +809,8 @@ std::list<ref<Store>> getDefaultSubstituters() } -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute) +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute, bool dontCheckSigs) { PathSet valid = to->queryValidPaths(storePaths, substitute); @@ -822,7 +838,7 @@ void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool if (!to->isValidPath(storePath)) { Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); - copyStorePath(from, to, storePath); + copyStorePath(from, to, storePath, false, dontCheckSigs); logger->incProgress(copiedLabel); } else diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 067309c9e956..b06f5d86a93a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -18,6 +18,12 @@ namespace nix { +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) +MakeError(Unsupported, Error) + + struct BasicDerivation; struct Derivation; class FSAccessor; @@ -414,7 +420,7 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) = 0; + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for @@ -564,10 +570,6 @@ public: const Stats & getStats(); - /* Whether this store paths from this store can be imported even - if they lack a signature. */ - virtual bool isTrusted() { return false; } - /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr<std::string> getBuildLog(const Path & path) @@ -580,10 +582,20 @@ public: state.lock()->pathInfoCache.clear(); } + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect() { }; + protected: Stats stats; + /* Unsupported methods. */ + [[noreturn]] void unsupported() + { + throw Unsupported("requested operation is not supported by store ‘%s’", getUri()); + } + }; @@ -656,23 +668,35 @@ void removeTempRoots(); /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: - * ‘direct’: The Nix store in /nix/store and database in + * ‘local’: The Nix store in /nix/store and database in /nix/var/nix/db, accessed directly. * ‘daemon’: The Nix store accessed via a Unix domain socket connection to nix-daemon. + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + * ‘file://<path>’: A binary cache stored in <path>. - If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on - whether the user has write access to the local Nix store/database. - set to true *unless* you're going to collect garbage. */ -ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); + * ‘https://<path>’: A binary cache accessed via HTTP. + + * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]<host>’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. -ref<Store> openStore(const std::string & uri, const Store::Params & params); + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"), + const Store::Params & extraParams = Store::Params()); -void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, bool substitute = false); +void copyPaths(ref<Store> from, ref<Store> to, const PathSet & storePaths, + bool substitute = false, bool dontCheckSigs = false); enum StoreType { tDaemon, @@ -720,10 +744,4 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); - -MakeError(SubstError, Error) -MakeError(BuildError, Error) /* denotes a permanent build failure */ -MakeError(InvalidPath, Error) - - } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 62c6433c741b..497afaa1fed4 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -215,6 +215,7 @@ template class BaseSetting<unsigned long>; template class BaseSetting<long long>; template class BaseSetting<unsigned long long>; template class BaseSetting<bool>; +template class BaseSetting<std::string>; void PathSetting::set(const std::string & str) { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9f4afd93c2fc..fa1bb5d97183 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -224,7 +224,7 @@ static void start(HashType ht, Ctx & ctx) static void update(HashType ht, Ctx & ctx, - const unsigned char * bytes, unsigned int len) + const unsigned char * bytes, size_t len) { if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 026e493514ea..98c0aff1e722 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1078,9 +1078,9 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & suffix) +bool hasPrefix(const string & s, const string & prefix) { - return s.compare(0, suffix.size(), suffix) == 0; + return s.compare(0, prefix.size(), prefix) == 0; } diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index ed43bffbc8c8..dc324abcb3ba 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -58,6 +58,6 @@ int main(int argc, char ** argv) PathSet closure; from->computeFSClosure(storePaths2, closure, false, includeOutputs); - copyPaths(from, to, closure, useSubstitutes); + copyPaths(from, to, closure, useSubstitutes, true); }); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 07ad0b45b3e4..1b90fad165af 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -483,7 +483,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe }; try { - if (trusted + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted || name == settings.buildTimeout.name || name == settings.connectTimeout.name) settings.set(name, value); diff --git a/src/nix/command.hh b/src/nix/command.hh index dc7b2637d66a..cf0097d78926 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -78,7 +78,7 @@ struct InstallablesCommand : virtual Args, StoreCommand = import ...; bla = import ...; }’. */ Value * getSourceExpr(EvalState & state); - std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings installables); + std::vector<std::shared_ptr<Installable>> parseInstallables(ref<Store> store, Strings ss); PathSet buildInstallables(ref<Store> store, bool dryRun); @@ -86,6 +86,8 @@ struct InstallablesCommand : virtual Args, StoreCommand void prepare() override; + virtual bool useDefaultInstallables() { return true; } + private: Strings _installables; @@ -112,6 +114,8 @@ public: virtual void run(ref<Store> store, Paths storePaths) = 0; void run(ref<Store> store) override; + + bool useDefaultInstallables() override { return !all; } }; typedef std::map<std::string, ref<Command>> Commands; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 57580049f25f..4756fc44bba7 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -177,21 +177,21 @@ struct InstallableAttrPath : Installable std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); -std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings installables) +std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables(ref<Store> store, Strings ss) { std::vector<std::shared_ptr<Installable>> result; - if (installables.empty()) { + if (ss.empty() && useDefaultInstallables()) { if (file == "") file = "."; - installables = Strings{""}; + ss = Strings{""}; } - for (auto & installable : installables) { + for (auto & s : ss) { - if (installable.find("/") != std::string::npos) { + if (s.find("/") != std::string::npos) { - auto path = store->toStorePath(store->followLinksToStore(installable)); + auto path = store->toStorePath(store->followLinksToStore(s)); if (store->isStorePath(path)) { if (isDerivation(path)) @@ -201,14 +201,14 @@ std::vector<std::shared_ptr<Installable>> InstallablesCommand::parseInstallables } } - else if (installable.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(*this, installable)); + else if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared<InstallableExpr>(*this, s)); - else if (installable == "" || std::regex_match(installable, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(*this, installable)); + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared<InstallableAttrPath>(*this, s)); else - throw UsageError("don't know what to do with argument ‘%s’", installable); + throw UsageError("don't know what to do with argument ‘%s’", s); } return result; diff --git a/src/nix/local.mk b/src/nix/local.mk index 21f190e476f4..e71cf16fabf6 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -6,6 +6,8 @@ nix_SOURCES := $(wildcard $(d)/*.cc) nix_LIBS = libexpr libmain libstore libutil libformat -nix_LDFLAGS = -lreadline +ifeq ($(HAVE_READLINE), 1) + nix_LDFLAGS += -lreadline +endif $(eval $(call install-symlink, nix, $(bindir)/nix-hash)) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 17203d3c299f..13488bf1dbd4 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,3 +1,5 @@ +#if HAVE_LIBREADLINE + #include <iostream> #include <cstdlib> @@ -726,3 +728,5 @@ struct CmdRepl : StoreCommand static RegisterCommand r1(make_ref<CmdRepl>()); } + +#endif |