diff options
Diffstat (limited to 'src/libstore')
-rw-r--r-- | src/libstore/binary-cache-store.cc | 7 | ||||
-rw-r--r-- | src/libstore/build.cc | 137 | ||||
-rw-r--r-- | src/libstore/gc.cc | 5 | ||||
-rw-r--r-- | src/libstore/globals.hh | 5 | ||||
-rw-r--r-- | src/libstore/local-store.cc | 3 | ||||
-rw-r--r-- | src/libstore/misc.cc | 12 | ||||
-rw-r--r-- | src/libstore/nar-accessor.cc | 3 | ||||
-rw-r--r-- | src/libstore/optimise-store.cc | 25 | ||||
-rw-r--r-- | src/libstore/pathlocks.cc | 87 | ||||
-rw-r--r-- | src/libstore/remote-fs-accessor.cc | 57 | ||||
-rw-r--r-- | src/libstore/remote-fs-accessor.hh | 29 | ||||
-rw-r--r-- | src/libstore/remote-store.cc | 100 | ||||
-rw-r--r-- | src/libstore/remote-store.hh | 35 | ||||
-rw-r--r-- | src/libstore/s3-binary-cache-store.cc | 13 | ||||
-rw-r--r-- | src/libstore/ssh-store.cc | 130 | ||||
-rw-r--r-- | src/libstore/store-api.cc | 55 | ||||
-rw-r--r-- | src/libstore/store-api.hh | 43 | ||||
-rw-r--r-- | src/libstore/worker-protocol.hh | 4 |
18 files changed, 578 insertions, 172 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4125af118cf7..3e07a2aa2b60 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -6,9 +6,9 @@ #include "globals.hh" #include "nar-info.hh" #include "sync.hh" -#include "worker-protocol.hh" -#include "nar-accessor.hh" +#include "remote-fs-accessor.hh" #include "nar-info-disk-cache.hh" +#include "nar-accessor.hh" #include "json.hh" #include <chrono> @@ -379,8 +379,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, ref<FSAccessor> BinaryCacheStore::getFSAccessor() { - return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>( - std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this()))); + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b682a80195fb..c970fbdcaa65 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,9 +17,9 @@ #include <sstream> #include <thread> #include <future> +#include <chrono> #include <limits.h> -#include <time.h> #include <sys/time.h> #include <sys/wait.h> #include <sys/types.h> @@ -187,6 +187,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) { } +typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; + + /* A mapping used to remember for each child process to what goal it belongs, and file descriptors for receiving log data and output path creation commands. */ @@ -197,8 +200,8 @@ struct Child set<int> fds; bool respectTimeouts; bool inBuildSlot; - time_t lastOutput; /* time we last got output on stdout/stderr */ - time_t timeStarted; + steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + steady_time_point timeStarted; }; @@ -238,7 +241,7 @@ private: WeakGoals waitingForAWhile; /* Last time the goals in `waitingForAWhile' where woken up. */ - time_t lastWokenUp; + steady_time_point lastWokenUp; /* Cache for pathContentsGood(). */ std::map<Path, bool> pathContentsGoodCache; @@ -1249,8 +1252,7 @@ void DerivationGoal::inputsRealised() } /* Second, the input sources. */ - for (auto & i : drv->inputSrcs) - worker.store.computeFSClosure(i, inputPaths); + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); debug(format("added input paths %1%") % showPaths(inputPaths)); @@ -1270,6 +1272,8 @@ void DerivationGoal::inputsRealised() build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + + result = BuildResult(); } @@ -1343,6 +1347,7 @@ void DerivationGoal::tryToBuild() case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; return; case rpPostpone: @@ -1419,6 +1424,9 @@ void DerivationGoal::buildDone() debug(format("builder process for ‘%1%’ finished") % drvPath); + result.timesBuilt++; + result.stopTime = time(0); + /* So the child is gone now. */ worker.childTerminated(this); @@ -2102,6 +2110,8 @@ void DerivationGoal::startBuilder() /* Create a pipe to get the output of the builder. */ builderOut.create(); + result.startTime = time(0); + /* Fork a child to build the package. */ #if __linux__ if (useChroot) { @@ -2555,15 +2565,18 @@ void DerivationGoal::runChild() */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { - if (i.first != i.second) + if (i.first != i.second.source) throw Error(format( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second); + % i.first % i.second.source); string path = i.first; struct stat st; - if (lstat(path.c_str(), &st)) + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; throw SysError(format("getting attributes of path ‘%1%’") % path); + } if (S_ISDIR(st.st_mode)) sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); else @@ -2674,7 +2687,9 @@ void DerivationGoal::registerOutputs() outputs to allow hard links between outputs. */ InodesSeen inodesSeen; - Path checkSuffix = "-check"; + Path checkSuffix = ".check"; + bool runDiffHook = settings.get("run-diff-hook", false); + bool keepPreviousRound = settings.keepFailed || runDiffHook; /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -2904,30 +2919,42 @@ void DerivationGoal::registerOutputs() assert(prevInfos.size() == infos.size()); for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { + result.isNonDeterministic = true; Path prev = i->path + checkSuffix; - if (pathExists(prev)) - throw NotDeterministic( - format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round") - % i->path % drvPath % prev); - else - throw NotDeterministic( - format("output ‘%1%’ of ‘%2%’ differs from previous round") - % i->path % drvPath); + bool prevExists = keepPreviousRound && pathExists(prev); + auto msg = prevExists + ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev) + : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath); + + auto diffHook = settings.get("diff-hook", std::string("")); + if (prevExists && diffHook != "" && runDiffHook) { + try { + auto diff = runProgram(diffHook, true, {prev, i->path}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } + + if (settings.get("enforce-determinism", true)) + throw NotDeterministic(msg); + + printError(msg); + curRound = nrRounds; // we know enough, bail out early } - abort(); // shouldn't happen } - if (settings.keepFailed) { + /* If this is the first round of several, then move the output out + of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { Path prev = i.second.path + checkSuffix; deletePath(prev); - if (curRound < nrRounds) { - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); - } + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst); } - } if (curRound < nrRounds) { @@ -2935,6 +2962,15 @@ void DerivationGoal::registerOutputs() return; } + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + } + } + /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -3045,7 +3081,8 @@ void DerivationGoal::handleEOF(int fd) void DerivationGoal::flushLine() { - if (settings.verboseBuild) + if (settings.verboseBuild && + (settings.printRepeatedBuilds || curRound == 1)) printError(filterANSIEscapes(currentLogLine, true)); else { logTail.push_back(currentLogLine); @@ -3387,7 +3424,7 @@ Worker::Worker(LocalStore & store) if (working) abort(); working = true; nrLocalBuilds = 0; - lastWokenUp = 0; + lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; } @@ -3496,7 +3533,7 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds, child.goal = goal; child.goal2 = goal.get(); child.fds = fds; - child.timeStarted = child.lastOutput = time(0); + child.timeStarted = child.lastOutput = steady_time_point::clock::now(); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; children.emplace_back(child); @@ -3615,35 +3652,38 @@ void Worker::waitForInput() bool useTimeout = false; struct timeval timeout; timeout.tv_usec = 0; - time_t before = time(0); + auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there is a build timeout, then wait for input until the first deadline for any child. */ - assert(sizeof(time_t) >= sizeof(long)); - time_t nearest = LONG_MAX; // nearest deadline + auto nearest = steady_time_point::max(); // nearest deadline for (auto & i : children) { if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } - if (nearest != LONG_MAX) { - timeout.tv_sec = std::max((time_t) 1, nearest - before); + if (nearest != steady_time_point::max()) { + timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); useTimeout = true; - printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); } /* If we are polling goals that are waiting for a lock, then wake up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == 0) + if (lastWokenUp == steady_time_point::min()) printError("waiting for locks or build slots..."); - if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); - } else lastWokenUp = 0; + if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; + timeout.tv_sec = std::max(1L, + (long) std::chrono::duration_cast<std::chrono::seconds>( + lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); + } else lastWokenUp = steady_time_point::min(); + + if (useTimeout) + vomit("sleeping %d seconds", timeout.tv_sec); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) @@ -3663,7 +3703,7 @@ void Worker::waitForInput() throw SysError("waiting for input"); } - time_t after = time(0); + auto after = steady_time_point::clock::now(); /* Process all available file descriptors. */ decltype(children)::iterator i; @@ -3701,7 +3741,7 @@ void Worker::waitForInput() if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && j->respectTimeouts && - after - j->lastOutput >= (time_t) settings.maxSilentTime) + after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { printError( format("%1% timed out after %2% seconds of silence") @@ -3712,7 +3752,7 @@ void Worker::waitForInput() else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && j->respectTimeouts && - after - j->timeStarted >= (time_t) settings.buildTimeout) + after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { printError( format("%1% timed out after %2% seconds") @@ -3721,7 +3761,7 @@ void Worker::waitForInput() } } - if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) { + if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { lastWokenUp = after; for (auto & i : waitingForAWhile) { GoalPtr goal = i.lock(); @@ -3783,15 +3823,16 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) worker.run(goals); PathSet failed; - for (auto & i : goals) - if (i->getExitCode() == Goal::ecFailed) { + for (auto & i : goals) { + if (i->getExitCode() != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } + } if (!failed.empty()) - throw Error(worker.exitStatus(), "build of %s failed",showPaths(failed)); + throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ae03604faf98..f8c4a07238c7 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -621,6 +621,11 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) /* Don't delete .chroot directories for derivations that are currently being built. */ if (isActiveTempFile(state, path, ".chroot")) return; + + /* Don't delete .check directories for derivations that are + currently being built, because we may need to run + diff-hook. */ + if (isActiveTempFile(state, path, ".check")) return; } PathSet visited; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3194193bc842..a423b4e5c0f4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -149,6 +149,11 @@ struct Settings { before being killed (0 means no limit). */ unsigned long maxLogSize; + /* When build-repeat > 0 and verboseBuild == true, whether to + print repeated builds (i.e. builds other than the first one) to + stderr. Hack to prevent Hydra logs from being polluted. */ + bool printRepeatedBuilds = true; + /* How often (in seconds) to poll for locks. */ unsigned int pollInterval; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d3a641fd977b..612efde7bb8f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -37,7 +37,8 @@ namespace nix { LocalStore::LocalStore(const Params & params) - : LocalFSStore(params) + : Store(params) + , LocalFSStore(params) , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir)) , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0c2c49e5531f..9a88cdc317b6 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -8,7 +8,7 @@ namespace nix { -void Store::computeFSClosure(const Path & startPath, +void Store::computeFSClosure(const PathSet & startPaths, PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { struct State @@ -85,7 +85,8 @@ void Store::computeFSClosure(const Path & startPath, }); }; - enqueue(startPath); + for (auto & startPath : startPaths) + enqueue(startPath); { auto state(state_.lock()); @@ -95,6 +96,13 @@ void Store::computeFSClosure(const Path & startPath, } +void Store::computeFSClosure(const Path & startPath, + PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +{ + computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); +} + + void Store::queryMissing(const PathSet & targets, PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index ded19c05d2cd..4cb5de7449ea 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -52,8 +52,9 @@ struct NarIndexer : ParseSink, StringSource void preallocateContents(unsigned long long size) override { currentStart = string(s, pos, 16); + assert(size <= std::numeric_limits<size_t>::max()); members.emplace(currentPath, - NarMember{FSAccessor::Type::tRegular, isExec, pos, size}); + NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size}); } void receiveContents(unsigned char * data, unsigned int len) override diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 1bf8b7d83bbc..454c8b49d84b 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -5,6 +5,7 @@ #include "globals.hh" #include <cstdlib> +#include <cstring> #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> @@ -148,10 +149,24 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa inodeHash.insert(st.st_ino); return; } - if (errno != EEXIST) - throw SysError(format("cannot link ‘%1%’ to ‘%2%’") % linkPath % path); - /* Fall through if another process created ‘linkPath’ before - we did. */ + + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link ‘%s’ to ‘%s’: %s", linkPath, path, strerror(errno)); + return; + + default: + throw SysError("cannot link ‘%1%’ to ‘%2%’", linkPath, path); + } } /* Yes! We've seen a file with the same contents. Replace the @@ -195,7 +210,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa printInfo(format("‘%1%’ has maximum number of links") % linkPath); return; } - throw SysError(format("cannot link ‘%1%’ to ‘%2%’") % tempLink % linkPath); + throw SysError("cannot link ‘%1%’ to ‘%2%’", tempLink, linkPath); } /* Atomically replace the old file with the new hard link. */ diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 8788ee1649fb..fecd636877af 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,5 +1,6 @@ #include "pathlocks.hh" #include "util.hh" +#include "sync.hh" #include <cerrno> #include <cstdlib> @@ -72,7 +73,7 @@ bool lockFile(int fd, LockType lockType, bool wait) close a descriptor, the previous lock will be closed as well. And there is no way to query whether we already have a lock (F_GETLK only works on locks held by other processes). */ -static StringSet lockedPaths; /* !!! not thread-safe */ +static Sync<StringSet> lockedPaths_; PathLocks::PathLocks() @@ -108,49 +109,60 @@ bool PathLocks::lockPaths(const PathSet & _paths, debug(format("locking path ‘%1%’") % path); - if (lockedPaths.find(lockPath) != lockedPaths.end()) - throw Error("deadlock: trying to re-acquire self-held lock"); + { + auto lockedPaths(lockedPaths_.lock()); + if (lockedPaths->count(lockPath)) + throw Error("deadlock: trying to re-acquire self-held lock ‘%s’", lockPath); + lockedPaths->insert(lockPath); + } + + try { - AutoCloseFD fd; + AutoCloseFD fd; - while (1) { + while (1) { - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } } + + debug(format("lock acquired on ‘%1%’") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file ‘%1%’") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file ‘%1%’ has become stale") % lockPath); + else + break; } - debug(format("lock acquired on ‘%1%’") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file ‘%1%’") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file ‘%1%’ has become stale") % lockPath); - else - break; + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); + + } catch (...) { + lockedPaths_.lock()->erase(lockPath); + throw; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - lockedPaths.insert(lockPath); } return true; @@ -172,7 +184,8 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths.erase(i.second); + lockedPaths_.lock()->erase(i.second); + if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on ‘%1%’") % i.second); @@ -193,7 +206,7 @@ void PathLocks::setDeletion(bool deletePaths) bool pathIsLockedByMe(const Path & path) { Path lockPath = path + ".lock"; - return lockedPaths.find(lockPath) != lockedPaths.end(); + return lockedPaths_.lock()->count(lockPath); } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc new file mode 100644 index 000000000000..ca14057c2e28 --- /dev/null +++ b/src/libstore/remote-fs-accessor.cc @@ -0,0 +1,57 @@ +#include "remote-fs-accessor.hh" +#include "nar-accessor.hh" + +namespace nix { + + +RemoteFSAccessor::RemoteFSAccessor(ref<Store> store) + : store(store) +{ +} + +std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) +{ + auto path = canonPath(path_); + + auto storePath = store->toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + store->narFromPath(storePath, sink); + + auto accessor = makeNarAccessor(sink.s); + nars.emplace(storePath, accessor); + return {accessor, restPath}; +} + +FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +{ + auto res = fetch(path); + return res.first->stat(res.second); +} + +StringSet RemoteFSAccessor::readDirectory(const Path & path) +{ + auto res = fetch(path); + return res.first->readDirectory(res.second); +} + +std::string RemoteFSAccessor::readFile(const Path & path) +{ + auto res = fetch(path); + return res.first->readFile(res.second); +} + +std::string RemoteFSAccessor::readLink(const Path & path) +{ + auto res = fetch(path); + return res.first->readLink(res.second); +} + +} diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh new file mode 100644 index 000000000000..28f36c8296e1 --- /dev/null +++ b/src/libstore/remote-fs-accessor.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "fs-accessor.hh" +#include "ref.hh" +#include "store-api.hh" + +namespace nix { + +class RemoteFSAccessor : public FSAccessor +{ + ref<Store> store; + + std::map<Path, ref<FSAccessor>> nars; + + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_); +public: + + RemoteFSAccessor(ref<Store> store); + + Stat stat(const Path & path) override; + + StringSet readDirectory(const Path & path) override; + + std::string readFile(const Path & path) override; + + std::string readLink(const Path & path) override; +}; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7a041c6e0ad5..77faa2f801f1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,9 +38,9 @@ template<class T> T readStorePaths(Store & store, Source & from) template PathSet readStorePaths(Store & store, Source & from); - +/* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params, size_t maxConnections) - : LocalFSStore(params) + : Store(params) , connections(make_ref<Pool<Connection>>( maxConnections, [this]() { return openConnection(); }, @@ -50,13 +50,21 @@ RemoteStore::RemoteStore(const Params & params, size_t maxConnections) } -std::string RemoteStore::getUri() +UDSRemoteStore::UDSRemoteStore(const Params & params, size_t maxConnections) + : Store(params) + , LocalFSStore(params) + , RemoteStore(params, maxConnections) +{ +} + + +std::string UDSRemoteStore::getUri() { return "daemon"; } -ref<RemoteStore::Connection> RemoteStore::openConnection() +ref<RemoteStore::Connection> UDSRemoteStore::openConnection() { auto conn = make_ref<Connection>(); @@ -84,46 +92,52 @@ ref<RemoteStore::Connection> RemoteStore::openConnection() conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); + initConnection(*conn); + + return conn; +} + + +void RemoteStore::initConnection(Connection & conn) +{ /* Send the magic greeting, check for the reply. */ try { - conn->to << WORKER_MAGIC_1; - conn->to.flush(); - unsigned int magic = readInt(conn->from); + conn.to << WORKER_MAGIC_1; + conn.to.flush(); + unsigned int magic = readInt(conn.from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - conn->daemonVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + conn.daemonVersion = readInt(conn.from); + if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 10) + if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - conn->to << PROTOCOL_VERSION; + conn.to << PROTOCOL_VERSION; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 14) { + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; if (cpu != -1) - conn->to << 1 << cpu; + conn.to << 1 << cpu; else - conn->to << 0; + conn.to << 0; } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 11) - conn->to << false; + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) + conn.to << false; - conn->processStderr(); + conn.processStderr(); } catch (Error & e) { throw Error(format("cannot start daemon worker: %1%") % e.msg()); } setOptions(conn); - - return conn; } -void RemoteStore::setOptions(ref<Connection> conn) +void RemoteStore::setOptions(Connection & conn) { - conn->to << wopSetOptions + conn.to << wopSetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback @@ -137,16 +151,16 @@ void RemoteStore::setOptions(ref<Connection> conn) << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { Settings::SettingsMap overrides = settings.getOverrides(); if (overrides["ssh-auth-sock"] == "") overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK"); - conn->to << overrides.size(); + conn.to << overrides.size(); for (auto & i : overrides) - conn->to << i.first << i.second; + conn.to << i.first << i.second; } - conn->processStderr(); + conn.processStderr(); } @@ -335,7 +349,40 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { - throw Error("RemoteStore::addToStore() not implemented"); + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + conn->to << wopImportPaths; + + StringSink sink; + sink << 1 // == path follows + ; + assert(nar->size() % 8 == 0); + sink((unsigned char *) nar->data(), nar->size()); + sink + << exportMagic + << info.path + << info.references + << info.deriver + << 0 // == no legacy signature + << 0 // == no path follows + ; + + StringSource source(*sink.s); + conn->processStderr(0, &source); + + auto importedPaths = readStorePaths<PathSet>(*this, conn->from); + assert(importedPaths.size() <= 1); + } + + else { + conn->to << wopAddToStoreNar + << info.path << info.deriver << printHash(info.narHash) + << info.references << info.registrationTime << info.narSize + << info.ultimate << info.sigs << *nar << repair << dontCheckSigs; + // FIXME: don't send nar as a string + conn->processStderr(); + } } @@ -532,7 +579,6 @@ RemoteStore::Connection::~Connection() { try { to.flush(); - fd = -1; } catch (...) { ignoreException(); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b14ce4a97ff9..40f17da300d0 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -18,7 +18,7 @@ template<typename T> class Pool; /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public LocalFSStore +class RemoteStore : public virtual Store { public: @@ -26,8 +26,6 @@ public: /* Implementations of abstract store API methods. */ - std::string getUri() override; - bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet & paths) override; @@ -87,25 +85,46 @@ public: void addSignatures(const Path & storePath, const StringSet & sigs) override; -private: +protected: struct Connection { - AutoCloseFD fd; FdSink to; FdSource from; unsigned int daemonVersion; - ~Connection(); + virtual ~Connection(); void processStderr(Sink * sink = 0, Source * source = 0); }; + virtual ref<Connection> openConnection() = 0; + + void initConnection(Connection & conn); + ref<Pool<Connection>> connections; - ref<Connection> openConnection(); +private: + + void setOptions(Connection & conn); +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore +{ +public: + + UDSRemoteStore(const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max()); + + std::string getUri() override; + +private: + + struct Connection : RemoteStore::Connection + { + AutoCloseFD fd; + }; - void setOptions(ref<Connection> conn); + ref<RemoteStore::Connection> openConnection() override; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index c11f2b06b990..5361bd9da74a 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,6 +1,7 @@ #include "config.h" #if ENABLE_S3 +#if __linux__ #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -18,6 +19,15 @@ namespace nix { +struct istringstream_nocopy : public std::stringstream +{ + istringstream_nocopy(const std::string & s) + { + rdbuf()->pubsetbuf( + (char *) s.data(), s.size()); + } +}; + struct S3Error : public Error { Aws::S3::S3Errors err; @@ -145,7 +155,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore .WithBucket(bucketName) .WithKey(path); - auto stream = std::make_shared<std::stringstream>(data); + auto stream = std::make_shared<istringstream_nocopy>(data); request.SetBody(stream); @@ -260,3 +270,4 @@ static RegisterStoreImplementation regStore([]( } #endif +#endif diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc new file mode 100644 index 000000000000..5166485226d9 --- /dev/null +++ b/src/libstore/ssh-store.cc @@ -0,0 +1,130 @@ +#include "store-api.hh" +#include "remote-store.hh" +#include "remote-fs-accessor.hh" +#include "archive.hh" +#include "worker-protocol.hh" +#include "pool.hh" + +namespace nix { + +class SSHStore : public RemoteStore +{ +public: + + SSHStore(string uri, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max()); + + std::string getUri() override; + + void narFromPath(const Path & path, Sink & sink) override; + + ref<FSAccessor> getFSAccessor() override; + +private: + + struct Connection : RemoteStore::Connection + { + Pid sshPid; + AutoCloseFD out; + AutoCloseFD in; + }; + + ref<RemoteStore::Connection> openConnection() override; + + AutoDelete tmpDir; + + Path socketPath; + + Pid sshMaster; + + string uri; + + Path key; +}; + +SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) + : Store(params) + , RemoteStore(params, maxConnections) + , tmpDir(createTempDir("", "nix", true, true, 0700)) + , socketPath((Path) tmpDir + "/ssh.sock") + , uri(std::move(uri)) + , key(get(params, "ssh-key", "")) +{ +} + +string SSHStore::getUri() +{ + return "ssh://" + uri; +} + +class ForwardSource : public Source +{ + Source & readSource; + Sink & writeSink; +public: + ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {} + size_t read(unsigned char * data, size_t len) override + { + auto res = readSource.read(data, len); + writeSink(data, len); + return res; + } +}; + +void SSHStore::narFromPath(const Path & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << path; + conn->processStderr(); + ParseSink ps; + auto fwd = ForwardSource(conn->from, sink); + parseDump(ps, fwd); +} + +ref<FSAccessor> SSHStore::getFSAccessor() +{ + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); +} + +ref<RemoteStore::Connection> SSHStore::openConnection() +{ + if ((pid_t) sshMaster == -1) { + sshMaster = startProcess([&]() { + if (key.empty()) + execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL); + else + execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL); + throw SysError("starting ssh master"); + }); + } + + auto conn = make_ref<Connection>(); + Pipe in, out; + in.create(); + out.create(); + conn->sshPid = startProcess([&]() { + if (dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("duping over STDIN"); + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("duping over STDOUT"); + execlp("ssh", "ssh", "-S", socketPath.c_str(), uri.c_str(), "nix-daemon", "--stdio", NULL); + throw SysError("executing nix-daemon --stdio over ssh"); + }); + in.readSide = -1; + out.writeSide = -1; + conn->out = std::move(out.readSide); + conn->in = std::move(in.writeSide); + conn->to = FdSink(conn->in.get()); + conn->from = FdSource(conn->out.get()); + initConnection(*conn); + return conn; +} + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, 6) != "ssh://") return 0; + return std::make_shared<SSHStore>(uri.substr(6), params); +}); + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f365406cb961..37a2d45fefe0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -449,19 +449,19 @@ const Store::Stats & Store::getStats() void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, bool repair) + const Path & storePath, bool repair, bool dontCheckSigs) { auto info = srcStore->queryPathInfo(storePath); StringSink sink; srcStore->narFromPath({storePath}, sink); - dstStore->addToStore(*info, sink.s, repair); + dstStore->addToStore(*info, sink.s, repair, dontCheckSigs); } void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, bool repair) + const PathSet & storePaths, bool repair, bool dontCheckSigs) { PathSet closure; for (auto & path : storePaths) @@ -480,7 +480,7 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, printMsg(lvlDebug, format("copying %1% missing paths") % missing.size()); for (auto & i : missing) - copyStorePath(srcStore, dstStore, i, repair); + copyStorePath(srcStore, dstStore, i, repair, dontCheckSigs); } @@ -606,7 +606,7 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStoreAt(const std::string & uri_) +ref<Store> openStore(const std::string & uri_) { auto uri(uri_); Store::Params params; @@ -629,9 +629,22 @@ ref<Store> openStoreAt(const std::string & uri_) } -ref<Store> openStore() +StoreType getStoreType(const std::string & uri, const std::string & stateDir) { - return openStoreAt(getEnv("NIX_REMOTE")); + if (uri == "daemon") { + return tDaemon; + } else if (uri == "local") { + return tLocal; + } else if (uri == "") { + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return tLocal; + else if (pathExists(settings.nixDaemonSocketFile)) + return tDaemon; + else + return tLocal; + } else { + return tOther; + } } @@ -639,26 +652,14 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> { - enum { mDaemon, mLocal, mAuto } mode; - - if (uri == "daemon") mode = mDaemon; - else if (uri == "local") mode = mLocal; - else if (uri == "") mode = mAuto; - else return 0; - - if (mode == mAuto) { - auto stateDir = get(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - mode = mLocal; - else if (pathExists(settings.nixDaemonSocketFile)) - mode = mDaemon; - else - mode = mLocal; + switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + case tDaemon: + return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); + case tLocal: + return std::shared_ptr<Store>(std::make_shared<LocalStore>(params)); + default: + return nullptr; } - - return mode == mDaemon - ? std::shared_ptr<Store>(std::make_shared<RemoteStore>(params)) - : std::shared_ptr<Store>(std::make_shared<LocalStore>(params)); }); @@ -679,7 +680,7 @@ std::list<ref<Store>> getDefaultSubstituters() auto addStore = [&](const std::string & uri) { if (done.count(uri)) return; done.insert(uri); - state->stores.push_back(openStoreAt(uri)); + state->stores.push_back(openStore(uri)); }; for (auto uri : settings.get("substituters", Strings())) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3057106ec142..789526cc2b70 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -5,6 +5,7 @@ #include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" +#include "globals.hh" #include <atomic> #include <limits> @@ -207,7 +208,20 @@ struct BuildResult NotDeterministic, } status = MiscFailure; std::string errorMsg; - //time_t startTime = 0, stopTime = 0; + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + bool success() { return status == Built || status == Substituted || status == AlreadyValid; } @@ -476,15 +490,19 @@ public: ensurePath(). */ Derivation derivationFromPath(const Path & drvPath); - /* Place in `paths' the set of all store paths in the file system + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `paths' is not cleared. If + or indirectly reached from it. `out' is not cleared. If `flipDirection' is true, the set of paths that can reach `storePath' is returned; that is, the closures under the `referrers' relation instead of the `references' relation is returned. */ + void computeFSClosure(const PathSet & paths, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + void computeFSClosure(const Path & path, - PathSet & paths, bool flipDirection = false, + PathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); /* Given a set of paths that are to be built, return the set of @@ -537,7 +555,7 @@ protected: }; -class LocalFSStore : public Store +class LocalFSStore : public virtual Store { public: const Path rootDir; @@ -576,12 +594,12 @@ void checkStoreName(const string & name); /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, bool repair = false); + const Path & storePath, bool repair = false, bool dontCheckSigs = false); /* Copy the closure of the specified paths from one store to another. */ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, bool repair = false); + const PathSet & storePaths, bool repair = false, bool dontCheckSigs = false); /* Remove the temporary roots file for this process. Any temporary @@ -604,12 +622,17 @@ void removeTempRoots(); If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on whether the user has write access to the local Nix store/database. set to true *unless* you're going to collect garbage. */ -ref<Store> openStoreAt(const std::string & uri); +ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); -/* Open the store indicated by the ‘NIX_REMOTE’ environment variable. */ -ref<Store> openStore(); +enum StoreType { + tDaemon, + tLocal, + tOther +}; + +StoreType getStoreType(const std::string & uri = getEnv("NIX_REMOTE"), const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options like diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index f8cd7cc4be29..6a4ed47cc9fa 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x111 +#define PROTOCOL_VERSION 0x112 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -46,6 +46,8 @@ typedef enum { wopVerifyStore = 35, wopBuildDerivation = 36, wopAddSignatures = 37, + wopNarFromPath = 38, + wopAddToStoreNar = 39 } WorkerOp; |