diff options
Diffstat (limited to 'src/libstore')
| -rw-r--r-- | src/libstore/build.cc | 38 | ||||
| -rw-r--r-- | src/libstore/derivations.cc | 3 | ||||
| -rw-r--r-- | src/libstore/download.cc | 35 | ||||
| -rw-r--r-- | src/libstore/gc.cc | 16 | ||||
| -rw-r--r-- | src/libstore/optimise-store.cc | 8 | ||||
| -rw-r--r-- | src/libstore/pathlocks.cc | 2 | ||||
| -rw-r--r-- | src/libstore/remote-store.cc | 5 | ||||
| -rw-r--r-- | src/libstore/ssh-store.cc | 2 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 33 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 2 |
10 files changed, 98 insertions, 46 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c46b7cd641c4..7fc6ff0df0f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -257,7 +257,7 @@ public: LocalStore & store; - std::shared_ptr<HookInstance> hook; + std::unique_ptr<HookInstance> hook; Worker(LocalStore & store); ~Worker(); @@ -646,7 +646,7 @@ HookInstance::~HookInstance() { try { toHook.writeSide = -1; - pid.kill(true); + if (pid != -1) pid.kill(true); } catch (...) { ignoreException(); } @@ -751,7 +751,7 @@ private: Pipe userNamespaceSync; /* The build hook. */ - std::shared_ptr<HookInstance> hook; + std::unique_ptr<HookInstance> hook; /* Whether we're currently doing a chroot build. */ bool useChroot = false; @@ -960,7 +960,7 @@ void DerivationGoal::killChild() child. */ ::kill(-pid, SIGKILL); /* ignore the result */ buildUser.kill(); - pid.wait(true); + pid.wait(); } else pid.kill(); @@ -1416,11 +1416,9 @@ void DerivationGoal::buildDone() /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have - simply have closed its end of the pipe --- just don't do that - :-) */ - /* !!! this could block! security problem! solution: kill the - child */ - int status = hook ? hook->pid.wait(true) : pid.wait(true); + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill(true) : pid.kill(true); debug(format("builder process for ‘%1%’ finished") % drvPath); @@ -1566,7 +1564,7 @@ HookReply DerivationGoal::tryBuildHook() if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline; if (!worker.hook) - worker.hook = std::make_shared<HookInstance>(); + worker.hook = std::make_unique<HookInstance>(); /* Tell the hook about system features (beyond the system type) required from the build machine. (The hook could parse the @@ -1601,8 +1599,7 @@ HookReply DerivationGoal::tryBuildHook() printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths)); - hook = worker.hook; - worker.hook.reset(); + hook = std::move(worker.hook); /* Tell the hook all the inputs that have to be copied to the remote system. This unfortunately has to contain the entire @@ -2113,6 +2110,8 @@ void DerivationGoal::startBuilder() result.startTime = time(0); /* Fork a child to build the package. */ + ProcessOptions options; + #if __linux__ if (useChroot) { /* Set up private namespaces for the build: @@ -2154,7 +2153,6 @@ void DerivationGoal::startBuilder() userNamespaceSync.create(); - ProcessOptions options; options.allowVfork = false; Pid helper = startProcess([&]() { @@ -2190,7 +2188,7 @@ void DerivationGoal::startBuilder() _exit(0); }, options); - if (helper.wait(true) != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2221,7 +2219,6 @@ void DerivationGoal::startBuilder() } else #endif { - ProcessOptions options; options.allowVfork = !buildUser.enabled() && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); @@ -2295,12 +2292,8 @@ void DerivationGoal::runChild() outside of the namespace. Making a subtree private is local to the namespace, though, so setting MS_PRIVATE does not affect the outside world. */ - Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n"); - for (auto & i : mounts) { - vector<string> fields = tokenizeString<vector<string> >(i, " "); - string fs = decodeOctalEscaped(fields.at(4)); - if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1) - throw SysError(format("unable to make filesystem ‘%1%’ private") % fs); + if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { + throw SysError("unable to make ‘/’ private mount"); } /* Bind-mount chroot directory to itself, to treat it as a @@ -3707,7 +3700,8 @@ void Worker::waitForInput() auto after = steady_time_point::clock::now(); - /* Process all available file descriptors. */ + /* Process all available file descriptors. FIXME: this is + O(children * fds). */ decltype(children)::iterator i; for (auto j = children.begin(); j != children.end(); j = i) { i = std::next(j); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d934bda38225..79526c594f71 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -88,9 +88,6 @@ Path writeDerivation(ref<Store> store, } -MakeError(FormatError, Error) - - /* Read string `s' from stream `str'. */ static void expect(std::istream & str, const string & s) { diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 954044c2344f..8030e83b0dd5 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -172,6 +172,13 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); } + static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) + { + if (type == CURLINFO_TEXT) + vomit("curl: %s", chomp(std::string(data, size))); + return 0; + } + void init() { // FIXME: handle parallel downloads. @@ -184,6 +191,12 @@ struct CurlDownloader : public Downloader if (!req) req = curl_easy_init(); curl_easy_reset(req); + + if (verbosity >= lvlVomit) { + curl_easy_setopt(req, CURLOPT_VERBOSE, 1); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + } + curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); @@ -324,20 +337,30 @@ struct CurlDownloader : public Downloader ~CurlDownloader() { + stopWorkerThread(); + + workerThread.join(); + + if (curlm) curl_multi_cleanup(curlm); + } + + void stopWorkerThread() + { /* Signal the worker thread to exit. */ { auto state(state_.lock()); state->quit = true; } - writeFull(wakeupPipe.writeSide.get(), " "); - - workerThread.join(); - - if (curlm) curl_multi_cleanup(curlm); + writeFull(wakeupPipe.writeSide.get(), " ", false); } void workerThreadMain() { + /* Cause this thread to be notified on SIGINT. */ + auto callback = createInterruptCallback([&]() { + stopWorkerThread(); + }); + std::map<CURL *, std::shared_ptr<DownloadItem>> items; bool quit = false; @@ -377,7 +400,7 @@ struct CurlDownloader : public Downloader nextWakeup != std::chrono::steady_clock::time_point() ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count()) : 1000000000; - //printMsg(lvlVomit, format("download thread waiting for %d ms") % sleepTimeMs); + vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f8c4a07238c7..8e90913cc3f1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -379,7 +379,7 @@ void LocalStore::findRuntimeRoots(PathSet & roots) auto digitsRegex = std::regex(R"(^\d+$)"); auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); - while (errno = 0, ent = readdir(procDir)) { + while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); @@ -393,14 +393,14 @@ void LocalStore::findRuntimeRoots(PathSet & roots) throw SysError(format("opening %1%") % fdStr); } struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir)) { + while (errno = 0, fd_ent = readdir(fdDir.get())) { if (fd_ent->d_name[0] != '.') { readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); } } if (errno) throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); - fdDir.close(); + fdDir.reset(); auto mapLines = tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); @@ -651,13 +651,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) the link count. */ void LocalStore::removeUnusedLinks(const GCState & state) { - AutoCloseDir dir = opendir(linksDir.c_str()); + AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir); long long actualSize = 0, unsharedSize = 0; struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { + while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -776,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) try { - AutoCloseDir dir = opendir(realStoreDir.c_str()); + AutoCloseDir dir(opendir(realStoreDir.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir); /* Read the store and immediately delete all paths that @@ -787,7 +787,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) can start faster. */ Paths entries; struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { + while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -798,7 +798,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) tryToDelete(state, path); } - dir.close(); + dir.reset(); /* Now delete the unreachable valid paths. Randomise the order in which we delete entries to make the collector diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 454c8b49d84b..b71c7e905ff1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -47,11 +47,11 @@ LocalStore::InodeHash LocalStore::loadInodeHash() debug("loading hash inodes in memory"); InodeHash inodeHash; - AutoCloseDir dir = opendir(linksDir.c_str()); + AutoCloseDir dir(opendir(linksDir.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir); struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ checkInterrupt(); // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); @@ -68,11 +68,11 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa { Strings names; - AutoCloseDir dir = opendir(path.c_str()); + AutoCloseDir dir(opendir(path.c_str())); if (!dir) throw SysError(format("opening directory ‘%1%’") % path); struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index fecd636877af..620c9a6b752d 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -54,6 +54,8 @@ bool lockFile(int fd, LockType lockType, bool wait) checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); + else + return false; } } else { while (fcntl(fd, F_SETLK, &lock) != 0) { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 77faa2f801f1..816d95ba6075 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -599,9 +599,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else if (msg == STDERR_READ) { if (!source) throw Error("no source"); size_t len = readInt(from); - unsigned char * buf = new unsigned char[len]; - AutoDeleteArray<unsigned char> d(buf); - writeString(buf, source->read(buf, len), to); + auto buf = std::make_unique<unsigned char[]>(len); + writeString(buf.get(), source->read(buf.get(), len), to); to.flush(); } else diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 5166485226d9..3d01594009a0 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -49,6 +49,8 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections) , uri(std::move(uri)) , key(get(params, "ssh-key", "")) { + /* open a connection and perform the handshake to verify all is well */ + connections->get(); } string SSHStore::getUri() diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 37a2d45fefe0..8fdd62771552 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "util.hh" #include "nar-info-disk-cache.hh" +#include "thread-pool.hh" #include <future> @@ -698,4 +699,36 @@ std::list<ref<Store>> getDefaultSubstituters() } +void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths) +{ + std::string copiedLabel = "copied"; + + logger->setExpected(copiedLabel, storePaths.size()); + + ThreadPool pool; + + processGraph<Path>(pool, + PathSet(storePaths.begin(), storePaths.end()), + + [&](const Path & storePath) { + return from->queryPathInfo(storePath)->references; + }, + + [&](const Path & storePath) { + checkInterrupt(); + + if (!to->isValidPath(storePath)) { + Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); + + copyStorePath(from, to, storePath); + + logger->incProgress(copiedLabel); + } else + logger->incExpected(copiedLabel, -1); + }); + + pool.process(); +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 789526cc2b70..ec3bf5a6fd83 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -625,6 +625,8 @@ void removeTempRoots(); ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE")); +void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths); + enum StoreType { tDaemon, tLocal, |