about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/build.cc177
-rw-r--r--src/libstore/download.cc20
-rw-r--r--src/libstore/gc.cc21
-rw-r--r--src/libstore/globals.hh5
-rw-r--r--src/libstore/misc.cc12
-rw-r--r--src/libstore/nar-accessor.cc3
-rw-r--r--src/libstore/optimise-store.cc33
-rw-r--r--src/libstore/pathlocks.cc87
-rw-r--r--src/libstore/remote-store.cc5
-rw-r--r--src/libstore/s3-binary-cache-store.cc31
-rw-r--r--src/libstore/store-api.hh25
11 files changed, 278 insertions, 141 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index b61ea5298e1e..7fc6ff0df0f8 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -17,9 +17,9 @@
 #include <sstream>
 #include <thread>
 #include <future>
+#include <chrono>
 
 #include <limits.h>
-#include <time.h>
 #include <sys/time.h>
 #include <sys/wait.h>
 #include <sys/types.h>
@@ -187,6 +187,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) {
 }
 
 
+typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
+
+
 /* A mapping used to remember for each child process to what goal it
    belongs, and file descriptors for receiving log data and output
    path creation commands. */
@@ -197,8 +200,8 @@ struct Child
     set<int> fds;
     bool respectTimeouts;
     bool inBuildSlot;
-    time_t lastOutput; /* time we last got output on stdout/stderr */
-    time_t timeStarted;
+    steady_time_point lastOutput; /* time we last got output on stdout/stderr */
+    steady_time_point timeStarted;
 };
 
 
@@ -238,7 +241,7 @@ private:
     WeakGoals waitingForAWhile;
 
     /* Last time the goals in `waitingForAWhile' where woken up. */
-    time_t lastWokenUp;
+    steady_time_point lastWokenUp;
 
     /* Cache for pathContentsGood(). */
     std::map<Path, bool> pathContentsGoodCache;
@@ -254,7 +257,7 @@ public:
 
     LocalStore & store;
 
-    std::shared_ptr<HookInstance> hook;
+    std::unique_ptr<HookInstance> hook;
 
     Worker(LocalStore & store);
     ~Worker();
@@ -643,7 +646,7 @@ HookInstance::~HookInstance()
 {
     try {
         toHook.writeSide = -1;
-        pid.kill(true);
+        if (pid != -1) pid.kill(true);
     } catch (...) {
         ignoreException();
     }
@@ -748,7 +751,7 @@ private:
     Pipe userNamespaceSync;
 
     /* The build hook. */
-    std::shared_ptr<HookInstance> hook;
+    std::unique_ptr<HookInstance> hook;
 
     /* Whether we're currently doing a chroot build. */
     bool useChroot = false;
@@ -957,7 +960,7 @@ void DerivationGoal::killChild()
                child. */
             ::kill(-pid, SIGKILL); /* ignore the result */
             buildUser.kill();
-            pid.wait(true);
+            pid.wait();
         } else
             pid.kill();
 
@@ -1249,8 +1252,7 @@ void DerivationGoal::inputsRealised()
         }
 
     /* Second, the input sources. */
-    for (auto & i : drv->inputSrcs)
-        worker.store.computeFSClosure(i, inputPaths);
+    worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
 
     debug(format("added input paths %1%") % showPaths(inputPaths));
 
@@ -1270,6 +1272,8 @@ void DerivationGoal::inputsRealised()
        build hook. */
     state = &DerivationGoal::tryToBuild;
     worker.wakeUp(shared_from_this());
+
+    result = BuildResult();
 }
 
 
@@ -1343,6 +1347,7 @@ void DerivationGoal::tryToBuild()
             case rpAccept:
                 /* Yes, it has started doing so.  Wait until we get
                    EOF from the hook. */
+                result.startTime = time(0); // inexact
                 state = &DerivationGoal::buildDone;
                 return;
             case rpPostpone:
@@ -1411,14 +1416,15 @@ void DerivationGoal::buildDone()
 
     /* Since we got an EOF on the logger pipe, the builder is presumed
        to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe --- just don't do that
-       :-) */
-    /* !!! this could block! security problem! solution: kill the
-       child */
-    int status = hook ? hook->pid.wait(true) : pid.wait(true);
+       simply have closed its end of the pipe, so just to be sure,
+       kill it. */
+    int status = hook ? hook->pid.kill(true) : pid.kill(true);
 
     debug(format("builder process for ‘%1%’ finished") % drvPath);
 
+    result.timesBuilt++;
+    result.stopTime = time(0);
+
     /* So the child is gone now. */
     worker.childTerminated(this);
 
@@ -1558,7 +1564,7 @@ HookReply DerivationGoal::tryBuildHook()
     if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "" || !useDerivation) return rpDecline;
 
     if (!worker.hook)
-        worker.hook = std::make_shared<HookInstance>();
+        worker.hook = std::make_unique<HookInstance>();
 
     /* Tell the hook about system features (beyond the system type)
        required from the build machine.  (The hook could parse the
@@ -1593,8 +1599,7 @@ HookReply DerivationGoal::tryBuildHook()
 
     printMsg(lvlTalkative, format("using hook to build path(s) %1%") % showPaths(missingPaths));
 
-    hook = worker.hook;
-    worker.hook.reset();
+    hook = std::move(worker.hook);
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system.  This unfortunately has to contain the entire
@@ -2102,7 +2107,11 @@ void DerivationGoal::startBuilder()
     /* Create a pipe to get the output of the builder. */
     builderOut.create();
 
+    result.startTime = time(0);
+
     /* Fork a child to build the package. */
+    ProcessOptions options;
+
 #if __linux__
     if (useChroot) {
         /* Set up private namespaces for the build:
@@ -2144,7 +2153,6 @@ void DerivationGoal::startBuilder()
 
         userNamespaceSync.create();
 
-        ProcessOptions options;
         options.allowVfork = false;
 
         Pid helper = startProcess([&]() {
@@ -2155,7 +2163,8 @@ void DerivationGoal::startBuilder()
                namespace, we can't drop additional groups; they will
                be mapped to nogroup in the child namespace. There does
                not seem to be a workaround for this. (But who can tell
-               from reading user_namespaces(7)?)*/
+               from reading user_namespaces(7)?)
+               See also https://lwn.net/Articles/621612/. */
             if (getuid() == 0 && setgroups(0, 0) == -1)
                 throw SysError("setgroups failed");
 
@@ -2179,7 +2188,7 @@ void DerivationGoal::startBuilder()
             _exit(0);
         }, options);
 
-        if (helper.wait(true) != 0)
+        if (helper.wait() != 0)
             throw Error("unable to start build process");
 
         userNamespaceSync.readSide = -1;
@@ -2210,7 +2219,6 @@ void DerivationGoal::startBuilder()
     } else
 #endif
     {
-        ProcessOptions options;
         options.allowVfork = !buildUser.enabled() && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();
@@ -2284,12 +2292,8 @@ void DerivationGoal::runChild()
                outside of the namespace.  Making a subtree private is
                local to the namespace, though, so setting MS_PRIVATE
                does not affect the outside world. */
-            Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n");
-            for (auto & i : mounts) {
-                vector<string> fields = tokenizeString<vector<string> >(i, " ");
-                string fs = decodeOctalEscaped(fields.at(4));
-                if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1)
-                    throw SysError(format("unable to make filesystem ‘%1%’ private") % fs);
+            if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) {
+                throw SysError("unable to make ‘/’ private mount");
             }
 
             /* Bind-mount chroot directory to itself, to treat it as a
@@ -2329,6 +2333,7 @@ void DerivationGoal::runChild()
                 ss.push_back("/etc/nsswitch.conf");
                 ss.push_back("/etc/services");
                 ss.push_back("/etc/hosts");
+                ss.push_back("/var/run/nscd/socket");
             }
 
             for (auto & i : ss) dirsInChroot[i] = i;
@@ -2555,15 +2560,18 @@ void DerivationGoal::runChild()
              */
             sandboxProfile += "(allow file-read* file-write* process-exec\n";
             for (auto & i : dirsInChroot) {
-                if (i.first != i.second)
+                if (i.first != i.second.source)
                     throw Error(format(
                         "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin")
-                        % i.first % i.second);
+                        % i.first % i.second.source);
 
                 string path = i.first;
                 struct stat st;
-                if (lstat(path.c_str(), &st))
+                if (lstat(path.c_str(), &st)) {
+                    if (i.second.optional && errno == ENOENT)
+                        continue;
                     throw SysError(format("getting attributes of path ‘%1%’") % path);
+                }
                 if (S_ISDIR(st.st_mode))
                     sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str();
                 else
@@ -2674,7 +2682,9 @@ void DerivationGoal::registerOutputs()
        outputs to allow hard links between outputs. */
     InodesSeen inodesSeen;
 
-    Path checkSuffix = "-check";
+    Path checkSuffix = ".check";
+    bool runDiffHook = settings.get("run-diff-hook", false);
+    bool keepPreviousRound = settings.keepFailed || runDiffHook;
 
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
@@ -2904,30 +2914,42 @@ void DerivationGoal::registerOutputs()
         assert(prevInfos.size() == infos.size());
         for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
             if (!(*i == *j)) {
+                result.isNonDeterministic = true;
                 Path prev = i->path + checkSuffix;
-                if (pathExists(prev))
-                    throw NotDeterministic(
-                        format("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round")
-                        % i->path % drvPath % prev);
-                else
-                    throw NotDeterministic(
-                        format("output ‘%1%’ of ‘%2%’ differs from previous round")
-                        % i->path % drvPath);
+                bool prevExists = keepPreviousRound && pathExists(prev);
+                auto msg = prevExists
+                    ? fmt("output ‘%1%’ of ‘%2%’ differs from ‘%3%’ from previous round", i->path, drvPath, prev)
+                    : fmt("output ‘%1%’ of ‘%2%’ differs from previous round", i->path, drvPath);
+
+                auto diffHook = settings.get("diff-hook", std::string(""));
+                if (prevExists && diffHook != "" && runDiffHook) {
+                    try {
+                        auto diff = runProgram(diffHook, true, {prev, i->path});
+                        if (diff != "")
+                            printError(chomp(diff));
+                    } catch (Error & error) {
+                        printError("diff hook execution failed: %s", error.what());
+                    }
+                }
+
+                if (settings.get("enforce-determinism", true))
+                    throw NotDeterministic(msg);
+
+                printError(msg);
+                curRound = nrRounds; // we know enough, bail out early
             }
-        abort(); // shouldn't happen
     }
 
-    if (settings.keepFailed) {
+    /* If this is the first round of several, then move the output out
+       of the way. */
+    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
         for (auto & i : drv->outputs) {
             Path prev = i.second.path + checkSuffix;
             deletePath(prev);
-            if (curRound < nrRounds) {
-                Path dst = i.second.path + checkSuffix;
-                if (rename(i.second.path.c_str(), dst.c_str()))
-                    throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst);
-            }
+            Path dst = i.second.path + checkSuffix;
+            if (rename(i.second.path.c_str(), dst.c_str()))
+                throw SysError(format("renaming ‘%1%’ to ‘%2%’") % i.second.path % dst);
         }
-
     }
 
     if (curRound < nrRounds) {
@@ -2935,6 +2957,15 @@ void DerivationGoal::registerOutputs()
         return;
     }
 
+    /* Remove the .check directories if we're done. FIXME: keep them
+       if the result was not determistic? */
+    if (curRound == nrRounds) {
+        for (auto & i : drv->outputs) {
+            Path prev = i.second.path + checkSuffix;
+            deletePath(prev);
+        }
+    }
+
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
@@ -3045,7 +3076,8 @@ void DerivationGoal::handleEOF(int fd)
 
 void DerivationGoal::flushLine()
 {
-    if (settings.verboseBuild)
+    if (settings.verboseBuild &&
+        (settings.printRepeatedBuilds || curRound == 1))
         printError(filterANSIEscapes(currentLogLine, true));
     else {
         logTail.push_back(currentLogLine);
@@ -3387,7 +3419,7 @@ Worker::Worker(LocalStore & store)
     if (working) abort();
     working = true;
     nrLocalBuilds = 0;
-    lastWokenUp = 0;
+    lastWokenUp = steady_time_point::min();
     permanentFailure = false;
     timedOut = false;
 }
@@ -3496,7 +3528,7 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds,
     child.goal = goal;
     child.goal2 = goal.get();
     child.fds = fds;
-    child.timeStarted = child.lastOutput = time(0);
+    child.timeStarted = child.lastOutput = steady_time_point::clock::now();
     child.inBuildSlot = inBuildSlot;
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
@@ -3615,35 +3647,38 @@ void Worker::waitForInput()
     bool useTimeout = false;
     struct timeval timeout;
     timeout.tv_usec = 0;
-    time_t before = time(0);
+    auto before = steady_time_point::clock::now();
 
     /* If we're monitoring for silence on stdout/stderr, or if there
        is a build timeout, then wait for input until the first
        deadline for any child. */
-    assert(sizeof(time_t) >= sizeof(long));
-    time_t nearest = LONG_MAX; // nearest deadline
+    auto nearest = steady_time_point::max(); // nearest deadline
     for (auto & i : children) {
         if (!i.respectTimeouts) continue;
         if (settings.maxSilentTime != 0)
-            nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime);
+            nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime));
         if (settings.buildTimeout != 0)
-            nearest = std::min(nearest, i.timeStarted + settings.buildTimeout);
+            nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
     }
-    if (nearest != LONG_MAX) {
-        timeout.tv_sec = std::max((time_t) 1, nearest - before);
+    if (nearest != steady_time_point::max()) {
+        timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
         useTimeout = true;
-        printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
     }
 
     /* If we are polling goals that are waiting for a lock, then wake
        up after a few seconds at most. */
     if (!waitingForAWhile.empty()) {
         useTimeout = true;
-        if (lastWokenUp == 0)
+        if (lastWokenUp == steady_time_point::min())
             printError("waiting for locks or build slots...");
-        if (lastWokenUp == 0 || lastWokenUp > before) lastWokenUp = before;
-        timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before));
-    } else lastWokenUp = 0;
+        if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
+        timeout.tv_sec = std::max(1L,
+            (long) std::chrono::duration_cast<std::chrono::seconds>(
+                lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
+    } else lastWokenUp = steady_time_point::min();
+
+    if (useTimeout)
+        vomit("sleeping %d seconds", timeout.tv_sec);
 
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
@@ -3663,9 +3698,10 @@ void Worker::waitForInput()
         throw SysError("waiting for input");
     }
 
-    time_t after = time(0);
+    auto after = steady_time_point::clock::now();
 
-    /* Process all available file descriptors. */
+    /* Process all available file descriptors. FIXME: this is
+       O(children * fds). */
     decltype(children)::iterator i;
     for (auto j = children.begin(); j != children.end(); j = i) {
         i = std::next(j);
@@ -3701,7 +3737,7 @@ void Worker::waitForInput()
         if (goal->getExitCode() == Goal::ecBusy &&
             settings.maxSilentTime != 0 &&
             j->respectTimeouts &&
-            after - j->lastOutput >= (time_t) settings.maxSilentTime)
+            after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime))
         {
             printError(
                 format("%1% timed out after %2% seconds of silence")
@@ -3712,7 +3748,7 @@ void Worker::waitForInput()
         else if (goal->getExitCode() == Goal::ecBusy &&
             settings.buildTimeout != 0 &&
             j->respectTimeouts &&
-            after - j->timeStarted >= (time_t) settings.buildTimeout)
+            after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout))
         {
             printError(
                 format("%1% timed out after %2% seconds")
@@ -3721,7 +3757,7 @@ void Worker::waitForInput()
         }
     }
 
-    if (!waitingForAWhile.empty() && lastWokenUp + (time_t) settings.pollInterval <= after) {
+    if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) {
         lastWokenUp = after;
         for (auto & i : waitingForAWhile) {
             GoalPtr goal = i.lock();
@@ -3783,12 +3819,13 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
     worker.run(goals);
 
     PathSet failed;
-    for (auto & i : goals)
-        if (i->getExitCode() == Goal::ecFailed) {
+    for (auto & i : goals) {
+        if (i->getExitCode() != Goal::ecSuccess) {
             DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
             if (i2) failed.insert(i2->getDrvPath());
             else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
         }
+    }
 
     if (!failed.empty())
         throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 954044c2344f..42873d9e8a10 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -324,20 +324,30 @@ struct CurlDownloader : public Downloader
 
     ~CurlDownloader()
     {
+        stopWorkerThread();
+
+        workerThread.join();
+
+        if (curlm) curl_multi_cleanup(curlm);
+    }
+
+    void stopWorkerThread()
+    {
         /* Signal the worker thread to exit. */
         {
             auto state(state_.lock());
             state->quit = true;
         }
-        writeFull(wakeupPipe.writeSide.get(), " ");
-
-        workerThread.join();
-
-        if (curlm) curl_multi_cleanup(curlm);
+        writeFull(wakeupPipe.writeSide.get(), " ", false);
     }
 
     void workerThreadMain()
     {
+        /* Cause this thread to be notified on SIGINT. */
+        auto callback = createInterruptCallback([&]() {
+            stopWorkerThread();
+        });
+
         std::map<CURL *, std::shared_ptr<DownloadItem>> items;
 
         bool quit = false;
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ae03604faf98..8e90913cc3f1 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -379,7 +379,7 @@ void LocalStore::findRuntimeRoots(PathSet & roots)
         auto digitsRegex = std::regex(R"(^\d+$)");
         auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
         auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
-        while (errno = 0, ent = readdir(procDir)) {
+        while (errno = 0, ent = readdir(procDir.get())) {
             checkInterrupt();
             if (std::regex_match(ent->d_name, digitsRegex)) {
                 readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths);
@@ -393,14 +393,14 @@ void LocalStore::findRuntimeRoots(PathSet & roots)
                     throw SysError(format("opening %1%") % fdStr);
                 }
                 struct dirent * fd_ent;
-                while (errno = 0, fd_ent = readdir(fdDir)) {
+                while (errno = 0, fd_ent = readdir(fdDir.get())) {
                     if (fd_ent->d_name[0] != '.') {
                         readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths);
                     }
                 }
                 if (errno)
                     throw SysError(format("iterating /proc/%1%/fd") % ent->d_name);
-                fdDir.close();
+                fdDir.reset();
 
                 auto mapLines =
                     tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n");
@@ -621,6 +621,11 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
         /* Don't delete .chroot directories for derivations that are
            currently being built. */
         if (isActiveTempFile(state, path, ".chroot")) return;
+
+        /* Don't delete .check directories for derivations that are
+           currently being built, because we may need to run
+           diff-hook. */
+        if (isActiveTempFile(state, path, ".check")) return;
     }
 
     PathSet visited;
@@ -646,13 +651,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
    the link count. */
 void LocalStore::removeUnusedLinks(const GCState & state)
 {
-    AutoCloseDir dir = opendir(linksDir.c_str());
+    AutoCloseDir dir(opendir(linksDir.c_str()));
     if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir);
 
     long long actualSize = 0, unsharedSize = 0;
 
     struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir)) {
+    while (errno = 0, dirent = readdir(dir.get())) {
         checkInterrupt();
         string name = dirent->d_name;
         if (name == "." || name == "..") continue;
@@ -771,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         try {
 
-            AutoCloseDir dir = opendir(realStoreDir.c_str());
+            AutoCloseDir dir(opendir(realStoreDir.c_str()));
             if (!dir) throw SysError(format("opening directory ‘%1%’") % realStoreDir);
 
             /* Read the store and immediately delete all paths that
@@ -782,7 +787,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                can start faster. */
             Paths entries;
             struct dirent * dirent;
-            while (errno = 0, dirent = readdir(dir)) {
+            while (errno = 0, dirent = readdir(dir.get())) {
                 checkInterrupt();
                 string name = dirent->d_name;
                 if (name == "." || name == "..") continue;
@@ -793,7 +798,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                     tryToDelete(state, path);
             }
 
-            dir.close();
+            dir.reset();
 
             /* Now delete the unreachable valid paths.  Randomise the
                order in which we delete entries to make the collector
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3194193bc842..a423b4e5c0f4 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -149,6 +149,11 @@ struct Settings {
        before being killed (0 means no limit). */
     unsigned long maxLogSize;
 
+    /* When build-repeat > 0 and verboseBuild == true, whether to
+       print repeated builds (i.e. builds other than the first one) to
+       stderr. Hack to prevent Hydra logs from being polluted. */
+    bool printRepeatedBuilds = true;
+
     /* How often (in seconds) to poll for locks. */
     unsigned int pollInterval;
 
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 0c2c49e5531f..9a88cdc317b6 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -8,7 +8,7 @@
 namespace nix {
 
 
-void Store::computeFSClosure(const Path & startPath,
+void Store::computeFSClosure(const PathSet & startPaths,
     PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
 {
     struct State
@@ -85,7 +85,8 @@ void Store::computeFSClosure(const Path & startPath,
             });
     };
 
-    enqueue(startPath);
+    for (auto & startPath : startPaths)
+        enqueue(startPath);
 
     {
         auto state(state_.lock());
@@ -95,6 +96,13 @@ void Store::computeFSClosure(const Path & startPath,
 }
 
 
+void Store::computeFSClosure(const Path & startPath,
+    PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
+{
+    computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers);
+}
+
+
 void Store::queryMissing(const PathSet & targets,
     PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_,
     unsigned long long & downloadSize_, unsigned long long & narSize_)
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index ded19c05d2cd..4cb5de7449ea 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -52,8 +52,9 @@ struct NarIndexer : ParseSink, StringSource
     void preallocateContents(unsigned long long size) override
     {
         currentStart = string(s, pos, 16);
+        assert(size <= std::numeric_limits<size_t>::max());
         members.emplace(currentPath,
-            NarMember{FSAccessor::Type::tRegular, isExec, pos, size});
+            NarMember{FSAccessor::Type::tRegular, isExec, pos, (size_t) size});
     }
 
     void receiveContents(unsigned char * data, unsigned int len) override
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 1bf8b7d83bbc..b71c7e905ff1 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -5,6 +5,7 @@
 #include "globals.hh"
 
 #include <cstdlib>
+#include <cstring>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -46,11 +47,11 @@ LocalStore::InodeHash LocalStore::loadInodeHash()
     debug("loading hash inodes in memory");
     InodeHash inodeHash;
 
-    AutoCloseDir dir = opendir(linksDir.c_str());
+    AutoCloseDir dir(opendir(linksDir.c_str()));
     if (!dir) throw SysError(format("opening directory ‘%1%’") % linksDir);
 
     struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir)) { /* sic */
+    while (errno = 0, dirent = readdir(dir.get())) { /* sic */
         checkInterrupt();
         // We don't care if we hit non-hash files, anything goes
         inodeHash.insert(dirent->d_ino);
@@ -67,11 +68,11 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
 {
     Strings names;
 
-    AutoCloseDir dir = opendir(path.c_str());
+    AutoCloseDir dir(opendir(path.c_str()));
     if (!dir) throw SysError(format("opening directory ‘%1%’") % path);
 
     struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir)) { /* sic */
+    while (errno = 0, dirent = readdir(dir.get())) { /* sic */
         checkInterrupt();
 
         if (inodeHash.count(dirent->d_ino)) {
@@ -148,10 +149,24 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
             inodeHash.insert(st.st_ino);
             return;
         }
-        if (errno != EEXIST)
-            throw SysError(format("cannot link ‘%1%’ to ‘%2%’") % linkPath % path);
-        /* Fall through if another process created ‘linkPath’ before
-           we did. */
+
+        switch (errno) {
+        case EEXIST:
+            /* Fall through if another process created ‘linkPath’ before
+               we did. */
+            break;
+
+        case ENOSPC:
+            /* On ext4, that probably means the directory index is
+               full.  When that happens, it's fine to ignore it: we
+               just effectively disable deduplication of this
+               file.  */
+            printInfo("cannot link ‘%s’ to ‘%s’: %s", linkPath, path, strerror(errno));
+            return;
+
+        default:
+            throw SysError("cannot link ‘%1%’ to ‘%2%’", linkPath, path);
+        }
     }
 
     /* Yes!  We've seen a file with the same contents.  Replace the
@@ -195,7 +210,7 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa
                 printInfo(format("‘%1%’ has maximum number of links") % linkPath);
             return;
         }
-        throw SysError(format("cannot link ‘%1%’ to ‘%2%’") % tempLink % linkPath);
+        throw SysError("cannot link ‘%1%’ to ‘%2%’", tempLink, linkPath);
     }
 
     /* Atomically replace the old file with the new hard link. */
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 8fc862073030..620c9a6b752d 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -1,5 +1,6 @@
 #include "pathlocks.hh"
 #include "util.hh"
+#include "sync.hh"
 
 #include <cerrno>
 #include <cstdlib>
@@ -74,7 +75,7 @@ bool lockFile(int fd, LockType lockType, bool wait)
    close a descriptor, the previous lock will be closed as well.  And
    there is no way to query whether we already have a lock (F_GETLK
    only works on locks held by other processes). */
-static StringSet lockedPaths; /* !!! not thread-safe */
+static Sync<StringSet> lockedPaths_;
 
 
 PathLocks::PathLocks()
@@ -110,49 +111,60 @@ bool PathLocks::lockPaths(const PathSet & _paths,
 
         debug(format("locking path ‘%1%’") % path);
 
-        if (lockedPaths.find(lockPath) != lockedPaths.end())
-            throw Error("deadlock: trying to re-acquire self-held lock");
+        {
+            auto lockedPaths(lockedPaths_.lock());
+            if (lockedPaths->count(lockPath))
+                throw Error("deadlock: trying to re-acquire self-held lock ‘%s’", lockPath);
+            lockedPaths->insert(lockPath);
+        }
+
+        try {
 
-        AutoCloseFD fd;
+            AutoCloseFD fd;
 
-        while (1) {
+            while (1) {
 
-            /* Open/create the lock file. */
-            fd = openLockFile(lockPath, true);
+                /* Open/create the lock file. */
+                fd = openLockFile(lockPath, true);
 
-            /* Acquire an exclusive lock. */
-            if (!lockFile(fd.get(), ltWrite, false)) {
-                if (wait) {
-                    if (waitMsg != "") printError(waitMsg);
-                    lockFile(fd.get(), ltWrite, true);
-                } else {
-                    /* Failed to lock this path; release all other
-                       locks. */
-                    unlock();
-                    return false;
+                /* Acquire an exclusive lock. */
+                if (!lockFile(fd.get(), ltWrite, false)) {
+                    if (wait) {
+                        if (waitMsg != "") printError(waitMsg);
+                        lockFile(fd.get(), ltWrite, true);
+                    } else {
+                        /* Failed to lock this path; release all other
+                           locks. */
+                        unlock();
+                        return false;
+                    }
                 }
+
+                debug(format("lock acquired on ‘%1%’") % lockPath);
+
+                /* Check that the lock file hasn't become stale (i.e.,
+                   hasn't been unlinked). */
+                struct stat st;
+                if (fstat(fd.get(), &st) == -1)
+                    throw SysError(format("statting lock file ‘%1%’") % lockPath);
+                if (st.st_size != 0)
+                    /* This lock file has been unlinked, so we're holding
+                       a lock on a deleted file.  This means that other
+                       processes may create and acquire a lock on
+                       `lockPath', and proceed.  So we must retry. */
+                    debug(format("open lock file ‘%1%’ has become stale") % lockPath);
+                else
+                    break;
             }
 
-            debug(format("lock acquired on ‘%1%’") % lockPath);
-
-            /* Check that the lock file hasn't become stale (i.e.,
-               hasn't been unlinked). */
-            struct stat st;
-            if (fstat(fd.get(), &st) == -1)
-                throw SysError(format("statting lock file ‘%1%’") % lockPath);
-            if (st.st_size != 0)
-                /* This lock file has been unlinked, so we're holding
-                   a lock on a deleted file.  This means that other
-                   processes may create and acquire a lock on
-                   `lockPath', and proceed.  So we must retry. */
-                debug(format("open lock file ‘%1%’ has become stale") % lockPath);
-            else
-                break;
+            /* Use borrow so that the descriptor isn't closed. */
+            fds.push_back(FDPair(fd.release(), lockPath));
+
+        } catch (...) {
+            lockedPaths_.lock()->erase(lockPath);
+            throw;
         }
 
-        /* Use borrow so that the descriptor isn't closed. */
-        fds.push_back(FDPair(fd.release(), lockPath));
-        lockedPaths.insert(lockPath);
     }
 
     return true;
@@ -174,7 +186,8 @@ void PathLocks::unlock()
     for (auto & i : fds) {
         if (deletePaths) deleteLockFile(i.second, i.first);
 
-        lockedPaths.erase(i.second);
+        lockedPaths_.lock()->erase(i.second);
+
         if (close(i.first) == -1)
             printError(
                 format("error (ignored): cannot close lock file on ‘%1%’") % i.second);
@@ -195,7 +208,7 @@ void PathLocks::setDeletion(bool deletePaths)
 bool pathIsLockedByMe(const Path & path)
 {
     Path lockPath = path + ".lock";
-    return lockedPaths.find(lockPath) != lockedPaths.end();
+    return lockedPaths_.lock()->count(lockPath);
 }
 
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 77faa2f801f1..816d95ba6075 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -599,9 +599,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         else if (msg == STDERR_READ) {
             if (!source) throw Error("no source");
             size_t len = readInt(from);
-            unsigned char * buf = new unsigned char[len];
-            AutoDeleteArray<unsigned char> d(buf);
-            writeString(buf, source->read(buf, len), to);
+            auto buf = std::make_unique<unsigned char[]>(len);
+            writeString(buf.get(), source->read(buf.get(), len), to);
             to.flush();
         }
         else
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index c11f2b06b990..ccb71f1eefe5 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -1,23 +1,34 @@
 #include "config.h"
 
 #if ENABLE_S3
+#if __linux__
 
 #include "s3-binary-cache-store.hh"
 #include "nar-info.hh"
 #include "nar-info-disk-cache.hh"
 #include "globals.hh"
 
+#include <aws/core/Aws.h>
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/s3/S3Client.h>
 #include <aws/s3/model/CreateBucketRequest.h>
 #include <aws/s3/model/GetBucketLocationRequest.h>
 #include <aws/s3/model/GetObjectRequest.h>
 #include <aws/s3/model/HeadObjectRequest.h>
-#include <aws/s3/model/PutObjectRequest.h>
 #include <aws/s3/model/ListObjectsRequest.h>
+#include <aws/s3/model/PutObjectRequest.h>
 
 namespace nix {
 
+struct istringstream_nocopy : public std::stringstream
+{
+    istringstream_nocopy(const std::string & s)
+    {
+        rdbuf()->pubsetbuf(
+            (char *) s.data(), s.size());
+    }
+};
+
 struct S3Error : public Error
 {
     Aws::S3::S3Errors err;
@@ -37,6 +48,20 @@ R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome)
     return outcome.GetResultWithOwnership();
 }
 
+static void initAWS()
+{
+    static std::once_flag flag;
+    std::call_once(flag, []() {
+        Aws::SDKOptions options;
+
+        /* We install our own OpenSSL locking function (see
+           shared.cc), so don't let aws-sdk-cpp override it. */
+        options.cryptoOptions.initAndCleanupOpenSSL = false;
+
+        Aws::InitAPI(options);
+    });
+}
+
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     std::string bucketName;
@@ -63,6 +88,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
     ref<Aws::Client::ClientConfiguration> makeConfig()
     {
+        initAWS();
         auto res = make_ref<Aws::Client::ClientConfiguration>();
         res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
         res->requestTimeoutMs = 600 * 1000;
@@ -145,7 +171,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             .WithBucket(bucketName)
             .WithKey(path);
 
-        auto stream = std::make_shared<std::stringstream>(data);
+        auto stream = std::make_shared<istringstream_nocopy>(data);
 
         request.SetBody(stream);
 
@@ -260,3 +286,4 @@ static RegisterStoreImplementation regStore([](
 }
 
 #endif
+#endif
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 32523dc785aa..ec3bf5a6fd83 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -208,7 +208,20 @@ struct BuildResult
         NotDeterministic,
     } status = MiscFailure;
     std::string errorMsg;
-    //time_t startTime = 0, stopTime = 0;
+
+    /* How many times this build was performed. */
+    unsigned int timesBuilt = 0;
+
+    /* If timesBuilt > 1, whether some builds did not produce the same
+       result. (Note that 'isNonDeterministic = false' does not mean
+       the build is deterministic, just that we don't have evidence of
+       non-determinism.) */
+    bool isNonDeterministic = false;
+
+    /* The start/stop times of the build (or one of the rounds, if it
+       was repeated). */
+    time_t startTime = 0, stopTime = 0;
+
     bool success() {
         return status == Built || status == Substituted || status == AlreadyValid;
     }
@@ -477,15 +490,19 @@ public:
        ensurePath(). */
     Derivation derivationFromPath(const Path & drvPath);
 
-    /* Place in `paths' the set of all store paths in the file system
+    /* Place in `out' the set of all store paths in the file system
        closure of `storePath'; that is, all paths than can be directly
-       or indirectly reached from it.  `paths' is not cleared.  If
+       or indirectly reached from it.  `out' is not cleared.  If
        `flipDirection' is true, the set of paths that can reach
        `storePath' is returned; that is, the closures under the
        `referrers' relation instead of the `references' relation is
        returned. */
+    void computeFSClosure(const PathSet & paths,
+        PathSet & out, bool flipDirection = false,
+        bool includeOutputs = false, bool includeDerivers = false);
+
     void computeFSClosure(const Path & path,
-        PathSet & paths, bool flipDirection = false,
+        PathSet & out, bool flipDirection = false,
         bool includeOutputs = false, bool includeDerivers = false);
 
     /* Given a set of paths that are to be built, return the set of
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-10-17T21·06+0000