about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc64
-rw-r--r--src/libstore/binary-cache-store.hh8
-rw-r--r--src/libstore/build.cc404
-rw-r--r--src/libstore/builtins.cc2
-rw-r--r--src/libstore/download.cc2
-rw-r--r--src/libstore/gc.cc4
-rw-r--r--src/libstore/globals.cc15
-rw-r--r--src/libstore/globals.hh30
-rw-r--r--src/libstore/http-binary-cache-store.cc12
-rw-r--r--src/libstore/local-binary-cache-store.cc49
-rw-r--r--src/libstore/local-store.cc233
-rw-r--r--src/libstore/local-store.hh29
-rw-r--r--src/libstore/local.mk2
-rw-r--r--src/libstore/optimise-store.cc2
-rw-r--r--src/libstore/remote-store.cc18
-rw-r--r--src/libstore/remote-store.hh2
-rw-r--r--src/libstore/s3-binary-cache-store.cc16
-rw-r--r--src/libstore/s3-binary-cache-store.hh4
-rw-r--r--src/libstore/store-api.cc56
-rw-r--r--src/libstore/store-api.hh13
20 files changed, 338 insertions, 627 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 93863b95fe7f..411d10130a31 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -15,14 +15,13 @@
 namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore,
-    const Path & secretKeyFile)
+    const StoreParams & params)
     : localStore(localStore)
+    , compression(get(params, "compression", "xz"))
 {
-    if (secretKeyFile != "") {
+    auto secretKeyFile = get(params, "secret-key", "");
+    if (secretKeyFile != "")
         secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
-        publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys);
-        publicKeys->emplace(secretKey->name, secretKey->toPublicKey());
-    }
 
     StringSink sink;
     sink << narVersionMagic1;
@@ -47,8 +46,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
     return storePathToHash(storePath) + ".narinfo";
 }
 
-void BinaryCacheStore::addToCache(const ValidPathInfo & info,
-    const string & nar)
+void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref<std::string> nar)
 {
     /* Verify that all references are valid. This may do some .narinfo
        reads, but typically they'll already be cached. */
@@ -64,40 +62,43 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
     auto narInfoFile = narInfoFileFor(info.path);
     if (fileExists(narInfoFile)) return;
 
-    assert(nar.compare(0, narMagic.size(), narMagic) == 0);
+    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
 
     auto narInfo = make_ref<NarInfo>(info);
 
-    narInfo->narSize = nar.size();
-    narInfo->narHash = hashString(htSHA256, nar);
+    narInfo->narSize = nar->size();
+    narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
     /* Compress the NAR. */
-    narInfo->compression = "xz";
+    narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    string narXz = compressXZ(nar);
+    auto narCompressed = compress(compression, nar);
     auto now2 = std::chrono::steady_clock::now();
-    narInfo->fileHash = hashString(htSHA256, narXz);
-    narInfo->fileSize = narXz.size();
+    narInfo->fileHash = hashString(htSHA256, *narCompressed);
+    narInfo->fileSize = narCompressed->size();
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
         % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
+        % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
 
     /* Atomically write the NAR file. */
-    narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz";
+    narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar"
+        + (compression == "xz" ? ".xz" :
+           compression == "bzip2" ? ".bz2" :
+           "");
     if (!fileExists(narInfo->url)) {
         stats.narWrite++;
-        upsertFile(narInfo->url, narXz);
+        upsertFile(narInfo->url, *narCompressed);
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar.size();
-    stats.narWriteCompressedBytes += narXz.size();
+    stats.narWriteBytes += nar->size();
+    stats.narWriteCompressedBytes += narCompressed->size();
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
@@ -139,12 +140,12 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
 
     /* Decompress the NAR. FIXME: would be nice to have the remote
        side do this. */
-    if (info->compression == "none")
-        ;
-    else if (info->compression == "xz")
-        nar = decompressXZ(*nar);
-    else
-        throw Error(format("unknown NAR compression type ‘%1%’") % nar);
+    try {
+        nar = decompress(info->compression, ref<std::string>(nar));
+    } catch (UnknownCompressionMethod &) {
+        throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’")
+            % storePath % info->compression);
+    }
 
     stats.narReadBytes += nar->size();
 
@@ -213,11 +214,6 @@ std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Pat
 
     stats.narInfoRead++;
 
-    if (publicKeys) {
-        if (!narInfo->checkSignatures(*publicKeys))
-            throw Error(format("no good signature on NAR info file ‘%1%’") % narInfoFile);
-    }
-
     return std::shared_ptr<NarInfo>(narInfo);
 }
 
@@ -268,7 +264,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     info.path = makeFixedOutputPath(recursive, hashAlgo, h, name);
 
     if (repair || !isValidPath(info.path))
-        addToCache(info, *sink.s);
+        addToCache(info, sink.s);
 
     return info.path;
 }
@@ -283,7 +279,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToCache(info, *sink.s);
+        addToCache(info, sink.s);
     }
 
     return info.path;
@@ -313,7 +309,7 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
         StringSink sink;
         dumpPath(storePath, sink);
 
-        addToCache(*info, *sink.s);
+        addToCache(*info, sink.s);
     }
 }
 
@@ -411,7 +407,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> a
     bool haveSignature = readInt(source) == 1;
     assert(!haveSignature);
 
-    addToCache(info, *tee.data);
+    addToCache(info, tee.data);
 
     auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
     if (accessor_)
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 4e4346a43438..46a38a1e0fc3 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -16,13 +16,15 @@ class BinaryCacheStore : public Store
 private:
 
     std::unique_ptr<SecretKey> secretKey;
-    std::unique_ptr<PublicKeys> publicKeys;
 
     std::shared_ptr<Store> localStore;
 
+    std::string compression;
+
 protected:
 
-    BinaryCacheStore(std::shared_ptr<Store> localStore, const Path & secretKeyFile);
+    BinaryCacheStore(std::shared_ptr<Store> localStore,
+        const StoreParams & params);
 
     [[noreturn]] void notImpl();
 
@@ -44,7 +46,7 @@ private:
 
     std::string narInfoFileFor(const Path & storePath);
 
-    void addToCache(const ValidPathInfo & info, const string & nar);
+    void addToCache(const ValidPathInfo & info, ref<std::string> nar);
 
 public:
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ae8078069d07..65df2eea59a0 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -8,11 +8,14 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "finally.hh"
 
 #include <algorithm>
 #include <iostream>
 #include <map>
 #include <sstream>
+#include <thread>
+#include <future>
 
 #include <limits.h>
 #include <time.h>
@@ -199,8 +202,6 @@ struct Child
     time_t timeStarted;
 };
 
-typedef map<pid_t, Child> Children;
-
 
 /* The worker class. */
 class Worker
@@ -220,7 +221,7 @@ private:
     WeakGoals wantingToBuild;
 
     /* Child processes currently running. */
-    Children children;
+    std::list<Child> children;
 
     /* Number of build slots occupied.  This includes local builds and
        substitutions but not remote builds via the build hook. */
@@ -278,14 +279,14 @@ public:
 
     /* Registers a running child process.  `inBuildSlot' means that
        the process counts towards the jobs limit. */
-    void childStarted(GoalPtr goal, pid_t pid,
-        const set<int> & fds, bool inBuildSlot, bool respectTimeouts);
+    void childStarted(GoalPtr goal, const set<int> & fds,
+        bool inBuildSlot, bool respectTimeouts);
 
     /* Unregisters a running child process.  `wakeSleepers' should be
        false if there is no sense in waking up goals that are sleeping
        because they can't run yet (e.g., there is no free build slot,
        or the hook would still say `postpone'). */
-    void childTerminated(pid_t pid, bool wakeSleepers = true);
+    void childTerminated(GoalPtr goal, bool wakeSleepers = true);
 
     /* Put `goal' to sleep until a build slot becomes available (which
        might be right away). */
@@ -633,7 +634,6 @@ HookInstance::HookInstance()
             baseNameOf(buildHook),
             settings.thisSystem,
             (format("%1%") % settings.maxSilentTime).str(),
-            (format("%1%") % settings.printBuildTrace).str(),
             (format("%1%") % settings.buildTimeout).str()
         };
 
@@ -748,6 +748,12 @@ private:
     /* Number of bytes received from the builder's stdout/stderr. */
     unsigned long logSize;
 
+    /* The most recent log lines. */
+    std::list<std::string> logTail;
+
+    std::string currentLogLine;
+    size_t currentLogLinePos = 0; // to handle carriage return
+
     /* Pipe for the builder's standard output/error. */
     Pipe builderOut;
 
@@ -873,6 +879,7 @@ private:
     /* Callback used by the worker to write to the log. */
     void handleChildOutput(int fd, const string & data) override;
     void handleEOF(int fd) override;
+    void flushLine();
 
     /* Return the set of (in)valid paths. */
     PathSet checkPathValidity(bool returnValid, bool checkHash);
@@ -936,7 +943,7 @@ DerivationGoal::~DerivationGoal()
 void DerivationGoal::killChild()
 {
     if (pid != -1) {
-        worker.childTerminated(pid);
+        worker.childTerminated(shared_from_this());
 
         if (buildUser.enabled()) {
             /* If we're using a build user, then there is a tricky
@@ -960,8 +967,6 @@ void DerivationGoal::killChild()
 
 void DerivationGoal::timedOut()
 {
-    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
     killChild();
     done(BuildResult::TimedOut);
 }
@@ -1362,9 +1367,6 @@ void DerivationGoal::tryToBuild()
         printMsg(lvlError, e.msg());
         outputLocks.unlock();
         buildUser.release();
-        if (settings.printBuildTrace)
-            printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
-                % drvPath % 0 % e.msg());
         worker.permanentFailure = true;
         done(BuildResult::InputRejected, e.msg());
         return;
@@ -1402,22 +1404,14 @@ void DerivationGoal::buildDone()
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe --- just don't do that
        :-) */
-    int status;
-    pid_t savedPid;
-    if (hook) {
-        savedPid = hook->pid;
-        status = hook->pid.wait(true);
-    } else {
-        /* !!! this could block! security problem! solution: kill the
-           child */
-        savedPid = pid;
-        status = pid.wait(true);
-    }
+    /* !!! this could block! security problem! solution: kill the
+       child */
+    int status = hook ? hook->pid.wait(true) : pid.wait(true);
 
     debug(format("builder process for ‘%1%’ finished") % drvPath);
 
     /* So the child is gone now. */
-    worker.childTerminated(savedPid);
+    worker.childTerminated(shared_from_this());
 
     /* Close the read side of the logger pipe. */
     if (hook) {
@@ -1468,11 +1462,19 @@ void DerivationGoal::buildDone()
                     if (pathExists(chrootRootDir + i))
                         rename((chrootRootDir + i).c_str(), i.c_str());
 
+            std::string msg = (format("builder for ‘%1%’ %2%")
+                % drvPath % statusToString(status)).str();
+
+            if (!settings.verboseBuild && !logTail.empty()) {
+                msg += (format("; last %d log lines:") % logTail.size()).str();
+                for (auto & line : logTail)
+                    msg += "\n  " + line;
+            }
+
             if (diskFull)
-                printMsg(lvlError, "note: build failure may have been caused by lack of free disk space");
+                msg += "\nnote: build failure may have been caused by lack of free disk space";
 
-            throw BuildError(format("builder for ‘%1%’ %2%")
-                % drvPath % statusToString(status));
+            throw BuildError(msg);
         }
 
         /* Compute the FS closure of the outputs and register them as
@@ -1517,23 +1519,13 @@ void DerivationGoal::buildDone()
 
         BuildResult::Status st = BuildResult::MiscFailure;
 
-        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) {
-            if (settings.printBuildTrace)
-                printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath);
+        if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
             st = BuildResult::TimedOut;
-        }
 
         else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
-            if (settings.printBuildTrace)
-                printMsg(lvlError, format("@ hook-failed %1% - %2% %3%")
-                    % drvPath % status % e.msg());
         }
 
         else {
-            if (settings.printBuildTrace)
-                printMsg(lvlError, format("@ build-failed %1% - %2% %3%")
-                    % drvPath % 1 % e.msg());
-
             st =
                 dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
                 statusOk(status) ? BuildResult::OutputRejected :
@@ -1548,9 +1540,6 @@ void DerivationGoal::buildDone()
     /* Release the build user, if applicable. */
     buildUser.release();
 
-    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath);
-
     done(BuildResult::Built);
 }
 
@@ -1625,11 +1614,7 @@ HookReply DerivationGoal::tryBuildHook()
     set<int> fds;
     fds.insert(hook->fromHook.readSide);
     fds.insert(hook->builderOut.readSide);
-    worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
-
-    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
-            % drvPath % drv->platform % logFile);
+    worker.childStarted(shared_from_this(), fds, false, false);
 
     return rpAccept;
 }
@@ -1657,12 +1642,10 @@ void DerivationGoal::startBuilder()
         nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" :
         "building path(s) %1%");
     f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
-    startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds);
+    printMsg(lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds);
 
     /* Right platform? */
     if (!drv->canBuildLocally()) {
-        if (settings.printBuildTrace)
-            printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform);
         throw Error(
             format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’")
             % drv->platform % settings.thisSystem % drvPath);
@@ -2165,7 +2148,7 @@ void DerivationGoal::startBuilder()
     /* parent */
     pid.setSeparatePG(true);
     builderOut.writeSide.close();
-    worker.childStarted(shared_from_this(), pid,
+    worker.childStarted(shared_from_this(),
         singleton<set<int> >(builderOut.readSide), true, true);
 
     /* Check if setting up the build environment failed. */
@@ -2177,11 +2160,6 @@ void DerivationGoal::startBuilder()
         }
         printMsg(lvlDebug, msg);
     }
-
-    if (settings.printBuildTrace) {
-        printMsg(lvlError, format("@ build-started %1% - %2% %3%")
-            % drvPath % drv->platform % logFile);
-    }
 }
 
 
@@ -2192,8 +2170,6 @@ void DerivationGoal::runChild()
 
     try { /* child */
 
-        logType = ltFlat;
-
         commonChildInit(builderOut);
 
 #if __linux__
@@ -2535,7 +2511,6 @@ void DerivationGoal::runChild()
         /* Execute the program.  This should not return. */
         if (drv->isBuiltin()) {
             try {
-                logType = ltFlat;
                 if (drv->builder == "builtin:fetchurl")
                     builtinFetchurl(*drv);
                 else
@@ -2667,8 +2642,7 @@ void DerivationGoal::registerOutputs()
             rewritten = true;
         }
 
-        startNest(nest, lvlTalkative,
-            format("scanning for references inside ‘%1%’") % path);
+        Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path);
 
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
@@ -2954,8 +2928,18 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
             done(BuildResult::LogLimitExceeded);
             return;
         }
-        if (verbosity >= settings.buildVerbosity)
-            writeToStderr(filterANSIEscapes(data, true));
+
+        for (auto c : data)
+            if (c == '\r')
+                currentLogLinePos = 0;
+            else if (c == '\n')
+                flushLine();
+            else {
+                if (currentLogLinePos >= currentLogLine.size())
+                    currentLogLine.resize(currentLogLinePos + 1);
+                currentLogLine[currentLogLinePos++] = c;
+            }
+
         if (bzLogFile) {
             int err;
             BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
@@ -2965,16 +2949,30 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
     }
 
     if (hook && fd == hook->fromHook.readSide)
-        writeToStderr(data);
+        printMsg(lvlError, data); // FIXME?
 }
 
 
 void DerivationGoal::handleEOF(int fd)
 {
+    if (!currentLogLine.empty()) flushLine();
     worker.wakeUp(shared_from_this());
 }
 
 
+void DerivationGoal::flushLine()
+{
+    if (settings.verboseBuild)
+        printMsg(lvlInfo, filterANSIEscapes(currentLogLine, true));
+    else {
+        logTail.push_back(currentLogLine);
+        if (logTail.size() > settings.logLines) logTail.pop_front();
+    }
+    currentLogLine = "";
+    currentLogLinePos = 0;
+}
+
+
 PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
 {
     PathSet result;
@@ -3027,28 +3025,24 @@ private:
     Path storePath;
 
     /* The remaining substituters. */
-    Paths subs;
+    std::list<ref<Store>> subs;
 
     /* The current substituter. */
-    Path sub;
+    std::shared_ptr<Store> sub;
 
-    /* Whether any substituter can realise this path */
+    /* Whether any substituter can realise this path. */
     bool hasSubstitute;
 
     /* Path info returned by the substituter's query info operation. */
-    SubstitutablePathInfo info;
+    std::shared_ptr<const ValidPathInfo> info;
 
     /* Pipe for the substituter's standard output. */
     Pipe outPipe;
 
-    /* Pipe for the substituter's standard error. */
-    Pipe logPipe;
+    /* The substituter thread. */
+    std::thread thr;
 
-    /* The process ID of the builder. */
-    Pid pid;
-
-    /* Lock on the store path. */
-    std::shared_ptr<PathLocks> outputLock;
+    std::promise<void> promise;
 
     /* Whether to try to repair a valid path. */
     bool repair;
@@ -3064,7 +3058,7 @@ public:
     SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
     ~SubstitutionGoal();
 
-    void timedOut();
+    void timedOut() { abort(); };
 
     string key()
     {
@@ -3105,20 +3099,14 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool
 
 SubstitutionGoal::~SubstitutionGoal()
 {
-    if (pid != -1) worker.childTerminated(pid);
-}
-
-
-void SubstitutionGoal::timedOut()
-{
-    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath);
-    if (pid != -1) {
-        pid_t savedPid = pid;
-        pid.kill();
-        worker.childTerminated(savedPid);
+    try {
+        if (thr.joinable()) {
+            thr.join();
+            worker.childTerminated(shared_from_this());
+        }
+    } catch (...) {
+        ignoreException();
     }
-    amDone(ecFailed);
 }
 
 
@@ -3143,7 +3131,7 @@ void SubstitutionGoal::init()
     if (settings.readOnlyMode)
         throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath);
 
-    subs = settings.substituters;
+    subs = getDefaultSubstituters();
 
     tryNext();
 }
@@ -3168,17 +3156,19 @@ void SubstitutionGoal::tryNext()
     sub = subs.front();
     subs.pop_front();
 
-    SubstitutablePathInfos infos;
-    PathSet dummy(singleton<PathSet>(storePath));
-    worker.store.querySubstitutablePathInfos(sub, dummy, infos);
-    SubstitutablePathInfos::iterator k = infos.find(storePath);
-    if (k == infos.end()) { tryNext(); return; }
-    info = k->second;
+    try {
+        // FIXME: make async
+        info = sub->queryPathInfo(storePath);
+    } catch (InvalidPath &) {
+        tryNext();
+        return;
+    }
+
     hasSubstitute = true;
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    for (auto & i : info.references)
+    for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
             addWaitee(worker.makeSubstitutionGoal(i));
 
@@ -3199,7 +3189,7 @@ void SubstitutionGoal::referencesValid()
         return;
     }
 
-    for (auto & i : info.references)
+    for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
             assert(worker.store.isValidPath(i));
 
@@ -3221,75 +3211,32 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
-    /* Maybe a derivation goal has already locked this path
-       (exceedingly unlikely, since it should have used a substitute
-       first, but let's be defensive). */
-    outputLock.reset(); // make sure this goal's lock is gone
-    if (pathIsLockedByMe(storePath)) {
-        debug(format("restarting substitution of ‘%1%’ because it's locked by another goal")
-            % storePath);
-        worker.waitForAnyGoal(shared_from_this());
-        return; /* restart in the tryToRun() state when another goal finishes */
-    }
-
-    /* Acquire a lock on the output path. */
-    outputLock = std::make_shared<PathLocks>();
-    if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) {
-        worker.waitForAWhile(shared_from_this());
-        return;
-    }
-
-    /* Check again whether the path is invalid. */
-    if (!repair && worker.store.isValidPath(storePath)) {
-        debug(format("store path ‘%1%’ has become valid") % storePath);
-        outputLock->setDeletion(true);
-        amDone(ecSuccess);
-        return;
-    }
-
     printMsg(lvlInfo, format("fetching path ‘%1%’...") % storePath);
 
     outPipe.create();
-    logPipe.create();
 
-    destPath = repair ? storePath + ".tmp" : storePath;
+    promise = std::promise<void>();
 
-    /* Remove the (stale) output path if it exists. */
-    deletePath(destPath);
+    thr = std::thread([this]() {
+        try {
+            /* Wake up the worker loop when we're done. */
+            Finally updateStats([this]() { outPipe.writeSide.close(); });
 
-    worker.store.setSubstituterEnv();
-
-    /* Fill in the arguments. */
-    Strings args;
-    args.push_back(baseNameOf(sub));
-    args.push_back("--substitute");
-    args.push_back(storePath);
-    args.push_back(destPath);
-
-    /* Fork the substitute program. */
-    pid = startProcess([&]() {
-
-        commonChildInit(logPipe);
-
-        if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
-            throw SysError("cannot dup output pipe into stdout");
+            StringSink sink;
+            sub->exportPaths({storePath}, false, sink);
 
-        execv(sub.c_str(), stringsToCharPtrs(args).data());
+            StringSource source(*sink.s);
+            worker.store.importPaths(false, source, 0);
 
-        throw SysError(format("executing ‘%1%’") % sub);
+            promise.set_value();
+        } catch (...) {
+            promise.set_exception(std::current_exception());
+        }
     });
 
-    pid.setSeparatePG(true);
-    pid.setKillSignal(SIGTERM);
-    outPipe.writeSide.close();
-    logPipe.writeSide.close();
-    worker.childStarted(shared_from_this(),
-        pid, singleton<set<int> >(logPipe.readSide), true, true);
+    worker.childStarted(shared_from_this(), {outPipe.readSide}, true, false);
 
     state = &SubstitutionGoal::finished;
-
-    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub);
 }
 
 
@@ -3297,110 +3244,40 @@ void SubstitutionGoal::finished()
 {
     trace("substitute finished");
 
-    /* Since we got an EOF on the logger pipe, the substitute is
-       presumed to have terminated.  */
-    pid_t savedPid = pid;
-    int status = pid.wait(true);
+    thr.join();
+    worker.childTerminated(shared_from_this());
 
-    /* So the child is gone now. */
-    worker.childTerminated(savedPid);
-
-    /* Close the read side of the logger pipe. */
-    logPipe.readSide.close();
-
-    /* Get the hash info from stdout. */
-    string dummy = readLine(outPipe.readSide);
-    string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
-    outPipe.readSide.close();
-
-    /* Check the exit status and the build result. */
-    HashResult hash;
     try {
-
-        if (!statusOk(status))
-            throw SubstError(format("fetching path ‘%1%’ %2%")
-                % storePath % statusToString(status));
-
-        if (!pathExists(destPath))
-            throw SubstError(format("substitute did not produce path ‘%1%’") % destPath);
-
-        hash = hashPath(htSHA256, destPath);
-
-        /* Verify the expected hash we got from the substituer. */
-        if (expectedHashStr != "") {
-            size_t n = expectedHashStr.find(':');
-            if (n == string::npos)
-                throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
-            HashType hashType = parseHashType(string(expectedHashStr, 0, n));
-            if (hashType == htUnknown)
-                throw Error(format("unknown hash algorithm in ‘%1%’") % expectedHashStr);
-            Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
-            Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first;
-            if (expectedHash != actualHash)
-                throw SubstError(format("hash mismatch in downloaded path ‘%1%’: expected %2%, got %3%")
-                    % storePath % printHash(expectedHash) % printHash(actualHash));
-        }
-
-    } catch (SubstError & e) {
-
+        promise.get_future().get();
+    } catch (Error & e) {
         printMsg(lvlInfo, e.msg());
 
-        if (settings.printBuildTrace) {
-            printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
-                % storePath % status % e.msg());
-        }
-
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
         worker.wakeUp(shared_from_this());
         return;
     }
 
-    if (repair) replaceValidPath(storePath, destPath);
-
-    canonicalisePathMetaData(storePath, -1);
-
-    worker.store.optimisePath(storePath); // FIXME: combine with hashPath()
-
-    ValidPathInfo info2;
-    info2.path = storePath;
-    info2.narHash = hash.first;
-    info2.narSize = hash.second;
-    info2.references = info.references;
-    info2.deriver = info.deriver;
-    worker.store.registerValidPath(info2);
-
-    outputLock->setDeletion(true);
-    outputLock.reset();
-
     worker.markContentsGood(storePath);
 
     printMsg(lvlChatty,
         format("substitution of path ‘%1%’ succeeded") % storePath);
 
-    if (settings.printBuildTrace)
-        printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
-
     amDone(ecSuccess);
 }
 
 
 void SubstitutionGoal::handleChildOutput(int fd, const string & data)
 {
-    assert(fd == logPipe.readSide);
-    if (verbosity >= settings.buildVerbosity) writeToStderr(data);
-    /* Don't write substitution output to a log file for now.  We
-       probably should, though. */
 }
 
 
 void SubstitutionGoal::handleEOF(int fd)
 {
-    if (fd == logPipe.readSide) worker.wakeUp(shared_from_this());
+    if (fd == outPipe.readSide) worker.wakeUp(shared_from_this());
 }
 
 
-
 //////////////////////////////////////////////////////////////////////
 
 
@@ -3516,9 +3393,8 @@ unsigned Worker::getNrLocalBuilds()
 }
 
 
-void Worker::childStarted(GoalPtr goal,
-    pid_t pid, const set<int> & fds, bool inBuildSlot,
-    bool respectTimeouts)
+void Worker::childStarted(GoalPtr goal, const set<int> & fds,
+    bool inBuildSlot, bool respectTimeouts)
 {
     Child child;
     child.goal = goal;
@@ -3526,30 +3402,29 @@ void Worker::childStarted(GoalPtr goal,
     child.timeStarted = child.lastOutput = time(0);
     child.inBuildSlot = inBuildSlot;
     child.respectTimeouts = respectTimeouts;
-    children[pid] = child;
+    children.emplace_back(child);
     if (inBuildSlot) nrLocalBuilds++;
 }
 
 
-void Worker::childTerminated(pid_t pid, bool wakeSleepers)
+void Worker::childTerminated(GoalPtr goal, bool wakeSleepers)
 {
-    assert(pid != -1); /* common mistake */
-
-    Children::iterator i = children.find(pid);
+    auto i = std::find_if(children.begin(), children.end(),
+        [&](const Child & child) { return child.goal.lock() == goal; });
     assert(i != children.end());
 
-    if (i->second.inBuildSlot) {
+    if (i->inBuildSlot) {
         assert(nrLocalBuilds > 0);
         nrLocalBuilds--;
     }
 
-    children.erase(pid);
+    children.erase(i);
 
     if (wakeSleepers) {
 
         /* Wake up goals waiting for a build slot. */
-        for (auto & i : wantingToBuild) {
-            GoalPtr goal = i.lock();
+        for (auto & j : wantingToBuild) {
+            GoalPtr goal = j.lock();
             if (goal) wakeUp(goal);
         }
 
@@ -3586,7 +3461,7 @@ void Worker::run(const Goals & _topGoals)
 {
     for (auto & i : _topGoals) topGoals.insert(i);
 
-    startNest(nest, lvlDebug, format("entered goal loop"));
+    Activity act(*logger, lvlDebug, "entered goal loop");
 
     while (1) {
 
@@ -3651,11 +3526,11 @@ void Worker::waitForInput()
     assert(sizeof(time_t) >= sizeof(long));
     time_t nearest = LONG_MAX; // nearest deadline
     for (auto & i : children) {
-        if (!i.second.respectTimeouts) continue;
+        if (!i.respectTimeouts) continue;
         if (settings.maxSilentTime != 0)
-            nearest = std::min(nearest, i.second.lastOutput + settings.maxSilentTime);
+            nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime);
         if (settings.buildTimeout != 0)
-            nearest = std::min(nearest, i.second.timeStarted + settings.buildTimeout);
+            nearest = std::min(nearest, i.timeStarted + settings.buildTimeout);
     }
     if (nearest != LONG_MAX) {
         timeout.tv_sec = std::max((time_t) 1, nearest - before);
@@ -3673,7 +3548,6 @@ void Worker::waitForInput()
         timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before));
     } else lastWokenUp = 0;
 
-    using namespace std;
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
@@ -3681,7 +3555,7 @@ void Worker::waitForInput()
     FD_ZERO(&fds);
     int fdMax = 0;
     for (auto & i : children) {
-        for (auto & j : i.second.fds) {
+        for (auto & j : i.fds) {
             FD_SET(j, &fds);
             if (j >= fdMax) fdMax = j + 1;
         }
@@ -3695,22 +3569,16 @@ void Worker::waitForInput()
     time_t after = time(0);
 
     /* Process all available file descriptors. */
+    decltype(children)::iterator i;
+    for (auto j = children.begin(); j != children.end(); j = i) {
+        i = std::next(j);
 
-    /* Since goals may be canceled from inside the loop below (causing
-       them go be erased from the `children' map), we have to be
-       careful that we don't keep iterators alive across calls to
-       timedOut(). */
-    set<pid_t> pids;
-    for (auto & i : children) pids.insert(i.first);
-
-    for (auto & i : pids) {
         checkInterrupt();
-        Children::iterator j = children.find(i);
-        if (j == children.end()) continue; // child destroyed
-        GoalPtr goal = j->second.goal.lock();
+
+        GoalPtr goal = j->goal.lock();
         assert(goal);
 
-        set<int> fds2(j->second.fds);
+        set<int> fds2(j->fds);
         for (auto & k : fds2) {
             if (FD_ISSET(k, &fds)) {
                 unsigned char buffer[4096];
@@ -3722,12 +3590,12 @@ void Worker::waitForInput()
                 } else if (rd == 0) {
                     debug(format("%1%: got EOF") % goal->getName());
                     goal->handleEOF(k);
-                    j->second.fds.erase(k);
+                    j->fds.erase(k);
                 } else {
                     printMsg(lvlVomit, format("%1%: read %2% bytes")
                         % goal->getName() % rd);
                     string data((char *) buffer, rd);
-                    j->second.lastOutput = after;
+                    j->lastOutput = after;
                     goal->handleChildOutput(k, data);
                 }
             }
@@ -3735,8 +3603,8 @@ void Worker::waitForInput()
 
         if (goal->getExitCode() == Goal::ecBusy &&
             settings.maxSilentTime != 0 &&
-            j->second.respectTimeouts &&
-            after - j->second.lastOutput >= (time_t) settings.maxSilentTime)
+            j->respectTimeouts &&
+            after - j->lastOutput >= (time_t) settings.maxSilentTime)
         {
             printMsg(lvlError,
                 format("%1% timed out after %2% seconds of silence")
@@ -3746,8 +3614,8 @@ void Worker::waitForInput()
 
         else if (goal->getExitCode() == Goal::ecBusy &&
             settings.buildTimeout != 0 &&
-            j->second.respectTimeouts &&
-            after - j->second.timeStarted >= (time_t) settings.buildTimeout)
+            j->respectTimeouts &&
+            after - j->timeStarted >= (time_t) settings.buildTimeout)
         {
             printMsg(lvlError,
                 format("%1% timed out after %2% seconds")
@@ -3804,8 +3672,6 @@ void Worker::markContentsGood(const Path & path)
 
 void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 {
-    startNest(nest, lvlDebug, format("building %1%") % showPaths(drvPaths));
-
     Worker worker(*this);
 
     Goals goals;
@@ -3835,8 +3701,6 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
 BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
-    startNest(nest, lvlDebug, format("building %1%") % showPaths({drvPath}));
-
     Worker worker(*this);
     auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode);
 
diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc
index 50417b644a02..a4785d6905bb 100644
--- a/src/libstore/builtins.cc
+++ b/src/libstore/builtins.cc
@@ -31,7 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv)
     auto unpack = drv.env.find("unpack");
     if (unpack != drv.env.end() && unpack->second == "1") {
         if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6))
-            data.data = decompressXZ(*data.data);
+            data.data = decompress("xz", ref<std::string>(data.data));
         StringSource source(*data.data);
         restorePath(storePath, source);
     } else
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index eed630517d11..6e39330e40d9 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -313,7 +313,7 @@ bool isUri(const string & s)
     size_t pos = s.find("://");
     if (pos == string::npos) return false;
     string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel";
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git";
 }
 
 
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 1536dcb590e0..8fc582f4c20d 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -305,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 
         else if (type == DT_REG) {
             Path storePath = settings.nixStore + "/" + baseNameOf(path);
-            if (isValidPath(storePath))
+            if (isStorePath(storePath) && isValidPath(storePath))
                 roots[path] = storePath;
         }
 
@@ -514,7 +514,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     if (path == linksDir || path == state.trashDir) return;
 
-    startNest(nest, lvlDebug, format("considering whether to delete ‘%1%’") % path);
+    Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path);
 
     if (!isValidPath(path)) {
         /* A lock file belonging to a path that we're building right
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index b6078253319f..c12178e4028a 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -28,7 +28,6 @@ Settings::Settings()
     keepFailed = false;
     keepGoing = false;
     tryFallback = false;
-    buildVerbosity = lvlError;
     maxBuildJobs = 1;
     buildCores = 1;
 #ifdef _SC_NPROCESSORS_ONLN
@@ -40,7 +39,6 @@ Settings::Settings()
     maxSilentTime = 0;
     buildTimeout = 0;
     useBuildHook = true;
-    printBuildTrace = false;
     reservedSize = 8 * 1024 * 1024;
     fsyncMetadata = true;
     useSQLiteWAL = true;
@@ -186,19 +184,6 @@ void Settings::update()
     _get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
     _get(useCaseHack, "use-case-hack");
     _get(preBuildHook, "pre-build-hook");
-
-    string subs = getEnv("NIX_SUBSTITUTERS", "default");
-    if (subs == "default") {
-        substituters.clear();
-#if 0
-        if (getEnv("NIX_OTHER_STORES") != "")
-            substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
-#endif
-        substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
-        if (useSshSubstituter && !sshSubstituterHosts.empty())
-            substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh");
-    } else
-        substituters = tokenizeString<Strings>(subs, ":");
 }
 
 
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 572fa7188c14..65f763ace3c7 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "types.hh"
+#include "logging.hh"
 
 #include <map>
 #include <sys/types.h>
@@ -77,8 +78,12 @@ struct Settings {
        instead. */
     bool tryFallback;
 
-    /* Verbosity level for build output. */
-    Verbosity buildVerbosity;
+    /* Whether to show build log output in real time. */
+    bool verboseBuild = true;
+
+    /* If verboseBuild is false, the number of lines of the tail of
+       the log to show if a build fails. */
+    size_t logLines = 10;
 
     /* Maximum number of parallel build jobs.  0 means unlimited. */
     unsigned int maxBuildJobs;
@@ -105,31 +110,10 @@ struct Settings {
        means infinity.  */
     time_t buildTimeout;
 
-    /* The substituters.  There are programs that can somehow realise
-       a store path without building, e.g., by downloading it or
-       copying it from a CD. */
-    Paths substituters;
-
     /* Whether to use build hooks (for distributed builds).  Sometimes
        users want to disable this from the command-line. */
     bool useBuildHook;
 
-    /* Whether buildDerivations() should print out lines on stderr in
-       a fixed format to allow its progress to be monitored.  Each
-       line starts with a "@".  The following are defined:
-
-       @ build-started <drvpath> <outpath> <system> <logfile>
-       @ build-failed <drvpath> <outpath> <exitcode> <error text>
-       @ build-succeeded <drvpath> <outpath>
-       @ substituter-started <outpath> <substituter>
-       @ substituter-failed <outpath> <exitcode> <error text>
-       @ substituter-succeeded <outpath>
-
-       Best combined with --no-build-output, otherwise stderr might
-       conceivably contain lines in this format printed by the
-       builders. */
-    bool printBuildTrace;
-
     /* Amount of reserved space for the garbage collector
        (/nix/var/nix/db/reserved). */
     off_t reservedSize;
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 771eb42ee561..92d94aeeacd5 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -16,8 +16,8 @@ private:
 public:
 
     HttpBinaryCacheStore(std::shared_ptr<Store> localStore,
-        const Path & secretKeyFile, const Path & _cacheUri)
-        : BinaryCacheStore(localStore, secretKeyFile)
+        const StoreParams & params, const Path & _cacheUri)
+        : BinaryCacheStore(localStore, params)
         , cacheUri(_cacheUri)
         , downloaders(
             std::numeric_limits<size_t>::max(),
@@ -85,12 +85,14 @@ protected:
 
 };
 
-static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> {
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const StoreParams & params)
+    -> std::shared_ptr<Store>
+{
     if (std::string(uri, 0, 7) != "http://" &&
         std::string(uri, 0, 8) != "https://") return 0;
     auto store = std::make_shared<HttpBinaryCacheStore>(std::shared_ptr<Store>(0),
-        settings.get("binary-cache-secret-key-file", string("")),
-        uri);
+        params, uri);
     store->init();
     return store;
 });
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 7968c98b9558..2c2944938761 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -12,10 +12,19 @@ private:
 public:
 
     LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
-        const Path & secretKeyFile, const Path & binaryCacheDir);
+        const StoreParams & params, const Path & binaryCacheDir)
+        : BinaryCacheStore(localStore, params)
+        , binaryCacheDir(binaryCacheDir)
+    {
+    }
 
     void init() override;
 
+    std::string getUri() override
+    {
+        return "file://" + binaryCacheDir;
+    }
+
 protected:
 
     bool fileExists(const std::string & path) override;
@@ -24,14 +33,21 @@ protected:
 
     std::shared_ptr<std::string> getFile(const std::string & path) override;
 
-};
+    PathSet queryAllValidPaths() override
+    {
+        PathSet paths;
 
-LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
-    const Path & secretKeyFile, const Path & binaryCacheDir)
-    : BinaryCacheStore(localStore, secretKeyFile)
-    , binaryCacheDir(binaryCacheDir)
-{
-}
+        for (auto & entry : readDirectory(binaryCacheDir)) {
+            if (entry.name.size() != 40 ||
+                !hasSuffix(entry.name, ".narinfo"))
+                continue;
+            paths.insert(settings.nixStore + "/" + entry.name.substr(0, entry.name.size() - 8));
+        }
+
+        return paths;
+    }
+
+};
 
 void LocalBinaryCacheStore::init()
 {
@@ -69,20 +85,15 @@ std::shared_ptr<std::string> LocalBinaryCacheStore::getFile(const std::string &
     }
 }
 
-ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore,
-    const Path & secretKeyFile, const Path & binaryCacheDir)
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const StoreParams & params)
+    -> std::shared_ptr<Store>
 {
-    auto store = make_ref<LocalBinaryCacheStore>(
-        localStore, secretKeyFile, binaryCacheDir);
+    if (std::string(uri, 0, 7) != "file://") return 0;
+    auto store = std::make_shared<LocalBinaryCacheStore>(
+        std::shared_ptr<Store>(0), params, std::string(uri, 7));
     store->init();
     return store;
-}
-
-static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> {
-    if (std::string(uri, 0, 7) != "file://") return 0;
-    return openLocalBinaryCacheStore(std::shared_ptr<Store>(0),
-        settings.get("binary-cache-secret-key-file", string("")),
-        std::string(uri, 7));
 });
 
 }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index baf3a528b705..01a11f11f65d 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -5,12 +5,11 @@
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
 #include "derivations.hh"
-#include "affinity.hh"
+#include "nar-info.hh"
 
 #include <iostream>
 #include <algorithm>
 #include <cstring>
-#include <atomic>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -220,19 +219,6 @@ LocalStore::~LocalStore()
     auto state(_state.lock());
 
     try {
-        for (auto & i : state->runningSubstituters) {
-            if (i.second.disabled) continue;
-            i.second.to.close();
-            i.second.from.close();
-            i.second.error.close();
-            if (i.second.pid != -1)
-                i.second.pid.wait(true);
-        }
-    } catch (...) {
-        ignoreException();
-    }
-
-    try {
         if (state->fdTempRoots != -1) {
             state->fdTempRoots.close();
             unlink(state->fnTempRoots.c_str());
@@ -243,6 +229,12 @@ LocalStore::~LocalStore()
 }
 
 
+std::string LocalStore::getUri()
+{
+    return "local";
+}
+
+
 int LocalStore::getSchema()
 {
     int curSchema = 0;
@@ -792,205 +784,42 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
 }
 
 
-void LocalStore::setSubstituterEnv()
-{
-    static std::atomic_flag done;
-
-    if (done.test_and_set()) return;
-
-    /* Pass configuration options (including those overridden with
-       --option) to substituters. */
-    setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
-}
-
-
-void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
-{
-    if (run.disabled || run.pid != -1) return;
-
-    debug(format("starting substituter program ‘%1%’") % substituter);
-
-    Pipe toPipe, fromPipe, errorPipe;
-
-    toPipe.create();
-    fromPipe.create();
-    errorPipe.create();
-
-    setSubstituterEnv();
-
-    run.pid = startProcess([&]() {
-        if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
-            throw SysError("dupping stdin");
-        if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
-            throw SysError("dupping stdout");
-        if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
-            throw SysError("dupping stderr");
-        execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
-        throw SysError(format("executing ‘%1%’") % substituter);
-    });
-
-    run.program = baseNameOf(substituter);
-    run.to = toPipe.writeSide.borrow();
-    run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
-    run.error = errorPipe.readSide.borrow();
-
-    toPipe.readSide.close();
-    fromPipe.writeSide.close();
-    errorPipe.writeSide.close();
-
-    /* The substituter may exit right away if it's disabled in any way
-       (e.g. copy-from-other-stores.pl will exit if no other stores
-       are configured). */
-    try {
-        getLineFromSubstituter(run);
-    } catch (EndOfFile & e) {
-        run.to.close();
-        run.from.close();
-        run.error.close();
-        run.disabled = true;
-        if (run.pid.wait(true) != 0) throw;
-    }
-}
-
-
-/* Read a line from the substituter's stdout, while also processing
-   its stderr. */
-string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
-{
-    string res, err;
-
-    /* We might have stdout data left over from the last time. */
-    if (run.fromBuf.hasData()) goto haveData;
-
-    while (1) {
-        checkInterrupt();
-
-        fd_set fds;
-        FD_ZERO(&fds);
-        FD_SET(run.from, &fds);
-        FD_SET(run.error, &fds);
-
-        /* Wait for data to appear on the substituter's stdout or
-           stderr. */
-        if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) {
-            if (errno == EINTR) continue;
-            throw SysError("waiting for input from the substituter");
-        }
-
-        /* Completely drain stderr before dealing with stdout. */
-        if (FD_ISSET(run.error, &fds)) {
-            char buf[4096];
-            ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf));
-            if (n == -1) {
-                if (errno == EINTR) continue;
-                throw SysError("reading from substituter's stderr");
-            }
-            if (n == 0) throw EndOfFile(format("substituter ‘%1%’ died unexpectedly") % run.program);
-            err.append(buf, n);
-            string::size_type p;
-            while ((p = err.find('\n')) != string::npos) {
-                printMsg(lvlError, run.program + ": " + string(err, 0, p));
-                err = string(err, p + 1);
-            }
-        }
-
-        /* Read from stdout until we get a newline or the buffer is empty. */
-        else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) {
-        haveData:
-            do {
-                unsigned char c;
-                run.fromBuf(&c, 1);
-                if (c == '\n') {
-                    if (!err.empty()) printMsg(lvlError, run.program + ": " + err);
-                    return res;
-                }
-                res += c;
-            } while (run.fromBuf.hasData());
-        }
-    }
-}
-
-
-template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run)
-{
-    string s = getLineFromSubstituter(run);
-    T res;
-    if (!string2Int(s, res)) throw Error("integer expected from stream");
-    return res;
-}
-
-
 PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 {
-    auto state(_state.lock());
-
     PathSet res;
-    for (auto & i : settings.substituters) {
-        if (res.size() == paths.size()) break;
-        RunningSubstituter & run(state->runningSubstituters[i]);
-        startSubstituter(i, run);
-        if (run.disabled) continue;
-        string s = "have ";
-        for (auto & j : paths)
-            if (res.find(j) == res.end()) { s += j; s += " "; }
-        writeLine(run.to, s);
-        while (true) {
-            /* FIXME: we only read stderr when an error occurs, so
-               substituters should only write (short) messages to
-               stderr when they fail.  I.e. they shouldn't write debug
-               output. */
-            Path path = getLineFromSubstituter(run);
-            if (path == "") break;
-            res.insert(path);
+    for (auto & sub : getDefaultSubstituters()) {
+        for (auto & path : paths) {
+            if (res.count(path)) continue;
+            debug(format("checking substituter ‘%s’ for path ‘%s’")
+                % sub->getUri() % path);
+            if (sub->isValidPath(path))
+                res.insert(path);
         }
     }
-
     return res;
 }
 
 
-void LocalStore::querySubstitutablePathInfos(const Path & substituter,
-    PathSet & paths, SubstitutablePathInfos & infos)
-{
-    auto state(_state.lock());
-
-    RunningSubstituter & run(state->runningSubstituters[substituter]);
-    startSubstituter(substituter, run);
-    if (run.disabled) return;
-
-    string s = "info ";
-    for (auto & i : paths)
-        if (infos.find(i) == infos.end()) { s += i; s += " "; }
-    writeLine(run.to, s);
-
-    while (true) {
-        Path path = getLineFromSubstituter(run);
-        if (path == "") break;
-        if (paths.find(path) == paths.end())
-            throw Error(format("got unexpected path ‘%1%’ from substituter") % path);
-        paths.erase(path);
-        SubstitutablePathInfo & info(infos[path]);
-        info.deriver = getLineFromSubstituter(run);
-        if (info.deriver != "") assertStorePath(info.deriver);
-        int nrRefs = getIntLineFromSubstituter<int>(run);
-        while (nrRefs--) {
-            Path p = getLineFromSubstituter(run);
-            assertStorePath(p);
-            info.references.insert(p);
-        }
-        info.downloadSize = getIntLineFromSubstituter<long long>(run);
-        info.narSize = getIntLineFromSubstituter<long long>(run);
-    }
-}
-
-
 void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
     SubstitutablePathInfos & infos)
 {
-    PathSet todo = paths;
-    for (auto & i : settings.substituters) {
-        if (todo.empty()) break;
-        querySubstitutablePathInfos(i, todo, infos);
+    for (auto & sub : getDefaultSubstituters()) {
+        for (auto & path : paths) {
+            if (infos.count(path)) continue;
+            debug(format("checking substituter ‘%s’ for path ‘%s’")
+                % sub->getUri() % path);
+            try {
+                auto info = sub->queryPathInfo(path);
+                auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
+                    std::shared_ptr<const ValidPathInfo>(info));
+                infos[path] = SubstitutablePathInfo{
+                    info->deriver,
+                    info->references,
+                    narInfo ? narInfo->fileSize : 0,
+                    info->narSize};
+            } catch (InvalidPath) {
+            }
+        }
     }
 }
 
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index daf394c92805..6f2341decfbd 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -40,17 +40,6 @@ struct OptimiseStats
 };
 
 
-struct RunningSubstituter
-{
-    Path program;
-    Pid pid;
-    AutoCloseFD to, from, error;
-    FdSource fromBuf;
-    bool disabled;
-    RunningSubstituter() : disabled(false) { };
-};
-
-
 class LocalStore : public LocalFSStore
 {
 private:
@@ -80,10 +69,6 @@ private:
         /* The file to which we write our temporary roots. */
         Path fnTempRoots;
         AutoCloseFD fdTempRoots;
-
-        typedef std::map<Path, RunningSubstituter> RunningSubstituters;
-        RunningSubstituters runningSubstituters;
-
     };
 
     Sync<State, std::recursive_mutex> _state;
@@ -102,6 +87,8 @@ public:
 
     /* Implementations of abstract store API methods. */
 
+    std::string getUri() override;
+
     bool isValidPathUncached(const Path & path) override;
 
     PathSet queryValidPaths(const PathSet & paths) override;
@@ -122,9 +109,6 @@ public:
 
     PathSet querySubstitutablePaths(const PathSet & paths) override;
 
-    void querySubstitutablePathInfos(const Path & substituter,
-        PathSet & paths, SubstitutablePathInfos & infos);
-
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos) override;
 
@@ -192,8 +176,6 @@ public:
        a substituter (if available). */
     void repairPath(const Path & path);
 
-    void setSubstituterEnv();
-
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
     static bool haveWriteAccess();
@@ -246,13 +228,6 @@ private:
 
     void removeUnusedLinks(const GCState & state);
 
-    void startSubstituter(const Path & substituter,
-        RunningSubstituter & runningSubstituter);
-
-    string getLineFromSubstituter(RunningSubstituter & run);
-
-    template<class T> T getIntLineFromSubstituter(RunningSubstituter & run);
-
     Path createTempDirInStore();
 
     Path importPath(bool requireSignature, Source & source);
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 15fa91b5cea6..22b0f235e0b2 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc)
 
 libstore_LIBS = libutil libformat
 
-libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -laws-cpp-sdk-s3 -laws-cpp-sdk-core
+libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -laws-cpp-sdk-s3 -laws-cpp-sdk-core -pthread
 
 ifeq ($(OS), SunOS)
 	libstore_LDFLAGS += -lsocket
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 23cbe7e26b47..ad7fe0e8bebf 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -228,7 +228,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
     for (auto & i : paths) {
         addTempRoot(i);
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
-        startNest(nest, lvlChatty, format("hashing files in ‘%1%’") % i);
+        Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i);
         optimisePath_(stats, i, inodeHash);
     }
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 1edf3662e8b8..5a254a6104f4 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -49,6 +49,12 @@ RemoteStore::RemoteStore(size_t maxConnections)
 }
 
 
+std::string RemoteStore::getUri()
+{
+    return "daemon";
+}
+
+
 ref<RemoteStore::Connection> RemoteStore::openConnection()
 {
     auto conn = make_ref<Connection>();
@@ -120,9 +126,9 @@ void RemoteStore::setOptions(ref<Connection> conn)
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 2)
         conn->to << settings.useBuildHook;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 4)
-        conn->to << settings.buildVerbosity
-           << logType
-           << settings.printBuildTrace;
+        conn->to << (settings.verboseBuild ? lvlError : lvlVomit)
+                 << 0 // obsolete log type
+                 << 0 /* obsolete print build trace */;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 6)
         conn->to << settings.buildCores;
     if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 10)
@@ -561,10 +567,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
             writeString(buf, source->read(buf, len), to);
             to.flush();
         }
-        else {
-            string s = readString(from);
-            writeToStderr(s);
-        }
+        else
+            printMsg(lvlError, chomp(readString(from)));
     }
     if (msg == STDERR_ERROR) {
         string error = readString(from);
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 4ea8e8a5be4d..8e45a7449e2e 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -26,6 +26,8 @@ public:
 
     /* Implementations of abstract store API methods. */
 
+    std::string getUri() override;
+
     bool isValidPathUncached(const Path & path) override;
 
     PathSet queryValidPaths(const PathSet & paths) override;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 6fadf7be28c9..cffcb1bf214f 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -43,8 +43,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     Stats stats;
 
     S3BinaryCacheStoreImpl(std::shared_ptr<Store> localStore,
-        const Path & secretKeyFile, const std::string & bucketName)
-        : S3BinaryCacheStore(localStore, secretKeyFile)
+        const StoreParams & params, const std::string & bucketName)
+        : S3BinaryCacheStore(localStore, params)
         , bucketName(bucketName)
         , config(makeConfig())
         , client(make_ref<Aws::S3::S3Client>(*config))
@@ -227,8 +227,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
             for (auto object : contents) {
                 auto & key = object.GetKey();
-                if (!hasSuffix(key, ".narinfo")) continue;
-                paths.insert(settings.nixStore + "/" + std::string(key, 0, key.size() - 8));
+                if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
+                paths.insert(settings.nixStore + "/" + key.substr(0, key.size() - 8));
             }
 
             marker = res.GetNextMarker();
@@ -239,11 +239,13 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 
 };
 
-static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> {
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const StoreParams & params)
+    -> std::shared_ptr<Store>
+{
     if (std::string(uri, 0, 5) != "s3://") return 0;
     auto store = std::make_shared<S3BinaryCacheStoreImpl>(std::shared_ptr<Store>(0),
-        settings.get("binary-cache-secret-key-file", string("")),
-        std::string(uri, 5));
+        params, std::string(uri, 5));
     store->init();
     return store;
 });
diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh
index 0425f6bb95d9..2751a9d01cdb 100644
--- a/src/libstore/s3-binary-cache-store.hh
+++ b/src/libstore/s3-binary-cache-store.hh
@@ -11,8 +11,8 @@ class S3BinaryCacheStore : public BinaryCacheStore
 protected:
 
     S3BinaryCacheStore(std::shared_ptr<Store> localStore,
-        const Path & secretKeyFile)
-        : BinaryCacheStore(localStore, secretKeyFile)
+        const StoreParams & params)
+        : BinaryCacheStore(localStore, params)
     { }
 
 public:
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 14932f9b0be7..463e132e0299 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -461,10 +461,22 @@ namespace nix {
 RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
 
 
-ref<Store> openStoreAt(const std::string & uri)
+ref<Store> openStoreAt(const std::string & uri_)
 {
+    auto uri(uri_);
+    StoreParams params;
+    auto q = uri.find('?');
+    if (q != std::string::npos) {
+        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
+            auto e = s.find('=');
+            if (e != std::string::npos)
+                params[s.substr(0, e)] = s.substr(e + 1);
+        }
+        uri = uri_.substr(0, q);
+    }
+
     for (auto fun : *RegisterStoreImplementation::implementations) {
-        auto store = fun(uri);
+        auto store = fun(uri, params);
         if (store) return ref<Store>(store);
     }
 
@@ -478,7 +490,10 @@ ref<Store> openStore()
 }
 
 
-static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> {
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const StoreParams & params)
+    -> std::shared_ptr<Store>
+{
     enum { mDaemon, mLocal, mAuto } mode;
 
     if (uri == "daemon") mode = mDaemon;
@@ -501,4 +516,39 @@ static RegisterStoreImplementation regStore([](const std::string & uri) -> std::
 });
 
 
+std::list<ref<Store>> getDefaultSubstituters()
+{
+    struct State {
+        bool done = false;
+        std::list<ref<Store>> stores;
+    };
+    static Sync<State> state_;
+
+    auto state(state_.lock());
+
+    if (state->done) return state->stores;
+
+    StringSet done;
+
+    auto addStore = [&](const std::string & uri) {
+        if (done.count(uri)) return;
+        done.insert(uri);
+        state->stores.push_back(openStoreAt(uri));
+    };
+
+    for (auto uri : settings.get("substituters", Strings()))
+        addStore(uri);
+
+    for (auto uri : settings.get("binary-caches", Strings()))
+        addStore(uri);
+
+    for (auto uri : settings.get("extra-binary-caches", Strings()))
+        addStore(uri);
+
+    state->done = true;
+
+    return state->stores;
+}
+
+
 }
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index e0cc32296a9b..29685c9d1676 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -192,7 +192,7 @@ public:
 
     virtual ~Store() { }
 
-    virtual std::string getUri();
+    virtual std::string getUri() = 0;
 
     /* Check whether a path is valid. */
     bool isValidPath(const Path & path);
@@ -529,12 +529,17 @@ ref<Store> openStoreAt(const std::string & uri);
 ref<Store> openStore();
 
 
-ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore,
-    const Path & secretKeyFile, const Path & binaryCacheDir);
+/* Return the default substituter stores, defined by the
+   ‘substituters’ option and various legacy options like
+   ‘binary-caches’. */
+std::list<ref<Store>> getDefaultSubstituters();
 
 
 /* Store implementation registration. */
-typedef std::function<std::shared_ptr<Store>(const std::string & uri)> OpenStore;
+typedef std::map<std::string, std::string> StoreParams;
+
+typedef std::function<std::shared_ptr<Store>(
+    const std::string & uri, const StoreParams & params)> OpenStore;
 
 struct RegisterStoreImplementation
 {
generated by cgit-pink 1.4.1 (git 2.44.2) at 2025-01-28T10·02+0000