about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc2
-rw-r--r--src/libstore/binary-cache-store.hh2
-rw-r--r--src/libstore/build.cc136
-rw-r--r--src/libstore/download.cc46
-rw-r--r--src/libstore/download.hh2
-rw-r--r--src/libstore/gc.cc4
-rw-r--r--src/libstore/globals.cc44
-rw-r--r--src/libstore/globals.hh35
-rw-r--r--src/libstore/http-binary-cache-store.cc11
-rw-r--r--src/libstore/local-store.cc3
-rw-r--r--src/libstore/local-store.hh5
-rw-r--r--src/libstore/local.mk6
-rw-r--r--src/libstore/nar-accessor.hh2
-rw-r--r--src/libstore/pathlocks.cc6
-rw-r--r--src/libstore/pathlocks.hh6
-rw-r--r--src/libstore/remote-store.cc26
-rw-r--r--src/libstore/remote-store.hh2
-rw-r--r--src/libstore/s3-binary-cache-store.cc10
-rw-r--r--src/libstore/store-api.cc13
-rw-r--r--src/libstore/store-api.hh23
20 files changed, 294 insertions, 90 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ab971dd8b6d9..d1b278b8efbe 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, *nar);
+    auto narCompressed = compress(compression, *nar, parallelCompression);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 8492ff600eba..e20b968442b7 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -19,6 +19,8 @@ public:
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
+    const Setting<bool> parallelCompression{this, false, "parallel-compression",
+        "enable multi-threading compression, available for xz only currently"};
 
 private:
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d4bd650baf22..1d611ffbaba5 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
@@ -48,7 +49,9 @@
 #include <sys/param.h>
 #include <sys/mount.h>
 #include <sys/syscall.h>
+#if HAVE_SECCOMP
 #include <seccomp.h>
+#endif
 #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
 #endif
 
@@ -1123,11 +1126,6 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    /* Reject doing a hash build of anything other than a fixed-output
-       derivation. */
-    if (buildMode == bmHash && !drv->isFixedOutput())
-        throw Error("cannot do a hash build of non-fixed-output derivation '%1%'", drvPath);
-
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -1319,9 +1317,7 @@ void DerivationGoal::inputsRealised()
     allPaths.insert(inputPaths.begin(), inputPaths.end());
 
     /* Is this a fixed-output derivation? */
-    fixedOutput = true;
-    for (auto & i : drv->outputs)
-        if (i.second.hash == "") fixedOutput = false;
+    fixedOutput = drv->isFixedOutput();
 
     /* Don't repeat fixed-output derivations since they're already
        verified by their output hash.*/
@@ -1341,19 +1337,6 @@ void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
 
-    /* Check for the possibility that some other goal in this process
-       has locked the output since we checked in haveDerivation().
-       (It can't happen between here and the lockPaths() call below
-       because we're not allowing multi-threading.)  If so, put this
-       goal to sleep until another goal finishes, then try again. */
-    for (auto & i : drv->outputs)
-        if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) {
-            debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal")
-                % drvPath % i.second.path);
-            worker.waitForAnyGoal(shared_from_this());
-            return;
-        }
-
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes.  If we
        can't acquire the lock, then continue; hopefully some other
@@ -1777,6 +1760,19 @@ PathSet exportReferences(Store & store, PathSet storePaths)
     return paths;
 }
 
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a download of an invalid URL to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    std::call_once(dns_resolve_flag, []() {
+        DownloadRequest request("http://this.pre-initializes.the.dns.resolvers.invalid");
+        request.tries = 1; // We only need to do it once, and this also suppresses an annoying warning
+        try { getDownloader()->download(request); } catch (...) {}
+    });
+}
 
 void DerivationGoal::startBuilder()
 {
@@ -1787,6 +1783,9 @@ void DerivationGoal::startBuilder()
             % drv->platform % settings.thisSystem % drvPath);
     }
 
+    if (drv->isBuiltin())
+        preloadNSS();
+
 #if __APPLE__
     additionalSandboxProfile = get(drv->env, "__sandboxProfile");
 #endif
@@ -1810,8 +1809,13 @@ void DerivationGoal::startBuilder()
             useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1";
     }
 
-    if (worker.store.storeDir != worker.store.realStoreDir)
-        useChroot = true;
+    if (worker.store.storeDir != worker.store.realStoreDir) {
+        #if __linux__
+            useChroot = true;
+        #else
+            throw Error("building using a diverted store is not supported on this platform");
+        #endif
+    }
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
@@ -2469,7 +2473,7 @@ void setupSeccomp()
 {
 #if __linux__
     if (!settings.filterSyscalls) return;
-
+#if HAVE_SECCOMP
     scmp_filter_ctx ctx;
 
     if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
@@ -2515,6 +2519,11 @@ void setupSeccomp()
 
     if (seccomp_load(ctx) != 0)
         throw SysError("unable to load seccomp BPF program");
+#else
+    throw Error(
+        "seccomp is not supported on this platform; "
+        "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
+#endif
 #endif
 }
 
@@ -2928,8 +2937,13 @@ void DerivationGoal::runChild()
         if (drv->isBuiltin()) {
             try {
                 logger = makeJSONLogger(*logger);
+
+                BasicDerivation drv2(*drv);
+                for (auto & e : drv2.env)
+                    e.second = rewriteStrings(e.second, inputRewrites);
+
                 if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(*drv, netrcData);
+                    builtinFetchurl(drv2, netrcData);
                 else
                     throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8));
                 _exit(0);
@@ -2992,6 +3006,8 @@ void DerivationGoal::registerOutputs()
     bool runDiffHook = settings.runDiffHook;
     bool keepPreviousRound = settings.keepFailed || runDiffHook;
 
+    std::exception_ptr delayedException;
+
     /* Check whether the output paths were created, and grep each
        output path to determine what other paths it references.  Also make all
        output paths read-only. */
@@ -3066,7 +3082,7 @@ void DerivationGoal::registerOutputs()
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */
-        if (i.second.hash != "") {
+        if (fixedOutput) {
 
             bool recursive; Hash h;
             i.second.parseHashInfo(recursive, h);
@@ -3082,27 +3098,34 @@ void DerivationGoal::registerOutputs()
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
             Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath);
-            if (buildMode == bmHash) {
-                Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
-                printError(format("build produced path '%1%' with %2% hash '%3%'")
-                    % dest % printHashType(h.type) % printHash16or32(h2));
-                if (worker.store.isValidPath(dest))
-                    return;
+
+            Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]);
+
+            if (h != h2) {
+
+                /* Throw an error after registering the path as
+                   valid. */
+                delayedException = std::make_exception_ptr(
+                    BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'",
+                        dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h)));
+
                 Path actualDest = worker.store.toRealPath(dest);
+
+                if (worker.store.isValidPath(dest))
+                    std::rethrow_exception(delayedException);
+
                 if (actualPath != actualDest) {
                     PathLocks outputLocks({actualDest});
                     deletePath(actualDest);
                     if (rename(actualPath.c_str(), actualDest.c_str()) == -1)
                         throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest);
                 }
+
                 path = dest;
                 actualPath = actualDest;
-            } else {
-                if (h != h2)
-                    throw BuildError(
-                        format("output path '%1%' has %2% hash '%3%' when '%4%' was expected")
-                        % path % i.second.hashAlgo % printHash16or32(h2) % printHash16or32(h));
             }
+            else
+                assert(path == dest);
 
             info.ca = makeFixedOutputCA(recursive, h2);
         }
@@ -3279,6 +3302,11 @@ void DerivationGoal::registerOutputs()
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
     worker.store.registerValidPaths(infos);
+
+    /* In case of a fixed-output derivation hash mismatch, throw an
+       exception now that we have registered the output as valid. */
+    if (delayedException)
+        std::rethrow_exception(delayedException);
 }
 
 
@@ -3394,7 +3422,7 @@ void DerivationGoal::flushLine()
     else {
         if (settings.verboseBuild &&
             (settings.printRepeatedBuilds || curRound == 1))
-            printError(filterANSIEscapes(currentLogLine, true));
+            printError(currentLogLine);
         else {
             logTail.push_back(currentLogLine);
             if (logTail.size() > settings.logLines) logTail.pop_front();
@@ -3636,7 +3664,7 @@ void SubstitutionGoal::tryNext()
     /* Update the total expected download size. */
     auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
 
-    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, narInfo->narSize);
+    maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>(worker.expectedNarSize, info->narSize);
 
     maintainExpectedDownload =
         narInfo && narInfo->fileSize
@@ -3650,9 +3678,12 @@ void SubstitutionGoal::tryNext()
     /* Bail out early if this substituter lacks a valid
        signature. LocalStore::addToStore() also checks for this, but
        only after we've downloaded the path. */
-    if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) {
-        printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
-            % sub->getUri() % storePath);
+    if (worker.store.requireSigs
+        && !sub->isTrusted
+        && !info->checkSignatures(worker.store, worker.store.publicKeys))
+    {
+        printError("warning: substituter '%s' does not have a valid signature for path '%s'",
+            sub->getUri(), storePath);
         tryNext();
         return;
     }
@@ -3702,6 +3733,17 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
+    /* If the store path is already locked (probably by a
+       DerivationGoal), then put this goal to sleep. Note: we don't
+       acquire a lock here since that breaks addToStore(), so below we
+       handle an AlreadyLocked exception from addToStore(). The check
+       here is just an optimisation to prevent having to redo a
+       download due to a locked path. */
+    if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
+        worker.waitForAWhile(shared_from_this());
+        return;
+    }
+
     maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
     worker.updateProgress();
 
@@ -3718,7 +3760,7 @@ void SubstitutionGoal::tryToRun()
             PushActivity pact(act.id);
 
             copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
-                storePath, repair);
+                storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
 
             promise.set_value();
         } catch (...) {
@@ -3741,8 +3783,14 @@ void SubstitutionGoal::finished()
 
     try {
         promise.get_future().get();
+    } catch (AlreadyLocked & e) {
+        /* Probably a DerivationGoal is already building this store
+           path. Sleep for a while and try again. */
+        state = &SubstitutionGoal::init;
+        worker.waitForAWhile(shared_from_this());
+        return;
     } catch (Error & e) {
-        printInfo(e.msg());
+        printError(e.msg());
 
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 4474dfd4b968..5ab625f42288 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -17,11 +17,13 @@
 
 #include <curl/curl.h>
 
-#include <queue>
-#include <iostream>
-#include <thread>
+#include <algorithm>
 #include <cmath>
+#include <cstring>
+#include <iostream>
+#include <queue>
 #include <random>
+#include <thread>
 
 using namespace std::string_literals;
 
@@ -91,6 +93,8 @@ struct CurlDownloader : public Downloader
         {
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
+            if (!request.mimeType.empty())
+                requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
         }
 
         ~DownloadItem()
@@ -185,6 +189,22 @@ struct CurlDownloader : public Downloader
             return 0;
         }
 
+        size_t readOffset = 0;
+        int readCallback(char *buffer, size_t size, size_t nitems)
+        {
+            if (readOffset == request.data->length())
+                return 0;
+            auto count = std::min(size * nitems, request.data->length() - readOffset);
+            memcpy(buffer, request.data->data() + readOffset, count);
+            readOffset += count;
+            return count;
+        }
+
+        static int readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
+        {
+            return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+        }
+
         long lowSpeedTimeout = 300;
 
         void init()
@@ -225,6 +245,13 @@ struct CurlDownloader : public Downloader
             if (request.head)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
+            if (request.data) {
+                curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
+                curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
+                curl_easy_setopt(req, CURLOPT_READDATA, this);
+                curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
+            }
+
             if (request.verifyTLS) {
                 if (settings.caFile != "")
                     curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
@@ -265,7 +292,7 @@ struct CurlDownloader : public Downloader
             }
 
             if (code == CURLE_OK &&
-                (httpStatus == 200 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
+                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 done = true;
@@ -303,6 +330,7 @@ struct CurlDownloader : public Downloader
                     // Don't bother retrying on certain cURL errors either
                     switch (code) {
                         case CURLE_FAILED_INIT:
+                        case CURLE_URL_MALFORMAT:
                         case CURLE_NOT_BUILT_IN:
                         case CURLE_REMOTE_ACCESS_DENIED:
                         case CURLE_FILE_COULDNT_READ_FILE:
@@ -311,10 +339,10 @@ struct CurlDownloader : public Downloader
                         case CURLE_BAD_FUNCTION_ARGUMENT:
                         case CURLE_INTERFACE_FAILED:
                         case CURLE_UNKNOWN_OPTION:
-                        err = Misc;
-                        break;
+                            err = Misc;
+                            break;
                         default: // Shut up warnings
-                        break;
+                            break;
                     }
                 }
 
@@ -689,7 +717,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
             // FIXME: this requires GNU tar for decompression.
-            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"});
+            runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"});
             unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair);
         }
         replaceSymlink(unpackedStorePath, unpackedLink);
@@ -699,7 +727,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
     if (expectedStorePath != "" && storePath != expectedStorePath)
         throw nix::Error("store path mismatch in file downloaded from '%s'", url);
 
-    return storePath;
+    return store->toRealPath(storePath);
 }
 
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index f2d65ad8d61d..d9d525d4e65f 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -18,6 +18,8 @@ struct DownloadRequest
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
+    std::shared_ptr<std::string> data;
+    std::string mimeType;
 
     DownloadRequest(const std::string & uri)
         : uri(uri), parentAct(curActivity) { }
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index ab2c5ca0274c..943b16c28fa3 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -324,10 +324,8 @@ Roots LocalStore::findRootsNoTemp()
 {
     Roots roots;
 
-    /* Process direct roots in {gcroots,manifests,profiles}. */
+    /* Process direct roots in {gcroots,profiles}. */
     findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
-    if (pathExists(stateDir + "/manifests"))
-        findRoots(stateDir + "/manifests", DT_UNKNOWN, roots);
     findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
 
     /* Add additional roots returned by the program specified by the
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index d3c96ddd6e61..f46e8326235f 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <map>
 #include <thread>
+#include <dlfcn.h>
 
 
 namespace nix {
@@ -37,6 +38,7 @@ Settings::Settings()
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR)))
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)))
+    , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
@@ -137,4 +139,46 @@ void MaxBuildJobsSetting::set(const std::string & str)
         throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
 }
 
+
+void initPlugins()
+{
+    for (const auto & pluginFile : settings.pluginFiles.get()) {
+        Paths pluginFiles;
+        try {
+            auto ents = readDirectory(pluginFile);
+            for (const auto & ent : ents)
+                pluginFiles.emplace_back(pluginFile + "/" + ent.name);
+        } catch (SysError & e) {
+            if (e.errNo != ENOTDIR)
+                throw;
+            pluginFiles.emplace_back(pluginFile);
+        }
+        for (const auto & file : pluginFiles) {
+            /* handle is purposefully leaked as there may be state in the
+               DSO needed by the action of the plugin. */
+            void *handle =
+                dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
+            if (!handle)
+                throw Error("could not dynamically open plugin file '%s%': %s%", file, dlerror());
+        }
+    }
+    /* We handle settings registrations here, since plugins can add settings */
+    if (RegisterSetting::settingRegistrations) {
+        for (auto & registration : *RegisterSetting::settingRegistrations)
+            settings.addSetting(registration);
+        delete RegisterSetting::settingRegistrations;
+    }
+    settings.handleUnknownSettings();
+}
+
+RegisterSetting::SettingRegistrations * RegisterSetting::settingRegistrations;
+
+RegisterSetting::RegisterSetting(AbstractSetting * s)
+{
+    if (!settingRegistrations)
+        settingRegistrations = new SettingRegistrations;
+    settingRegistrations->emplace_back(s);
+}
+
+
 }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 5c857cbb6a9c..dd01f832df0c 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -82,6 +82,9 @@ public:
     /* The directory where the main programs are stored. */
     Path nixBinDir;
 
+    /* The directory where the man pages are stored. */
+    Path nixManDir;
+
     /* File name of the socket the daemon listens to.  */
     Path nixDaemonSocketFile;
 
@@ -138,6 +141,11 @@ public:
     Setting<std::string> builders{this, "@" + nixConfDir + "/machines", "builders",
         "A semicolon-separated list of build machines, in the format of nix.machines."};
 
+    Setting<bool> buildersUseSubstitutes{this, false, "builders-use-substitutes",
+        "Whether build machines should use their own substitutes for obtaining "
+        "build dependencies if possible, rather than waiting for this host to "
+        "upload them."};
+
     Setting<off_t> reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space",
         "Amount of reserved disk space for the garbage collector."};
 
@@ -150,7 +158,7 @@ public:
     Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
         "Whether to call sync() before registering a path as valid."};
 
-    Setting<bool> useSubstitutes{this, true, "use-substitutes",
+    Setting<bool> useSubstitutes{this, true, "substitute",
         "Whether to use substitutes.",
         {"build-use-substitutes"}};
 
@@ -206,7 +214,8 @@ public:
     bool lockCPU;
 
     /* Whether to show a stack trace if Nix evaluation fails. */
-    bool showTrace = false;
+    Setting<bool> showTrace{this, false, "show-trace",
+        "Whether to show a stack trace on evaluation errors."};
 
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
@@ -227,6 +236,9 @@ public:
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and network access to the URI prefixes listed in 'allowed-uris'."};
 
+    Setting<bool> pureEval{this, false, "pure-eval",
+        "Whether to restrict file system and network access to files specified by cryptographic hash."};
+
     Setting<size_t> buildRepeat{this, 0, "repeat",
         "The number of times to repeat a build in order to verify determinism.",
         {"build-repeat"}};
@@ -278,10 +290,7 @@ public:
     Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl",
         "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."};
 
-    Setting<std::string> signedBinaryCaches{this, "*", "signed-binary-caches",
-        "Obsolete."};
-
-    Setting<bool> requireSigs{this, signedBinaryCaches == "*", "require-sigs",
+    Setting<bool> requireSigs{this, true, "require-sigs",
         "Whether to check that any non-content-addressed path added to the "
         "Nix store has a valid signature (that is, one signed using a key "
         "listed in 'trusted-public-keys'."};
@@ -361,14 +370,28 @@ public:
 
     Setting<Strings> allowedUris{this, {}, "allowed-uris",
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
+
+    Setting<Paths> pluginFiles{this, {}, "plugin-files",
+        "Plugins to dynamically load at nix initialization time."};
 };
 
 
 // FIXME: don't use a global variable.
 extern Settings settings;
 
+/* This should be called after settings are initialized, but before
+   anything else */
+void initPlugins();
+
 
 extern const string nixVersion;
 
+struct RegisterSetting
+{
+    typedef std::vector<AbstractSetting *> SettingRegistrations;
+    static SettingRegistrations * settingRegistrations;
+    RegisterSetting(AbstractSetting * s);
+};
+
 
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 057337685791..b9e9cd5daba5 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -38,7 +38,7 @@ public:
             try {
                 BinaryCacheStore::init();
             } catch (UploadToHTTP &) {
-                throw Error(format("'%s' does not appear to be a binary cache") % cacheUri);
+                throw Error("'%s' does not appear to be a binary cache", cacheUri);
             }
             diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority);
         }
@@ -67,7 +67,14 @@ protected:
         const std::string & data,
         const std::string & mimeType) override
     {
-        throw UploadToHTTP("uploading to an HTTP binary cache is not supported");
+        auto req = DownloadRequest(cacheUri + "/" + path);
+        req.data = std::make_shared<string>(data); // FIXME: inefficient
+        req.mimeType = mimeType;
+        try {
+            getDownloader()->download(req);
+        } catch (DownloadError & e) {
+            throw UploadToHTTP(format("uploading to HTTP binary cache at %1% not supported: %2%") % cacheUri % e.msg());
+        }
     }
 
     void getFile(const std::string & path,
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7afecc1cfc62..4afe51ea91ec 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -992,8 +992,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
            lock on this path). */
-        Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
-        if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
+        if (!locksHeld.count(info.path))
             outputLock.lockPaths({realPath});
 
         if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index d35cd1a949eb..bbd50e1c1451 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -19,7 +19,7 @@ namespace nix {
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
    Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
-   Nix 1.0.  Version 7 is Nix 1.3. Version 10 is 1.12. */
+   Nix 1.0.  Version 7 is Nix 1.3. Version 10 is 2.0. */
 const int nixSchemaVersion = 10;
 
 
@@ -104,6 +104,9 @@ private:
 
 public:
 
+    // Hack for build-remote.cc.
+    PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+
     /* Initialise the local store, upgrading the schema if
        necessary. */
     LocalStore(const Params & params);
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 50c46ce6fe99..e11efa5c2b54 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -9,6 +9,9 @@ libstore_SOURCES := $(wildcard $(d)/*.cc)
 libstore_LIBS = libutil libformat
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+ifneq ($(OS), FreeBSD)
+ libstore_LDFLAGS += -ldl
+endif
 
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 
@@ -22,7 +25,7 @@ ifeq ($(OS), SunOS)
 	libstore_LDFLAGS += -lsocket
 endif
 
-ifeq ($(OS), Linux)
+ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 
@@ -35,6 +38,7 @@ libstore_CXXFLAGS = \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
+ -DNIX_MAN_DIR=\"$(mandir)\" \
  -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \
  -DLSOF=\"$(lsof)\"
 
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 1903355a236e..2871199de16e 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <functional>
+
 #include "fs-accessor.hh"
 
 namespace nix {
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 587f29598851..08d1efdbeb01 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths,
 
         {
             auto lockedPaths(lockedPaths_.lock());
-            if (lockedPaths->count(lockPath))
-                throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            if (lockedPaths->count(lockPath)) {
+                if (!wait) return false;
+                throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+            }
             lockedPaths->insert(lockPath);
         }
 
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 2a7de611446e..db51f950a320 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -2,10 +2,8 @@
 
 #include "util.hh"
 
-
 namespace nix {
 
-
 /* Open (possibly create) a lock file and return the file descriptor.
    -1 is returned if create is false and the lock could not be opened
    because it doesn't exist.  Any other error throws an exception. */
@@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 
+MakeError(AlreadyLocked, Error);
 
 class PathLocks
 {
@@ -38,9 +37,6 @@ public:
     void setDeletion(bool deletePaths);
 };
 
-
-// FIXME: not thread-safe!
 bool pathIsLockedByMe(const Path & path);
 
-
 }
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 77b41b6bf8a8..8f0b65557ac4 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -78,9 +78,22 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
 }
 
 
+UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params)
+    : Store(params)
+    , LocalFSStore(params)
+    , RemoteStore(params)
+    , path(socket_path)
+{
+}
+
+
 std::string UDSRemoteStore::getUri()
 {
-    return "daemon";
+    if (path) {
+        return std::string("unix://") + *path;
+    } else {
+        return "daemon";
+    }
 }
 
 
@@ -98,7 +111,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
         throw SysError("cannot create Unix domain socket");
     closeOnExec(conn->fd.get());
 
-    string socketPath = settings.nixDaemonSocketFile;
+    string socketPath = path ? *path : settings.nixDaemonSocketFile;
 
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
@@ -721,5 +734,14 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
     }
 }
 
+static std::string uriScheme = "unix://";
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params);
+});
 
 }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 30c6beae6ff2..7f36e206416b 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -134,6 +134,7 @@ class UDSRemoteStore : public LocalFSStore, public RemoteStore
 public:
 
     UDSRemoteStore(const Params & params);
+    UDSRemoteStore(std::string path, const Params & params);
 
     std::string getUri() override;
 
@@ -145,6 +146,7 @@ private:
     };
 
     ref<RemoteStore::Connection> openConnection() override;
+    std::experimental::optional<std::string> path;
 };
 
 
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 0079da1becfb..23af452094cf 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -10,6 +10,7 @@
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
+#include <aws/core/VersionConfig.h>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/ClientConfiguration.h>
@@ -87,7 +88,14 @@ S3Helper::S3Helper(const std::string & profile, const std::string & region)
                 std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
             : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
                 std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
-            *config, true, false))
+            *config,
+            // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759
+#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3
+            false,
+#else
+            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
+#endif
+            false))
 {
 }
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 77ab87ef728e..8830edcc3449 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
 }
 
 
-std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
-    bool recursive, HashType hashAlgo, PathFilter & filter) const
+std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
+    const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
 {
     Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
-    string name = baseNameOf(srcPath);
     Path dstPath = makeFixedOutputPath(recursive, h, name);
     return std::pair<Path, Hash>(dstPath, h);
 }
@@ -840,7 +839,7 @@ ref<Store> openStore(const std::string & uri_,
     for (auto fun : *RegisterStoreImplementation::implementations) {
         auto store = fun(uri, params);
         if (store) {
-            store->warnUnknownSettings();
+            store->handleUnknownSettings();
             return ref<Store>(store);
         }
     }
@@ -897,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters()
         auto addStore = [&](const std::string & uri) {
             if (done.count(uri)) return;
             done.insert(uri);
-            stores.push_back(openStore(uri));
+            try {
+                stores.push_back(openStore(uri));
+            } catch (Error & e) {
+                printError("warning: %s", e.what());
+            }
         };
 
         for (auto uri : settings.substituters.get())
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index d1e1b5d6f452..563aa566bd37 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -192,7 +192,7 @@ struct ValidPathInfo
 typedef list<ValidPathInfo> ValidPathInfos;
 
 
-enum BuildMode { bmNormal, bmRepair, bmCheck, bmHash };
+enum BuildMode { bmNormal, bmRepair, bmCheck };
 
 
 struct BuildResult
@@ -248,6 +248,8 @@ public:
 
     const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"};
 
+    const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"};
+
 protected:
 
     struct State
@@ -305,9 +307,9 @@ public:
     /* This is the preparatory part of addToStore(); it computes the
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
-    std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-        bool recursive = true, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter) const;
+    std::pair<Path, Hash> computeStorePathForPath(const string & name,
+        const Path & srcPath, bool recursive = true,
+        HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
     /* Preparatory part of addTextToStore().
 
@@ -597,6 +599,11 @@ public:
        "nix-cache-info" file. Lower value means higher priority. */
     virtual int getPriority() { return 0; }
 
+    virtual Path toRealPath(const Path & storePath)
+    {
+        return storePath;
+    }
+
 protected:
 
     Stats stats;
@@ -639,9 +646,10 @@ public:
 
     virtual Path getRealStoreDir() { return storeDir; }
 
-    Path toRealPath(const Path & storePath)
+    Path toRealPath(const Path & storePath) override
     {
-        return getRealStoreDir() + "/" + baseNameOf(storePath);
+        assert(isInStore(storePath));
+        return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
     }
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;
@@ -699,6 +707,9 @@ void removeTempRoots();
    * ‘daemon’: The Nix store accessed via a Unix domain socket
      connection to nix-daemon.
 
+   * ‘unix://<path>’: The Nix store accessed via a Unix domain socket
+     connection to nix-daemon, with the socket located at <path>.
+
    * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
      whether the user has write access to the local Nix
      store/database.
generated by cgit-pink 1.4.1 (git 2.44.2) at 2025-01-17T08·17+0000