about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc82
-rw-r--r--src/libstore/binary-cache-store.hh5
-rw-r--r--src/libstore/build.cc12
-rw-r--r--src/libstore/download.cc4
-rw-r--r--src/libstore/nar-accessor.cc37
-rw-r--r--src/libstore/nar-accessor.hh7
-rw-r--r--src/libstore/s3-binary-cache-store.cc19
-rw-r--r--src/libstore/s3.hh2
-rw-r--r--src/libstore/sandbox-defaults.sb35
-rw-r--r--src/libstore/store-api.cc14
10 files changed, 151 insertions, 66 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 67607ab3d43a..68af85bf16d9 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -73,6 +73,23 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
     return storePathToHash(storePath) + ".narinfo";
 }
 
+void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
+{
+    auto narInfoFile = narInfoFileFor(narInfo->path);
+
+    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
+
+    auto hashPart = storePathToHash(narInfo->path);
+
+    {
+        auto state_(state.lock());
+        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
+    }
+
+    if (diskCache)
+        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+}
+
 void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
 {
@@ -89,8 +106,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
                 % info.path % ref);
         }
 
-    auto narInfoFile = narInfoFileFor(info.path);
-
     assert(nar->compare(0, narMagic.size(), narMagic) == 0);
 
     auto narInfo = make_ref<NarInfo>(info);
@@ -119,42 +134,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
                 accessor_->addToCache(info.path, *nar);
             }
 
-            std::function<void(const Path &, JSONPlaceholder &)> recurse;
-
-            recurse = [&](const Path & path, JSONPlaceholder & res) {
-                auto st = narAccessor->stat(path);
-
-                auto obj = res.object();
-
-                switch (st.type) {
-                case FSAccessor::Type::tRegular:
-                    obj.attr("type", "regular");
-                    obj.attr("size", st.fileSize);
-                    if (st.isExecutable)
-                        obj.attr("executable", true);
-                    break;
-                case FSAccessor::Type::tDirectory:
-                    obj.attr("type", "directory");
-                    {
-                        auto res2 = obj.object("entries");
-                        for (auto & name : narAccessor->readDirectory(path)) {
-                            auto res3 = res2.placeholder(name);
-                            recurse(path + "/" + name, res3);
-                        }
-                    }
-                    break;
-                case FSAccessor::Type::tSymlink:
-                    obj.attr("type", "symlink");
-                    obj.attr("target", narAccessor->readLink(path));
-                    break;
-                default:
-                    abort();
-                }
-            };
-
             {
                 auto res = jsonRoot.placeholder("root");
-                recurse("", res);
+                listNar(res, narAccessor, "", true);
             }
         }
 
@@ -201,17 +183,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     /* Atomically write the NAR info file.*/
     if (secretKey) narInfo->sign(*secretKey);
 
-    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
-
-    auto hashPart = storePathToHash(narInfo->path);
-
-    {
-        auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
-    }
-
-    if (diskCache)
-        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+    writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
 }
@@ -326,6 +298,22 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
 }
 
+void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
+{
+    /* Note: this is inherently racy since there is no locking on
+       binary caches. In particular, with S3 this unreliable, even
+       when addSignatures() is called sequentially on a path, because
+       S3 might return an outdated cached version. */
+
+    auto narInfo = make_ref<NarInfo>((NarInfo &) *queryPathInfo(storePath));
+
+    narInfo->sigs.insert(sigs.begin(), sigs.end());
+
+    auto narInfoFile = narInfoFileFor(narInfo->path);
+
+    writeNarInfo(narInfo);
+}
+
 std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
 {
     Path drvPath;
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index d3b0e0bd9332..8492ff600eba 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -59,6 +59,8 @@ private:
 
     std::string narInfoFileFor(const Path & storePath);
 
+    void writeNarInfo(ref<NarInfo> narInfo);
+
 public:
 
     bool isValidPathUncached(const Path & path) override;
@@ -119,8 +121,7 @@ public:
 
     ref<FSAccessor> getFSAccessor() override;
 
-    void addSignatures(const Path & storePath, const StringSet & sigs) override
-    { unsupported(); }
+    void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
     std::shared_ptr<std::string> getBuildLog(const Path & path) override;
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9f8edc826a04..061682377257 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2833,10 +2833,10 @@ void DerivationGoal::runChild()
                     sandboxProfile += "(deny default (with no-log))\n";
                 }
 
-                sandboxProfile += "(import \"sandbox-defaults.sb\")";
+                sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
 
                 if (fixedOutput)
-                    sandboxProfile += "(import \"sandbox-network.sb\")";
+                    sandboxProfile += "(import \"sandbox-network.sb\")\n";
 
                 /* Our rwx outputs */
                 sandboxProfile += "(allow file-read* file-write* process-exec\n";
@@ -2879,7 +2879,7 @@ void DerivationGoal::runChild()
 
                 sandboxProfile += additionalSandboxProfile;
             } else
-                sandboxProfile += "(import \"sandbox-minimal.sb\")";
+                sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
 
             debug("Generated sandbox profile:");
             debug(sandboxProfile);
@@ -2888,6 +2888,8 @@ void DerivationGoal::runChild()
 
             writeFile(sandboxFile, sandboxProfile);
 
+            bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1";
+
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
             Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true);
@@ -2903,6 +2905,10 @@ void DerivationGoal::runChild()
             args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
             args.push_back("-D");
             args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
+            if (allowLocalNetworking) {
+                args.push_back("-D");
+                args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+            }
             args.push_back(drv->builder);
         }
 #endif
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 579a5e8c1b59..da31029b4f87 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -533,7 +533,7 @@ struct CurlDownloader : public Downloader
             // FIXME: do this on a worker thread
             sync2async<DownloadResult>(success, failure, [&]() -> DownloadResult {
 #ifdef ENABLE_S3
-                S3Helper s3Helper(Aws::Region::US_EAST_1); // FIXME: make configurable
+                S3Helper s3Helper("", Aws::Region::US_EAST_1); // FIXME: make configurable
                 auto slash = request.uri.find('/', 5);
                 if (slash == std::string::npos)
                     throw nix::Error("bad S3 URI '%s'", request.uri);
@@ -707,7 +707,7 @@ bool isUri(const string & s)
     size_t pos = s.find("://");
     if (pos == string::npos) return false;
     string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3";
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
 }
 
 
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 2afdeb021a93..839a7991c89f 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -1,5 +1,6 @@
 #include "nar-accessor.hh"
 #include "archive.hh"
+#include "json.hh"
 
 #include <map>
 #include <stack>
@@ -181,4 +182,40 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
+void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
+    const Path & path, bool recurse)
+{
+    auto st = accessor->stat(path);
+
+    auto obj = res.object();
+
+    switch (st.type) {
+    case FSAccessor::Type::tRegular:
+        obj.attr("type", "regular");
+        obj.attr("size", st.fileSize);
+        if (st.isExecutable)
+            obj.attr("executable", true);
+        break;
+    case FSAccessor::Type::tDirectory:
+        obj.attr("type", "directory");
+        {
+            auto res2 = obj.object("entries");
+            for (auto & name : accessor->readDirectory(path)) {
+                if (recurse) {
+                    auto res3 = res2.placeholder(name);
+                    listNar(res3, accessor, path + "/" + name, true);
+                } else
+                    res2.object(name);
+            }
+        }
+        break;
+    case FSAccessor::Type::tSymlink:
+        obj.attr("type", "symlink");
+        obj.attr("target", accessor->readLink(path));
+        break;
+    default:
+        throw Error("path '%s' does not exist in NAR", path);
+    }
+}
+
 }
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 83c570be4c7b..ed8fe15cad23 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -8,4 +8,11 @@ namespace nix {
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
+class JSONPlaceholder;
+
+/* Write a JSON representation of the contents of a NAR (except file
+   contents). */
+void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
+    const Path & path, bool recurse);
+
 }
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 5fc7371a5198..0079da1becfb 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -10,6 +10,8 @@
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/core/auth/AWSCredentialsProviderChain.h>
 #include <aws/core/client/ClientConfiguration.h>
 #include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/utils/logging/FormattedLogSystem.h>
@@ -77,9 +79,15 @@ static void initAWS()
     });
 }
 
-S3Helper::S3Helper(const string & region)
+S3Helper::S3Helper(const std::string & profile, const std::string & region)
     : config(makeConfig(region))
-    , client(make_ref<Aws::S3::S3Client>(*config, true, false))
+    , client(make_ref<Aws::S3::S3Client>(
+            profile == ""
+            ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
+                std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>())
+            : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>(
+                std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(profile.c_str())),
+            *config, true, false))
 {
 }
 
@@ -148,6 +156,7 @@ S3Helper::DownloadResult S3Helper::getObject(
 
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
+    const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."};
     const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}};
     const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"};
     const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"};
@@ -163,7 +172,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , s3Helper(region)
+        , s3Helper(profile, region)
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -241,8 +250,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             auto & error = res.GetError();
             if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND
                 || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY
-                || (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME
-                    && error.GetMessage().find("404") != std::string::npos))
+                // If bucket listing is disabled, 404s turn into 403s
+                || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED)
                 return false;
             throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage());
         }
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index 08a7fbf96e98..4f996400343c 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -14,7 +14,7 @@ struct S3Helper
     ref<Aws::Client::ClientConfiguration> config;
     ref<Aws::S3::S3Client> client;
 
-    S3Helper(const std::string & region);
+    S3Helper(const std::string & profile, const std::string & region);
 
     ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region);
 
diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb
index d63c8f813c9e..0299d1ee45d2 100644
--- a/src/libstore/sandbox-defaults.sb
+++ b/src/libstore/sandbox-defaults.sb
@@ -21,8 +21,15 @@
 ; Allow sending signals within the sandbox.
 (allow signal (target same-sandbox))
 
+; Allow getpwuid.
+(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo"))
+
 ; Access to /tmp.
-(allow file* process-exec (literal "/tmp") (subpath TMPDIR))
+; The network-outbound/network-inbound ones are for unix domain sockets, which
+; we allow access to in TMPDIR (but if we allow them more broadly, you could in
+; theory escape the sandbox)
+(allow file* process-exec network-outbound network-inbound
+       (literal "/tmp") (subpath TMPDIR))
 
 ; Some packages like to read the system version.
 (allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist"))
@@ -30,6 +37,29 @@
 ; Without this line clang cannot write to /dev/null, breaking some configure tests.
 (allow file-read-metadata (literal "/dev"))
 
+; Many packages like to do local networking in their test suites, but let's only
+; allow it if the package explicitly asks for it.
+(if (param "_ALLOW_LOCAL_NETWORKING")
+    (begin
+      (allow network* (local ip) (local tcp) (local udp))
+
+      ; Allow access to /etc/resolv.conf (which is a symlink to
+      ; /private/var/run/resolv.conf).
+      ; TODO: deduplicate with sandbox-network.sb
+      (allow file-read-metadata
+             (literal "/var")
+             (literal "/etc")
+             (literal "/etc/resolv.conf")
+             (literal "/private/etc/resolv.conf"))
+
+      (allow file-read*
+             (literal "/private/var/run/resolv.conf"))
+
+      ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on
+      (allow file-read-metadata (literal "/etc/hosts"))
+      (allow file-read*         (literal "/private/etc/hosts"))
+      (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))))
+
 ; Standard devices.
 (allow file*
        (literal "/dev/null")
@@ -54,5 +84,4 @@
 (allow file-read-metadata
        (literal "/etc")
        (literal "/var")
-       (literal "/private/var/tmp")
-       )
+       (literal "/private/var/tmp"))
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index c57e42fec00d..8146513a4b6e 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -389,8 +389,10 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
     Sync<State> state_(State{paths.size(), PathSet()});
 
     std::condition_variable wakeup;
+    ThreadPool pool;
 
-    for (auto & path : paths)
+    auto doQuery = [&](const Path & path ) {
+        checkInterrupt();
         queryPathInfo(path,
             [path, &state_, &wakeup](ref<ValidPathInfo> info) {
                 auto state(state_.lock());
@@ -411,6 +413,12 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
                 if (!--state->left)
                     wakeup.notify_one();
             });
+    };
+
+    for (auto & path : paths)
+        pool.enqueue(std::bind(doQuery, path));
+
+    pool.process();
 
     while (true) {
         auto state(state_.lock());
@@ -569,9 +577,9 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     auto dstUri = dstStore->getUri();
 
     Activity act(*logger, lvlInfo, actCopyPath,
-        srcUri == "local"
+        srcUri == "local" || srcUri == "daemon"
           ? fmt("copying path '%s' to '%s'", storePath, dstUri)
-          : dstUri == "local"
+          : dstUri == "local" || dstUri == "daemon"
             ? fmt("copying path '%s' from '%s'", storePath, srcUri)
             : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
         {storePath, srcUri, dstUri});
generated by cgit-pink 1.4.1 (git 2.44.2) at 2025-02-07T21ยท38+0000