about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.hh21
-rw-r--r--src/libstore/build.cc2
-rw-r--r--src/libstore/download.cc28
-rw-r--r--src/libstore/download.hh2
-rw-r--r--src/libstore/globals.cc2
-rw-r--r--src/libstore/legacy-ssh-store.cc247
-rw-r--r--src/libstore/local-store.cc1
-rw-r--r--src/libstore/optimise-store.cc2
-rw-r--r--src/libstore/remote-store.cc1
-rw-r--r--src/libstore/s3-binary-cache-store.cc143
-rw-r--r--src/libstore/s3.hh33
-rw-r--r--src/libstore/serve-protocol.hh23
-rw-r--r--src/libstore/ssh-store.cc25
-rw-r--r--src/libstore/store-api.cc46
-rw-r--r--src/libstore/store-api.hh20
15 files changed, 483 insertions, 113 deletions
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 31878bbb2476..a70d50d4949c 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -71,9 +71,6 @@ public:
         PathSet & referrers) override
     { notImpl(); }
 
-    PathSet queryValidDerivers(const Path & path) override
-    { return {}; }
-
     PathSet queryDerivationOutputs(const Path & path) override
     { notImpl(); }
 
@@ -83,13 +80,6 @@ public:
     Path queryPathFromHashPart(const string & hashPart) override
     { notImpl(); }
 
-    PathSet querySubstitutablePaths(const PathSet & paths) override
-    { return {}; }
-
-    void querySubstitutablePathInfos(const PathSet & paths,
-        SubstitutablePathInfos & infos) override
-    { }
-
     bool wantMassQuery() override { return wantMassQuery_; }
 
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
@@ -121,25 +111,14 @@ public:
     void addIndirectRoot(const Path & path) override
     { notImpl(); }
 
-    void syncWithGC() override
-    { }
-
     Roots findRoots() override
     { notImpl(); }
 
     void collectGarbage(const GCOptions & options, GCResults & results) override
     { notImpl(); }
 
-    void optimiseStore() override
-    { }
-
-    bool verifyStore(bool checkContents, bool repair) override
-    { return true; }
-
     ref<FSAccessor> getFSAccessor() override;
 
-public:
-
     void addSignatures(const Path & storePath, const StringSet & sigs) override
     { notImpl(); }
 
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 5d6fff4e349f..1aee150fda37 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "references.hh"
 #include "pathlocks.hh"
 #include "globals.hh"
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index b2adc154818e..f93fb1e968a9 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -4,6 +4,7 @@
 #include "hash.hh"
 #include "store-api.hh"
 #include "archive.hh"
+#include "s3.hh"
 
 #include <unistd.h>
 #include <fcntl.h>
@@ -488,6 +489,31 @@ struct CurlDownloader : public Downloader
         std::function<void(const DownloadResult &)> success,
         std::function<void(std::exception_ptr exc)> failure) override
     {
+        /* Ugly hack to support s3:// URIs. */
+        if (hasPrefix(request.uri, "s3://")) {
+            // FIXME: do this on a worker thread
+            sync2async<DownloadResult>(success, failure, [&]() {
+#ifdef ENABLE_S3
+                S3Helper s3Helper;
+                auto slash = request.uri.find('/', 5);
+                if (slash == std::string::npos)
+                    throw nix::Error("bad S3 URI ‘%s’", request.uri);
+                std::string bucketName(request.uri, 5, slash - 5);
+                std::string key(request.uri, slash + 1);
+                // FIXME: implement ETag
+                auto s3Res = s3Helper.getObject(bucketName, key);
+                DownloadResult res;
+                if (!s3Res.data)
+                    throw DownloadError(NotFound, fmt("S3 object ‘%s’ does not exist", request.uri));
+                res.data = s3Res.data;
+                return res;
+#else
+                throw nix::Error("cannot download ‘%s’ because Nix is not built with S3 support", request.uri);
+#endif
+            });
+            return;
+        }
+
         auto item = std::make_shared<DownloadItem>(*this, request);
         item->success = success;
         item->failure = failure;
@@ -637,7 +663,7 @@ bool isUri(const string & s)
     size_t pos = s.find("://");
     if (pos == string::npos) return false;
     string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git";
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3";
 }
 
 
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index 82b5d641fde9..bdb5011e7830 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -23,7 +23,7 @@ struct DownloadRequest
 
 struct DownloadResult
 {
-    bool cached;
+    bool cached = false;
     std::string etag;
     std::string effectiveUrl;
     std::shared_ptr<std::string> data;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 00b468892529..90f83a5bbd95 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "globals.hh"
 #include "util.hh"
 #include "archive.hh"
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
new file mode 100644
index 000000000000..5d9e5aad6e0a
--- /dev/null
+++ b/src/libstore/legacy-ssh-store.cc
@@ -0,0 +1,247 @@
+#include "archive.hh"
+#include "pool.hh"
+#include "remote-store.hh"
+#include "serve-protocol.hh"
+#include "store-api.hh"
+#include "worker-protocol.hh"
+
+namespace nix {
+
+static std::string uriScheme = "legacy-ssh://";
+
+struct LegacySSHStore : public Store
+{
+    string host;
+
+    struct Connection
+    {
+        Pid sshPid;
+        AutoCloseFD out;
+        AutoCloseFD in;
+        FdSink to;
+        FdSource from;
+    };
+
+    AutoDelete tmpDir;
+
+    Path socketPath;
+
+    Pid sshMaster;
+
+    ref<Pool<Connection>> connections;
+
+    Path key;
+
+    LegacySSHStore(const string & host, const Params & params,
+        size_t maxConnections = std::numeric_limits<size_t>::max())
+        : Store(params)
+        , host(host)
+        , tmpDir(createTempDir("", "nix", true, true, 0700))
+        , socketPath((Path) tmpDir + "/ssh.sock")
+        , connections(make_ref<Pool<Connection>>(
+            maxConnections,
+            [this]() { return openConnection(); },
+            [](const ref<Connection> & r) { return true; }
+            ))
+        , key(get(params, "ssh-key", ""))
+    {
+    }
+
+    ref<Connection> openConnection()
+    {
+        if ((pid_t) sshMaster == -1) {
+            sshMaster = startProcess([&]() {
+                restoreSignals();
+                Strings args{ "ssh", "-M", "-S", socketPath, "-N", "-x", "-a", host };
+                if (!key.empty())
+                    args.insert(args.end(), {"-i", key});
+                execvp("ssh", stringsToCharPtrs(args).data());
+                throw SysError("starting SSH master connection to host ‘%s’", host);
+            });
+        }
+
+        auto conn = make_ref<Connection>();
+        Pipe in, out;
+        in.create();
+        out.create();
+        conn->sshPid = startProcess([&]() {
+            if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
+                throw SysError("duping over STDIN");
+            if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+                throw SysError("duping over STDOUT");
+            execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-store", "--serve", "--write", nullptr);
+            throw SysError("executing ‘nix-store --serve’ on remote host ‘%s’", host);
+        });
+        in.readSide = -1;
+        out.writeSide = -1;
+        conn->out = std::move(out.readSide);
+        conn->in = std::move(in.writeSide);
+        conn->to = FdSink(conn->in.get());
+        conn->from = FdSource(conn->out.get());
+
+        int remoteVersion;
+
+        try {
+            conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
+            conn->to.flush();
+
+            unsigned int magic = readInt(conn->from);
+            if (magic != SERVE_MAGIC_2)
+                throw Error("protocol mismatch with ‘nix-store --serve’ on ‘%s’", host);
+            remoteVersion = readInt(conn->from);
+            if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
+                throw Error("unsupported ‘nix-store --serve’ protocol version on ‘%s’", host);
+
+        } catch (EndOfFile & e) {
+            throw Error("cannot connect to ‘%1%’", host);
+        }
+
+        return conn;
+    };
+
+    string getUri() override
+    {
+        return uriScheme + host;
+    }
+
+    void queryPathInfoUncached(const Path & path,
+        std::function<void(std::shared_ptr<ValidPathInfo>)> success,
+        std::function<void(std::exception_ptr exc)> failure) override
+    {
+        sync2async<std::shared_ptr<ValidPathInfo>>(success, failure, [&]() -> std::shared_ptr<ValidPathInfo> {
+            auto conn(connections->get());
+
+            debug("querying remote host ‘%s’ for info on ‘%s’", host, path);
+
+            conn->to << cmdQueryPathInfos << PathSet{path};
+            conn->to.flush();
+
+            auto info = std::make_shared<ValidPathInfo>();
+            conn->from >> info->path;
+            if (info->path.empty()) return nullptr;
+            assert(path == info->path);
+
+            PathSet references;
+            conn->from >> info->deriver;
+            info->references = readStorePaths<PathSet>(*this, conn->from);
+            readLongLong(conn->from); // download size
+            info->narSize = readLongLong(conn->from);
+
+            auto s = readString(conn->from);
+            assert(s == "");
+
+            return info;
+        });
+    }
+
+    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
+        bool repair, bool dontCheckSigs,
+        std::shared_ptr<FSAccessor> accessor) override
+    {
+        debug("adding path ‘%s’ to remote host ‘%s’", info.path, host);
+
+        auto conn(connections->get());
+
+        conn->to
+            << cmdImportPaths
+            << 1;
+        conn->to(*nar);
+        conn->to
+            << exportMagic
+            << info.path
+            << info.references
+            << info.deriver
+            << 0
+            << 0;
+        conn->to.flush();
+
+        if (readInt(conn->from) != 1)
+            throw Error("failed to add path ‘%s’ to remote host ‘%s’, info.path, host");
+
+    }
+
+    void narFromPath(const Path & path, Sink & sink) override
+    {
+        auto conn(connections->get());
+
+        conn->to << cmdDumpStorePath << path;
+        conn->to.flush();
+
+        /* FIXME: inefficient. */
+        ParseSink parseSink; /* null sink; just parse the NAR */
+        SavingSourceAdapter savedNAR(conn->from);
+        parseDump(parseSink, savedNAR);
+        sink(savedNAR.s);
+    }
+
+    /* Unsupported methods. */
+    [[noreturn]] void unsupported()
+    {
+        throw Error("operation not supported on SSH stores");
+    }
+
+    PathSet queryAllValidPaths() override { unsupported(); }
+
+    void queryReferrers(const Path & path, PathSet & referrers) override
+    { unsupported(); }
+
+    PathSet queryDerivationOutputs(const Path & path) override
+    { unsupported(); }
+
+    StringSet queryDerivationOutputNames(const Path & path) override
+    { unsupported(); }
+
+    Path queryPathFromHashPart(const string & hashPart) override
+    { unsupported(); }
+
+    Path addToStore(const string & name, const Path & srcPath,
+        bool recursive, HashType hashAlgo,
+        PathFilter & filter, bool repair) override
+    { unsupported(); }
+
+    Path addTextToStore(const string & name, const string & s,
+        const PathSet & references, bool repair) override
+    { unsupported(); }
+
+    void buildPaths(const PathSet & paths, BuildMode buildMode) override
+    { unsupported(); }
+
+    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+        BuildMode buildMode) override
+    { unsupported(); }
+
+    void ensurePath(const Path & path) override
+    { unsupported(); }
+
+    void addTempRoot(const Path & path) override
+    { unsupported(); }
+
+    void addIndirectRoot(const Path & path) override
+    { unsupported(); }
+
+    Roots findRoots() override
+    { unsupported(); }
+
+    void collectGarbage(const GCOptions & options, GCResults & results) override
+    { unsupported(); }
+
+    ref<FSAccessor> getFSAccessor()
+    { unsupported(); }
+
+    void addSignatures(const Path & storePath, const StringSet & sigs) override
+    { unsupported(); }
+
+    bool isTrusted() override
+    { return true; }
+
+};
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params);
+});
+
+}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 612efde7bb8f..4c161cfb341f 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1,4 +1,3 @@
-#include "config.h"
 #include "local-store.hh"
 #include "globals.hh"
 #include "archive.hh"
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b71c7e905ff1..cf234e35d373 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -1,5 +1,3 @@
-#include "config.h"
-
 #include "util.hh"
 #include "local-store.hh"
 #include "globals.hh"
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 816d95ba6075..42c09ec7e0b6 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -37,6 +37,7 @@ template<class T> T readStorePaths(Store & store, Source & from)
 }
 
 template PathSet readStorePaths(Store & store, Source & from);
+template Paths readStorePaths(Store & store, Source & from);
 
 /* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params, size_t maxConnections)
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index ccb71f1eefe5..ac083410b353 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -1,8 +1,6 @@
-#include "config.h"
-
 #if ENABLE_S3
-#if __linux__
 
+#include "s3.hh"
 #include "s3-binary-cache-store.hh"
 #include "nar-info.hh"
 #include "nar-info-disk-cache.hh"
@@ -20,15 +18,6 @@
 
 namespace nix {
 
-struct istringstream_nocopy : public std::stringstream
-{
-    istringstream_nocopy(const std::string & s)
-    {
-        rdbuf()->pubsetbuf(
-            (char *) s.data(), s.size());
-    }
-};
-
 struct S3Error : public Error
 {
     Aws::S3::S3Errors err;
@@ -62,21 +51,81 @@ static void initAWS()
     });
 }
 
+S3Helper::S3Helper()
+    : config(makeConfig())
+    , client(make_ref<Aws::S3::S3Client>(*config))
+{
+}
+
+ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig()
+{
+    initAWS();
+    auto res = make_ref<Aws::Client::ClientConfiguration>();
+    res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
+    res->requestTimeoutMs = 600 * 1000;
+    return res;
+}
+
+S3Helper::DownloadResult S3Helper::getObject(
+    const std::string & bucketName, const std::string & key)
+{
+    debug("fetching ‘s3://%s/%s’...", bucketName, key);
+
+    auto request =
+        Aws::S3::Model::GetObjectRequest()
+        .WithBucket(bucketName)
+        .WithKey(key);
+
+    request.SetResponseStreamFactory([&]() {
+        return Aws::New<std::stringstream>("STRINGSTREAM");
+    });
+
+    DownloadResult res;
+
+    auto now1 = std::chrono::steady_clock::now();
+
+    try {
+
+        auto result = checkAws(fmt("AWS error fetching ‘%s’", key),
+            client->GetObject(request));
+
+        res.data = std::make_shared<std::string>(
+            dynamic_cast<std::stringstream &>(result.GetBody()).str());
+
+    } catch (S3Error & e) {
+        if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
+    }
+
+    auto now2 = std::chrono::steady_clock::now();
+
+    res.durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+
+    return res;
+}
+
+#if __linux__
+
+struct istringstream_nocopy : public std::stringstream
+{
+    istringstream_nocopy(const std::string & s)
+    {
+        rdbuf()->pubsetbuf(
+            (char *) s.data(), s.size());
+    }
+};
+
 struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
 {
     std::string bucketName;
 
-    ref<Aws::Client::ClientConfiguration> config;
-    ref<Aws::S3::S3Client> client;
-
     Stats stats;
 
+    S3Helper s3Helper;
+
     S3BinaryCacheStoreImpl(
         const Params & params, const std::string & bucketName)
         : S3BinaryCacheStore(params)
         , bucketName(bucketName)
-        , config(makeConfig())
-        , client(make_ref<Aws::S3::S3Client>(*config))
     {
         diskCache = getNarInfoDiskCache();
     }
@@ -86,15 +135,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         return "s3://" + bucketName;
     }
 
-    ref<Aws::Client::ClientConfiguration> makeConfig()
-    {
-        initAWS();
-        auto res = make_ref<Aws::Client::ClientConfiguration>();
-        res->region = Aws::Region::US_EAST_1; // FIXME: make configurable
-        res->requestTimeoutMs = 600 * 1000;
-        return res;
-    }
-
     void init() override
     {
         if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) {
@@ -102,7 +142,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             /* Create the bucket if it doesn't already exists. */
             // FIXME: HeadBucket would be more appropriate, but doesn't return
             // an easily parsed 404 message.
-            auto res = client->GetBucketLocation(
+            auto res = s3Helper.client->GetBucketLocation(
                 Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName));
 
             if (!res.IsSuccess()) {
@@ -110,7 +150,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
                     throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage());
 
                 checkAws(format("AWS error creating bucket ‘%s’") % bucketName,
-                    client->CreateBucket(
+                    s3Helper.client->CreateBucket(
                         Aws::S3::Model::CreateBucketRequest()
                         .WithBucket(bucketName)
                         .WithCreateBucketConfiguration(
@@ -148,7 +188,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
     {
         stats.head++;
 
-        auto res = client->HeadObject(
+        auto res = s3Helper.client->HeadObject(
             Aws::S3::Model::HeadObjectRequest()
             .WithBucket(bucketName)
             .WithKey(path));
@@ -181,7 +221,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         auto now1 = std::chrono::steady_clock::now();
 
         auto result = checkAws(format("AWS error uploading ‘%s’") % path,
-            client->PutObject(request));
+            s3Helper.client->PutObject(request));
 
         auto now2 = std::chrono::steady_clock::now();
 
@@ -200,42 +240,18 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
         sync2async<std::shared_ptr<std::string>>(success, failure, [&]() {
             debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
 
-            auto request =
-                Aws::S3::Model::GetObjectRequest()
-                .WithBucket(bucketName)
-                .WithKey(path);
-
-            request.SetResponseStreamFactory([&]() {
-                return Aws::New<std::stringstream>("STRINGSTREAM");
-            });
-
             stats.get++;
 
-            try {
-
-                auto now1 = std::chrono::steady_clock::now();
-
-                auto result = checkAws(format("AWS error fetching ‘%s’") % path,
-                    client->GetObject(request));
-
-                auto now2 = std::chrono::steady_clock::now();
+            auto res = s3Helper.getObject(bucketName, path);
 
-                auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str();
+            stats.getBytes += res.data ? res.data->size() : 0;
+            stats.getTimeMs += res.durationMs;
 
-                auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+            if (res.data)
+                printTalkative("downloaded ‘s3://%s/%s’ (%d bytes) in %d ms",
+                    bucketName, path, res.data->size(), res.durationMs);
 
-                printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms")
-                    % bucketName % path % res.size() % duration);
-
-                stats.getBytes += res.size();
-                stats.getTimeMs += duration;
-
-                return std::make_shared<std::string>(res);
-
-            } catch (S3Error & e) {
-                if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return std::shared_ptr<std::string>();
-                throw;
-            }
+            return res.data;
         });
     }
 
@@ -248,7 +264,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
             debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker);
 
             auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName,
-                client->ListObjects(
+                s3Helper.client->ListObjects(
                     Aws::S3::Model::ListObjectsRequest()
                     .WithBucket(bucketName)
                     .WithDelimiter("/")
@@ -283,7 +299,8 @@ static RegisterStoreImplementation regStore([](
     return store;
 });
 
+#endif
+
 }
 
 #endif
-#endif
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
new file mode 100644
index 000000000000..5d5d3475c449
--- /dev/null
+++ b/src/libstore/s3.hh
@@ -0,0 +1,33 @@
+#pragma once
+
+#if ENABLE_S3
+
+#include "ref.hh"
+
+namespace Aws { namespace Client { class ClientConfiguration; } }
+namespace Aws { namespace S3 { class S3Client; } }
+
+namespace nix {
+
+struct S3Helper
+{
+    ref<Aws::Client::ClientConfiguration> config;
+    ref<Aws::S3::S3Client> client;
+
+    S3Helper();
+
+    ref<Aws::Client::ClientConfiguration> makeConfig();
+
+    struct DownloadResult
+    {
+        std::shared_ptr<std::string> data;
+        unsigned int durationMs;
+    };
+
+    DownloadResult getObject(
+        const std::string & bucketName, const std::string & key);
+};
+
+}
+
+#endif
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
new file mode 100644
index 000000000000..f8cc9a4b6ebe
--- /dev/null
+++ b/src/libstore/serve-protocol.hh
@@ -0,0 +1,23 @@
+#pragma once
+
+namespace nix {
+
+#define SERVE_MAGIC_1 0x390c9deb
+#define SERVE_MAGIC_2 0x5452eecb
+
+#define SERVE_PROTOCOL_VERSION 0x203
+#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
+#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
+
+typedef enum {
+    cmdQueryValidPaths = 1,
+    cmdQueryPathInfos = 2,
+    cmdDumpStorePath = 3,
+    cmdImportPaths = 4,
+    cmdExportPaths = 5,
+    cmdBuildPaths = 6,
+    cmdQueryClosure = 7,
+    cmdBuildDerivation = 8,
+} ServeCommand;
+
+}
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index f5d0a270438d..6f1862afa899 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -7,11 +7,13 @@
 
 namespace nix {
 
+static std::string uriScheme = "ssh://";
+
 class SSHStore : public RemoteStore
 {
 public:
 
-    SSHStore(string uri, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+    SSHStore(string host, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
 
     std::string getUri() override;
 
@@ -36,18 +38,21 @@ private:
 
     Pid sshMaster;
 
-    string uri;
+    string host;
 
     Path key;
+
+    bool compress;
 };
 
-SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
+SSHStore::SSHStore(string host, const Params & params, size_t maxConnections)
     : Store(params)
     , RemoteStore(params, maxConnections)
     , tmpDir(createTempDir("", "nix", true, true, 0700))
     , socketPath((Path) tmpDir + "/ssh.sock")
-    , uri(std::move(uri))
+    , host(std::move(host))
     , key(get(params, "ssh-key", ""))
+    , compress(get(params, "compress", "") == "true")
 {
     /* open a connection and perform the handshake to verify all is well */
     connections->get();
@@ -55,7 +60,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
 
 string SSHStore::getUri()
 {
-    return "ssh://" + uri;
+    return uriScheme + host;
 }
 
 class ForwardSource : public Source
@@ -93,9 +98,9 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
         sshMaster = startProcess([&]() {
             restoreSignals();
             if (key.empty())
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), host.c_str(), NULL);
             else
-                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
+                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), host.c_str(), NULL);
             throw SysError("starting ssh master");
         });
     }
@@ -109,7 +114,7 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
             throw SysError("duping over STDIN");
         if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
             throw SysError("duping over STDOUT");
-        execlp("ssh", "ssh", "-S", socketPath.c_str(), uri.c_str(), "nix-daemon", "--stdio", NULL);
+        execlp("ssh", "ssh", "-S", socketPath.c_str(), host.c_str(), "nix-daemon", "--stdio", NULL);
         throw SysError("executing nix-daemon --stdio over ssh");
     });
     in.readSide = -1;
@@ -126,8 +131,8 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
-    if (std::string(uri, 0, 6) != "ssh://") return 0;
-    return std::make_shared<SSHStore>(uri.substr(6), params);
+    if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0;
+    return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params);
 });
 
 }
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index a42d11834053..b5934a0d1232 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -5,6 +5,7 @@
 #include "nar-info-disk-cache.hh"
 #include "thread-pool.hh"
 #include "json.hh"
+#include "derivations.hh"
 
 #include <future>
 
@@ -285,6 +286,19 @@ bool Store::isValidPath(const Path & storePath)
 }
 
 
+/* Default implementation for stores that only implement
+   queryPathInfoUncached(). */
+bool Store::isValidPathUncached(const Path & path)
+{
+    try {
+        queryPathInfo(path);
+        return true;
+    } catch (InvalidPath &) {
+        return false;
+    }
+}
+
+
 ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
 {
     std::promise<ref<ValidPathInfo>> promise;
@@ -516,6 +530,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     StringSink sink;
     srcStore->narFromPath({storePath}, sink);
 
+    if (srcStore->isTrusted())
+        dontCheckSigs = true;
+
+    if (!info->narHash && dontCheckSigs) {
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->narHash = hashString(htSHA256, *sink.s);
+        info = info2;
+    }
+
     dstStore->addToStore(*info, sink.s, repair, dontCheckSigs);
 }
 
@@ -758,8 +781,27 @@ std::list<ref<Store>> getDefaultSubstituters()
 }
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
-{
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
+{
+    if (substitute) {
+        /* Filter out .drv files (we don't want to build anything). */
+        PathSet paths2;
+        for (auto & path : storePaths)
+            if (!isDerivation(path)) paths2.insert(path);
+        unsigned long long downloadSize, narSize;
+        PathSet willBuild, willSubstitute, unknown;
+        to->queryMissing(PathSet(paths2.begin(), paths2.end()),
+            willBuild, willSubstitute, unknown, downloadSize, narSize);
+        /* FIXME: should use ensurePath(), but it only
+           does one path at a time. */
+        if (!willSubstitute.empty())
+            try {
+                to->buildPaths(willSubstitute);
+            } catch (Error & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
+    }
+
     std::string copiedLabel = "copied";
 
     logger->setExpected(copiedLabel, storePaths.size());
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3fee999072fa..d03e70849f93 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -320,7 +320,7 @@ public:
 
 protected:
 
-    virtual bool isValidPathUncached(const Path & path) = 0;
+    virtual bool isValidPathUncached(const Path & path);
 
 public:
 
@@ -360,7 +360,7 @@ public:
        output.  (Note that the result of `queryDeriver()' is the
        derivation that was actually used to produce `path', which may
        not exist anymore.) */
-    virtual PathSet queryValidDerivers(const Path & path) = 0;
+    virtual PathSet queryValidDerivers(const Path & path) { return {}; };
 
     /* Query the outputs of the derivation denoted by `path'. */
     virtual PathSet queryDerivationOutputs(const Path & path) = 0;
@@ -373,13 +373,13 @@ public:
     virtual Path queryPathFromHashPart(const string & hashPart) = 0;
 
     /* Query which of the given paths have substitutes. */
-    virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
+    virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; };
 
     /* Query substitute info (i.e. references, derivers and download
        sizes) of a set of paths.  If a path does not have substitute
        info, it's omitted from the resulting ‘infos’ map. */
     virtual void querySubstitutablePathInfos(const PathSet & paths,
-        SubstitutablePathInfos & infos) = 0;
+        SubstitutablePathInfos & infos) { return; };
 
     virtual bool wantMassQuery() { return false; }
 
@@ -454,7 +454,7 @@ public:
          permanent root and sees our's.
 
        In either case the permanent root is seen by the collector. */
-    virtual void syncWithGC() = 0;
+    virtual void syncWithGC() { };
 
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
@@ -485,11 +485,11 @@ public:
 
     /* Optimise the disk space usage of the Nix store by hard-linking files
        with the same contents. */
-    virtual void optimiseStore() = 0;
+    virtual void optimiseStore() { };
 
     /* Check the integrity of the Nix store.  Returns true if errors
        remain. */
-    virtual bool verifyStore(bool checkContents, bool repair) = 0;
+    virtual bool verifyStore(bool checkContents, bool repair) { return false; };
 
     /* Return an object to access files in the Nix store. */
     virtual ref<FSAccessor> getFSAccessor() = 0;
@@ -562,6 +562,10 @@ public:
 
     const Stats & getStats();
 
+    /* Whether this store paths from this store can be imported even
+       if they lack a signature. */
+    virtual bool isTrusted() { return false; }
+
 protected:
 
     Stats stats;
@@ -639,7 +643,7 @@ void removeTempRoots();
 ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
 
-void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
+void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
 
 enum StoreType {
     tDaemon,
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-10-25T08·02+0000