about summary refs log tree commit diff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc7
-rw-r--r--src/libstore/local-store.cc3
-rw-r--r--src/libstore/remote-fs-accessor.cc57
-rw-r--r--src/libstore/remote-fs-accessor.hh29
-rw-r--r--src/libstore/remote-store.cc115
-rw-r--r--src/libstore/remote-store.hh35
-rw-r--r--src/libstore/ssh-store.cc130
-rw-r--r--src/libstore/store-api.cc47
-rw-r--r--src/libstore/store-api.hh14
-rw-r--r--src/libstore/worker-protocol.hh4
10 files changed, 355 insertions, 86 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 4125af118cf7..3e07a2aa2b60 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -6,9 +6,9 @@
 #include "globals.hh"
 #include "nar-info.hh"
 #include "sync.hh"
-#include "worker-protocol.hh"
-#include "nar-accessor.hh"
+#include "remote-fs-accessor.hh"
 #include "nar-info-disk-cache.hh"
+#include "nar-accessor.hh"
 #include "json.hh"
 
 #include <chrono>
@@ -379,8 +379,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
 {
-    return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>(
-            std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
 }
 
 }
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index d3a641fd977b..612efde7bb8f 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -37,7 +37,8 @@ namespace nix {
 
 
 LocalStore::LocalStore(const Params & params)
-    : LocalFSStore(params)
+    : Store(params)
+    , LocalFSStore(params)
     , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir))
     , dbDir(stateDir + "/db")
     , linksDir(realStoreDir + "/.links")
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
new file mode 100644
index 000000000000..ca14057c2e28
--- /dev/null
+++ b/src/libstore/remote-fs-accessor.cc
@@ -0,0 +1,57 @@
+#include "remote-fs-accessor.hh"
+#include "nar-accessor.hh"
+
+namespace nix {
+
+
+RemoteFSAccessor::RemoteFSAccessor(ref<Store> store)
+    : store(store)
+{
+}
+
+std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
+{
+    auto path = canonPath(path_);
+
+    auto storePath = store->toStorePath(path);
+    std::string restPath = std::string(path, storePath.size());
+
+    if (!store->isValidPath(storePath))
+        throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
+
+    auto i = nars.find(storePath);
+    if (i != nars.end()) return {i->second, restPath};
+
+    StringSink sink;
+    store->narFromPath(storePath, sink);
+
+    auto accessor = makeNarAccessor(sink.s);
+    nars.emplace(storePath, accessor);
+    return {accessor, restPath};
+}
+
+FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
+{
+    auto res = fetch(path);
+    return res.first->stat(res.second);
+}
+
+StringSet RemoteFSAccessor::readDirectory(const Path & path)
+{
+    auto res = fetch(path);
+    return res.first->readDirectory(res.second);
+}
+
+std::string RemoteFSAccessor::readFile(const Path & path)
+{
+    auto res = fetch(path);
+    return res.first->readFile(res.second);
+}
+
+std::string RemoteFSAccessor::readLink(const Path & path)
+{
+    auto res = fetch(path);
+    return res.first->readLink(res.second);
+}
+
+}
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
new file mode 100644
index 000000000000..28f36c8296e1
--- /dev/null
+++ b/src/libstore/remote-fs-accessor.hh
@@ -0,0 +1,29 @@
+#pragma once
+
+#include "fs-accessor.hh"
+#include "ref.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+class RemoteFSAccessor : public FSAccessor
+{
+    ref<Store> store;
+
+    std::map<Path, ref<FSAccessor>> nars;
+
+    std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
+public:
+
+    RemoteFSAccessor(ref<Store> store);
+
+    Stat stat(const Path & path) override;
+
+    StringSet readDirectory(const Path & path) override;
+
+    std::string readFile(const Path & path) override;
+
+    std::string readLink(const Path & path) override;
+};
+
+}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a4bf7b587083..77faa2f801f1 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -38,9 +38,9 @@ template<class T> T readStorePaths(Store & store, Source & from)
 
 template PathSet readStorePaths(Store & store, Source & from);
 
-
+/* TODO: Separate these store impls into different files, give them better names */
 RemoteStore::RemoteStore(const Params & params, size_t maxConnections)
-    : LocalFSStore(params)
+    : Store(params)
     , connections(make_ref<Pool<Connection>>(
             maxConnections,
             [this]() { return openConnection(); },
@@ -50,13 +50,21 @@ RemoteStore::RemoteStore(const Params & params, size_t maxConnections)
 }
 
 
-std::string RemoteStore::getUri()
+UDSRemoteStore::UDSRemoteStore(const Params & params, size_t maxConnections)
+    : Store(params)
+    , LocalFSStore(params)
+    , RemoteStore(params, maxConnections)
+{
+}
+
+
+std::string UDSRemoteStore::getUri()
 {
     return "daemon";
 }
 
 
-ref<RemoteStore::Connection> RemoteStore::openConnection()
+ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
 {
     auto conn = make_ref<Connection>();
 
@@ -84,46 +92,52 @@ ref<RemoteStore::Connection> RemoteStore::openConnection()
     conn->from.fd = conn->fd.get();
     conn->to.fd = conn->fd.get();
 
+    initConnection(*conn);
+
+    return conn;
+}
+
+
+void RemoteStore::initConnection(Connection & conn)
+{
     /* Send the magic greeting, check for the reply. */
     try {
-        conn->to << WORKER_MAGIC_1;
-        conn->to.flush();
-        unsigned int magic = readInt(conn->from);
+        conn.to << WORKER_MAGIC_1;
+        conn.to.flush();
+        unsigned int magic = readInt(conn.from);
         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
 
-        conn->daemonVersion = readInt(conn->from);
-        if (GET_PROTOCOL_MAJOR(conn->daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
+        conn.daemonVersion = readInt(conn.from);
+        if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
             throw Error("Nix daemon protocol version not supported");
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 10)
+        if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
             throw Error("the Nix daemon version is too old");
-        conn->to << PROTOCOL_VERSION;
+        conn.to << PROTOCOL_VERSION;
 
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 14) {
+        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
             int cpu = settings.lockCPU ? lockToCurrentCPU() : -1;
             if (cpu != -1)
-                conn->to << 1 << cpu;
+                conn.to << 1 << cpu;
             else
-                conn->to << 0;
+                conn.to << 0;
         }
 
-        if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 11)
-            conn->to << false;
+        if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
+            conn.to << false;
 
-        conn->processStderr();
+        conn.processStderr();
     }
     catch (Error & e) {
         throw Error(format("cannot start daemon worker: %1%") % e.msg());
     }
 
     setOptions(conn);
-
-    return conn;
 }
 
 
-void RemoteStore::setOptions(ref<Connection> conn)
+void RemoteStore::setOptions(Connection & conn)
 {
-    conn->to << wopSetOptions
+    conn.to << wopSetOptions
        << settings.keepFailed
        << settings.keepGoing
        << settings.tryFallback
@@ -137,16 +151,16 @@ void RemoteStore::setOptions(ref<Connection> conn)
        << settings.buildCores
        << settings.useSubstitutes;
 
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 12) {
+    if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
         Settings::SettingsMap overrides = settings.getOverrides();
         if (overrides["ssh-auth-sock"] == "")
             overrides["ssh-auth-sock"] = getEnv("SSH_AUTH_SOCK");
-        conn->to << overrides.size();
+        conn.to << overrides.size();
         for (auto & i : overrides)
-            conn->to << i.first << i.second;
+            conn.to << i.first << i.second;
     }
 
-    conn->processStderr();
+    conn.processStderr();
 }
 
 
@@ -336,27 +350,39 @@ void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string>
     bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor)
 {
     auto conn(connections->get());
-    conn->to << wopImportPaths;
 
-    StringSink sink;
-    sink << 1 // == path follows
-        ;
-    assert(nar->size() % 8 == 0);
-    sink((unsigned char *) nar->data(), nar->size());
-    sink
-        << exportMagic
-        << info.path
-        << info.references
-        << info.deriver
-        << 0 // == no legacy signature
-        << 0 // == no path follows
-        ;
-
-    StringSource source(*sink.s);
-    conn->processStderr(0, &source);
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
+        conn->to << wopImportPaths;
+
+        StringSink sink;
+        sink << 1 // == path follows
+            ;
+        assert(nar->size() % 8 == 0);
+        sink((unsigned char *) nar->data(), nar->size());
+        sink
+            << exportMagic
+            << info.path
+            << info.references
+            << info.deriver
+            << 0 // == no legacy signature
+            << 0 // == no path follows
+            ;
+
+        StringSource source(*sink.s);
+        conn->processStderr(0, &source);
+
+        auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
+        assert(importedPaths.size() <= 1);
+    }
 
-    auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
-    assert(importedPaths.size() <= 1);
+    else {
+        conn->to << wopAddToStoreNar
+                 << info.path << info.deriver << printHash(info.narHash)
+                 << info.references << info.registrationTime << info.narSize
+                 << info.ultimate << info.sigs << *nar << repair << dontCheckSigs;
+        // FIXME: don't send nar as a string
+        conn->processStderr();
+    }
 }
 
 
@@ -553,7 +579,6 @@ RemoteStore::Connection::~Connection()
 {
     try {
         to.flush();
-        fd = -1;
     } catch (...) {
         ignoreException();
     }
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index b14ce4a97ff9..40f17da300d0 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -18,7 +18,7 @@ template<typename T> class Pool;
 
 /* FIXME: RemoteStore is a misnomer - should be something like
    DaemonStore. */
-class RemoteStore : public LocalFSStore
+class RemoteStore : public virtual Store
 {
 public:
 
@@ -26,8 +26,6 @@ public:
 
     /* Implementations of abstract store API methods. */
 
-    std::string getUri() override;
-
     bool isValidPathUncached(const Path & path) override;
 
     PathSet queryValidPaths(const PathSet & paths) override;
@@ -87,25 +85,46 @@ public:
 
     void addSignatures(const Path & storePath, const StringSet & sigs) override;
 
-private:
+protected:
 
     struct Connection
     {
-        AutoCloseFD fd;
         FdSink to;
         FdSource from;
         unsigned int daemonVersion;
 
-        ~Connection();
+        virtual ~Connection();
 
         void processStderr(Sink * sink = 0, Source * source = 0);
     };
 
+    virtual ref<Connection> openConnection() = 0;
+
+    void initConnection(Connection & conn);
+
     ref<Pool<Connection>> connections;
 
-    ref<Connection> openConnection();
+private:
+
+    void setOptions(Connection & conn);
+};
+
+class UDSRemoteStore : public LocalFSStore, public RemoteStore
+{
+public:
+
+    UDSRemoteStore(const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+
+    std::string getUri() override;
+
+private:
+
+    struct Connection : RemoteStore::Connection
+    {
+        AutoCloseFD fd;
+    };
 
-    void setOptions(ref<Connection> conn);
+    ref<RemoteStore::Connection> openConnection() override;
 };
 
 
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
new file mode 100644
index 000000000000..5166485226d9
--- /dev/null
+++ b/src/libstore/ssh-store.cc
@@ -0,0 +1,130 @@
+#include "store-api.hh"
+#include "remote-store.hh"
+#include "remote-fs-accessor.hh"
+#include "archive.hh"
+#include "worker-protocol.hh"
+#include "pool.hh"
+
+namespace nix {
+
+class SSHStore : public RemoteStore
+{
+public:
+
+    SSHStore(string uri, const Params & params, size_t maxConnections = std::numeric_limits<size_t>::max());
+
+    std::string getUri() override;
+
+    void narFromPath(const Path & path, Sink & sink) override;
+
+    ref<FSAccessor> getFSAccessor() override;
+
+private:
+
+    struct Connection : RemoteStore::Connection
+    {
+        Pid sshPid;
+        AutoCloseFD out;
+        AutoCloseFD in;
+    };
+
+    ref<RemoteStore::Connection> openConnection() override;
+
+    AutoDelete tmpDir;
+
+    Path socketPath;
+
+    Pid sshMaster;
+
+    string uri;
+
+    Path key;
+};
+
+SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
+    : Store(params)
+    , RemoteStore(params, maxConnections)
+    , tmpDir(createTempDir("", "nix", true, true, 0700))
+    , socketPath((Path) tmpDir + "/ssh.sock")
+    , uri(std::move(uri))
+    , key(get(params, "ssh-key", ""))
+{
+}
+
+string SSHStore::getUri()
+{
+    return "ssh://" + uri;
+}
+
+class ForwardSource : public Source
+{
+    Source & readSource;
+    Sink & writeSink;
+public:
+    ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {}
+    size_t read(unsigned char * data, size_t len) override
+    {
+        auto res = readSource.read(data, len);
+        writeSink(data, len);
+        return res;
+    }
+};
+
+void SSHStore::narFromPath(const Path & path, Sink & sink)
+{
+    auto conn(connections->get());
+    conn->to << wopNarFromPath << path;
+    conn->processStderr();
+    ParseSink ps;
+    auto fwd = ForwardSource(conn->from, sink);
+    parseDump(ps, fwd);
+}
+
+ref<FSAccessor> SSHStore::getFSAccessor()
+{
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
+}
+
+ref<RemoteStore::Connection> SSHStore::openConnection()
+{
+    if ((pid_t) sshMaster == -1) {
+        sshMaster = startProcess([&]() {
+            if (key.empty())
+                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
+            else
+                execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
+            throw SysError("starting ssh master");
+        });
+    }
+
+    auto conn = make_ref<Connection>();
+    Pipe in, out;
+    in.create();
+    out.create();
+    conn->sshPid = startProcess([&]() {
+        if (dup2(in.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("duping over STDIN");
+        if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("duping over STDOUT");
+        execlp("ssh", "ssh", "-S", socketPath.c_str(), uri.c_str(), "nix-daemon", "--stdio", NULL);
+        throw SysError("executing nix-daemon --stdio over ssh");
+    });
+    in.readSide = -1;
+    out.writeSide = -1;
+    conn->out = std::move(out.readSide);
+    conn->in = std::move(in.writeSide);
+    conn->to = FdSink(conn->in.get());
+    conn->from = FdSource(conn->out.get());
+    initConnection(*conn);
+    return conn;
+}
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, 6) != "ssh://") return 0;
+    return std::make_shared<SSHStore>(uri.substr(6), params);
+});
+
+}
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index f365406cb961..f7f6c9696688 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -606,7 +606,7 @@ namespace nix {
 RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0;
 
 
-ref<Store> openStoreAt(const std::string & uri_)
+ref<Store> openStore(const std::string & uri_)
 {
     auto uri(uri_);
     Store::Params params;
@@ -629,9 +629,22 @@ ref<Store> openStoreAt(const std::string & uri_)
 }
 
 
-ref<Store> openStore()
+StoreType getStoreType(const std::string & uri, const std::string & stateDir)
 {
-    return openStoreAt(getEnv("NIX_REMOTE"));
+    if (uri == "daemon") {
+        return tDaemon;
+    } else if (uri == "local") {
+        return tLocal;
+    } else if (uri == "") {
+        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
+            return tLocal;
+        else if (pathExists(settings.nixDaemonSocketFile))
+            return tDaemon;
+        else
+            return tLocal;
+    } else {
+        return tOther;
+    }
 }
 
 
@@ -639,26 +652,14 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
-    enum { mDaemon, mLocal, mAuto } mode;
-
-    if (uri == "daemon") mode = mDaemon;
-    else if (uri == "local") mode = mLocal;
-    else if (uri == "") mode = mAuto;
-    else return 0;
-
-    if (mode == mAuto) {
-        auto stateDir = get(params, "state", settings.nixStateDir);
-        if (access(stateDir.c_str(), R_OK | W_OK) == 0)
-            mode = mLocal;
-        else if (pathExists(settings.nixDaemonSocketFile))
-            mode = mDaemon;
-        else
-            mode = mLocal;
+    switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
+        case tDaemon:
+            return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
+        case tLocal:
+            return std::shared_ptr<Store>(std::make_shared<LocalStore>(params));
+        default:
+            return nullptr;
     }
-
-    return mode == mDaemon
-        ? std::shared_ptr<Store>(std::make_shared<RemoteStore>(params))
-        : std::shared_ptr<Store>(std::make_shared<LocalStore>(params));
 });
 
 
@@ -679,7 +680,7 @@ std::list<ref<Store>> getDefaultSubstituters()
     auto addStore = [&](const std::string & uri) {
         if (done.count(uri)) return;
         done.insert(uri);
-        state->stores.push_back(openStoreAt(uri));
+        state->stores.push_back(openStore(uri));
     };
 
     for (auto uri : settings.get("substituters", Strings()))
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 3057106ec142..6762852cf30e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -5,6 +5,7 @@
 #include "crypto.hh"
 #include "lru-cache.hh"
 #include "sync.hh"
+#include "globals.hh"
 
 #include <atomic>
 #include <limits>
@@ -537,7 +538,7 @@ protected:
 };
 
 
-class LocalFSStore : public Store
+class LocalFSStore : public virtual Store
 {
 public:
     const Path rootDir;
@@ -604,12 +605,17 @@ void removeTempRoots();
    If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on
    whether the user has write access to the local Nix store/database.
    set to true *unless* you're going to collect garbage. */
-ref<Store> openStoreAt(const std::string & uri);
+ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
 
 
-/* Open the store indicated by the ‘NIX_REMOTE’ environment variable. */
-ref<Store> openStore();
+enum StoreType {
+    tDaemon,
+    tLocal,
+    tOther
+};
+
 
+StoreType getStoreType(const std::string & uri = getEnv("NIX_REMOTE"), const std::string & stateDir = settings.nixStateDir);
 
 /* Return the default substituter stores, defined by the
    ‘substituters’ option and various legacy options like
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index f8cd7cc4be29..6a4ed47cc9fa 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x111
+#define PROTOCOL_VERSION 0x112
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -46,6 +46,8 @@ typedef enum {
     wopVerifyStore = 35,
     wopBuildDerivation = 36,
     wopAddSignatures = 37,
+    wopNarFromPath = 38,
+    wopAddToStoreNar = 39
 } WorkerOp;
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-11-02T20·29+0000