importPaths(): Optionally add NARs to binary cache accessor

This enables an optimisation in hydra-queue-runner, preventing a
download of a NAR it just uploaded to the cache when reading files
like hydra-build-products.

10 files changed, 58 insertions, 44 deletions

diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 44c88a87bfc6..bb322875d0ca 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -186,7 +186,7 @@ void importPaths(int fd)
     PPCODE:
         try {
             FdSource source(fd);
-            store()->importPaths(false, source);
+            store()->importPaths(false, source, 0);
         } catch (Error & e) {
             croak("%s", e.what());
         }
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 6a4e3a560578..02e73d2ce1fa 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
     sink << exportMagic << storePath << res.references << res.deriver << 0;
 }
 
-Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
+Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source,
+    std::shared_ptr<FSAccessor> accessor)
 {
     assert(!requireSignature);
     Paths res;
@@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
         unsigned long long n = readLongLong(source);
         if (n == 0) break;
         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
-        res.push_back(importPath(source));
+        res.push_back(importPath(source, accessor));
     }
     return res;
 }
@@ -215,34 +216,6 @@ struct NopSink : ParseSink
 {
 };
 
-Path BinaryCacheStore::importPath(Source & source)
-{
-    /* FIXME: some cut&paste of LocalStore::importPath(). */
-
-    /* Extract the NAR from the source. */
-    TeeSource tee(source);
-    NopSink sink;
-    parseDump(sink, tee);
-
-    uint32_t magic = readInt(source);
-    if (magic != exportMagic)
-        throw Error("Nix archive cannot be imported; wrong format");
-
-    ValidPathInfo info;
-    info.path = readStorePath(source);
-
-    info.references = readStorePaths<PathSet>(source);
-
-    readString(source); // deriver, don't care
-
-    bool haveSignature = readInt(source) == 1;
-    assert(!haveSignature);
-
-    addToCache(info, tee.data);
-
-    return info.path;
-}
-
 ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
 {
     return ValidPathInfo(readNarInfo(storePath));
@@ -416,4 +389,37 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
             std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
 }
 
+Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> accessor)
+{
+    /* FIXME: some cut&paste of LocalStore::importPath(). */
+
+    /* Extract the NAR from the source. */
+    TeeSource tee(source);
+    NopSink sink;
+    parseDump(sink, tee);
+
+    uint32_t magic = readInt(source);
+    if (magic != exportMagic)
+        throw Error("Nix archive cannot be imported; wrong format");
+
+    ValidPathInfo info;
+    info.path = readStorePath(source);
+
+    info.references = readStorePaths<PathSet>(source);
+
+    readString(source); // deriver, don't care
+
+    bool haveSignature = readInt(source) == 1;
+    assert(!haveSignature);
+
+    addToCache(info, tee.data);
+
+    auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
+    if (accessor_)
+        // FIXME: more gratuitous string copying
+        accessor_->nars.emplace(info.path, makeNarAccessor(make_ref<std::string>(tee.data)));
+
+    return info.path;
+}
+
 }
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index c6f319d1bc82..6feb84cd2b10 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -127,9 +127,10 @@ public:
 
     void exportPath(const Path & path, bool sign, Sink & sink) override;
 
-    Paths importPaths(bool requireSignature, Source & source) override;
+    Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) override;
 
-    Path importPath(Source & source);
+    Path importPath(Source & source, std::shared_ptr<FSAccessor> accessor);
 
     void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 33f2569128aa..9a570668128e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 }
 
 
-Paths LocalStore::importPaths(bool requireSignature, Source & source)
+Paths LocalStore::importPaths(bool requireSignature, Source & source,
+    std::shared_ptr<FSAccessor> accessor)
 {
     Paths res;
     while (true) {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index fded63ab38e1..c7ea9e5038b4 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -147,7 +147,8 @@ public:
     void exportPath(const Path & path, bool sign,
         Sink & sink) override;
 
-    Paths importPaths(bool requireSignature, Source & source) override;
+    Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     void buildPaths(const PathSet & paths, BuildMode buildMode) override;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a1ac169c2b9e..82b7cfd7c17d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign,
 }
 
 
-Paths RemoteStore::importPaths(bool requireSignature, Source & source)
+Paths RemoteStore::importPaths(bool requireSignature, Source & source,
+    std::shared_ptr<FSAccessor> accessor)
 {
     auto conn(connections->get());
     conn->to << wopImportPaths;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 0019cd8f9f94..85c8292c7698 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -65,7 +65,8 @@ public:
     void exportPath(const Path & path, bool sign,
         Sink & sink) override;
 
-    Paths importPaths(bool requireSignature, Source & source) override;
+    Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     void buildPaths(const PathSet & paths, BuildMode buildMode) override;
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 347387f03403..488f32e16456 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -227,8 +227,11 @@ public:
     void exportPaths(const Paths & paths, bool sign, Sink & sink);
 
     /* Import a sequence of NAR dumps created by exportPaths() into
-       the Nix store. */
-    virtual Paths importPaths(bool requireSignature, Source & source) = 0;
+       the Nix store. Optionally, the contents of the NARs are
+       preloaded into the specified FS accessor to speed up subsequent
+       access. */
+    virtual Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) = 0;
 
     /* For each path, if it's a derivation, build it.  Building a
        derivation means ensuring that the output paths are valid.  If
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc
index e3c12ea0e0ea..5189c9b4c179 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix-daemon/nix-daemon.cc
@@ -310,7 +310,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopImportPaths: {
         startWork();
         TunnelSource source(from);
-        Paths paths = store->importPaths(!trusted, source);
+        Paths paths = store->importPaths(!trusted, source, 0);
         stopWork();
         to << paths;
         break;
@@ -322,8 +322,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
             mode = (BuildMode)readInt(from);
 
-	    /* Repairing is not atomic, so disallowed for "untrusted"
-	       clients.  */
+            /* Repairing is not atomic, so disallowed for "untrusted"
+               clients.  */
             if (mode == bmRepair && !trusted)
                 throw Error("repairing is not supported when building through the Nix daemon");
         }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 48c2865e5ecc..a258c77c326f 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs)
     if (!opArgs.empty()) throw UsageError("no arguments expected");
 
     FdSource source(STDIN_FILENO);
-    Paths paths = store->importPaths(requireSignature, source);
+    Paths paths = store->importPaths(requireSignature, source, 0);
 
     for (auto & i : paths)
         cout << format("%1%\n") % i << std::flush;
@@ -935,7 +935,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case cmdImportPaths: {
                 if (!writeAllowed) throw Error("importing paths is not allowed");
-                store->importPaths(false, in);
+                store->importPaths(false, in, 0);
                 out << 1; // indicate success
                 break;
             }