importPaths(): Optionally add NARs to binary cache accessor

This enables an optimisation in hydra-queue-runner, preventing a
download of a NAR it just uploaded to the cache when reading files
like hydra-build-products.

7 files changed, 52 insertions, 38 deletions

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 6a4e3a560578..02e73d2ce1fa 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
     sink << exportMagic << storePath << res.references << res.deriver << 0;
 }
 
-Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
+Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source,
+    std::shared_ptr<FSAccessor> accessor)
 {
     assert(!requireSignature);
     Paths res;
@@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
         unsigned long long n = readLongLong(source);
         if (n == 0) break;
         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
-        res.push_back(importPath(source));
+        res.push_back(importPath(source, accessor));
     }
     return res;
 }
@@ -215,34 +216,6 @@ struct NopSink : ParseSink
 {
 };
 
-Path BinaryCacheStore::importPath(Source & source)
-{
-    /* FIXME: some cut&paste of LocalStore::importPath(). */
-
-    /* Extract the NAR from the source. */
-    TeeSource tee(source);
-    NopSink sink;
-    parseDump(sink, tee);
-
-    uint32_t magic = readInt(source);
-    if (magic != exportMagic)
-        throw Error("Nix archive cannot be imported; wrong format");
-
-    ValidPathInfo info;
-    info.path = readStorePath(source);
-
-    info.references = readStorePaths<PathSet>(source);
-
-    readString(source); // deriver, don't care
-
-    bool haveSignature = readInt(source) == 1;
-    assert(!haveSignature);
-
-    addToCache(info, tee.data);
-
-    return info.path;
-}
-
 ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
 {
     return ValidPathInfo(readNarInfo(storePath));
@@ -416,4 +389,37 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
             std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
 }
 
+Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> accessor)
+{
+    /* FIXME: some cut&paste of LocalStore::importPath(). */
+
+    /* Extract the NAR from the source. */
+    TeeSource tee(source);
+    NopSink sink;
+    parseDump(sink, tee);
+
+    uint32_t magic = readInt(source);
+    if (magic != exportMagic)
+        throw Error("Nix archive cannot be imported; wrong format");
+
+    ValidPathInfo info;
+    info.path = readStorePath(source);
+
+    info.references = readStorePaths<PathSet>(source);
+
+    readString(source); // deriver, don't care
+
+    bool haveSignature = readInt(source) == 1;
+    assert(!haveSignature);
+
+    addToCache(info, tee.data);
+
+    auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
+    if (accessor_)
+        // FIXME: more gratuitous string copying
+        accessor_->nars.emplace(info.path, makeNarAccessor(make_ref<std::string>(tee.data)));
+
+    return info.path;
+}
+
 }
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index c6f319d1bc82..6feb84cd2b10 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -127,9 +127,10 @@ public:
 
     void exportPath(const Path & path, bool sign, Sink & sink) override;
 
-    Paths importPaths(bool requireSignature, Source & source) override;
+    Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) override;
 
-    Path importPath(Source & source);
+    Path importPath(Source & source, std::shared_ptr<FSAccessor> accessor);
 
     void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 33f2569128aa..9a570668128e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 }
 
 
-Paths LocalStore::importPaths(bool requireSignature, Source & source)
+Paths LocalStore::importPaths(bool requireSignature, Source & source,
+    std::shared_ptr<FSAccessor> accessor)
 {
     Paths res;
     while (true) {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index fded63ab38e1..c7ea9e5038b4 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -147,7 +147,8 @@ public:
     void exportPath(const Path & path, bool sign,
         Sink & sink) override;
 
-    Paths importPaths(bool requireSignature, Source & source) override;
+    Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     void buildPaths(const PathSet & paths, BuildMode buildMode) override;
 
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index a1ac169c2b9e..82b7cfd7c17d 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign,
 }
 
 
-Paths RemoteStore::importPaths(bool requireSignature, Source & source)
+Paths RemoteStore::importPaths(bool requireSignature, Source & source,
+    std::shared_ptr<FSAccessor> accessor)
 {
     auto conn(connections->get());
     conn->to << wopImportPaths;
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 0019cd8f9f94..85c8292c7698 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -65,7 +65,8 @@ public:
     void exportPath(const Path & path, bool sign,
         Sink & sink) override;
 
-    Paths importPaths(bool requireSignature, Source & source) override;
+    Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) override;
 
     void buildPaths(const PathSet & paths, BuildMode buildMode) override;
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 347387f03403..488f32e16456 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -227,8 +227,11 @@ public:
     void exportPaths(const Paths & paths, bool sign, Sink & sink);
 
     /* Import a sequence of NAR dumps created by exportPaths() into
-       the Nix store. */
-    virtual Paths importPaths(bool requireSignature, Source & source) = 0;
+       the Nix store. Optionally, the contents of the NARs are
+       preloaded into the specified FS accessor to speed up subsequent
+       access. */
+    virtual Paths importPaths(bool requireSignature, Source & source,
+        std::shared_ptr<FSAccessor> accessor) = 0;
 
     /* For each path, if it's a derivation, build it.  Building a
        derivation means ensuring that the output paths are valid.  If