BinaryCacheStore: Optionally write a NAR listing

The store parameter "write-nar-listing=1" will cause BinaryCacheStore
to write a file ‘<store-hash>.ls.xz’ for each ‘<store-hash>.narinfo’
added to the binary cache. This file contains an XZ-compressed JSON
file describing the contents of the NAR, excluding the contents of
regular files.

E.g.

  {
    "version": 1,
    "root": {
      "type": "directory",
      "entries": {
        "lib": {
          "type": "directory",
          "entries": {
            "Mcrt1.o": {
              "type": "regular",
              "size": 1288
            },
            "Scrt1.o": {
              "type": "regular",
              "size": 3920
            },
          }
        }
      }
      ...
    }
  }

(The actual file has no indentation.)

This is intended to speed up the NixOS channels programs index
generator [1], since fetching gazillions of large NARs from
cache.nixos.org is currently a bottleneck for updating the regular
(non-small) channel.

[1] https://github.com/NixOS/nixos-channel-scripts/blob/master/generate-programs-index.cc

10 files changed, 81 insertions, 20 deletions

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 01d1a7e246d9..7692da742f23 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -9,6 +9,7 @@
 #include "worker-protocol.hh"
 #include "nar-accessor.hh"
 #include "nar-info-disk-cache.hh"
+#include "json.hh"
 
 #include <chrono>
 
@@ -19,6 +20,7 @@ namespace nix {
 BinaryCacheStore::BinaryCacheStore(const Params & params)
     : Store(params)
     , compression(get(params, "compression", "xz"))
+    , writeNARListing(get(params, "write-nar-listing", "0") == "1")
 {
     auto secretKeyFile = get(params, "secret-key", "");
     if (secretKeyFile != "")
@@ -79,7 +81,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
     return storePathToHash(storePath) + ".narinfo";
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar,
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     bool repair, bool dontCheckSigs)
 {
     if (!repair && isValidPath(info.path)) return;
@@ -97,20 +99,73 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
 
     auto narInfoFile = narInfoFileFor(info.path);
 
-    assert(nar.compare(0, narMagic.size(), narMagic) == 0);
+    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
 
     auto narInfo = make_ref<NarInfo>(info);
 
-    narInfo->narSize = nar.size();
-    narInfo->narHash = hashString(htSHA256, nar);
+    narInfo->narSize = nar->size();
+    narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
+    /* Optionally write a JSON file containing a listing of the
+       contents of the NAR. */
+    if (writeNARListing) {
+        std::ostringstream jsonOut;
+
+        {
+            JSONObject jsonRoot(jsonOut);
+            jsonRoot.attr("version", 1);
+
+            auto accessor = makeNarAccessor(nar);
+
+            std::function<void(const Path &, JSONPlaceholder &)> recurse;
+
+            recurse = [&](const Path & path, JSONPlaceholder & res) {
+                auto st = accessor->stat(path);
+
+                auto obj = res.object();
+
+                switch (st.type) {
+                case FSAccessor::Type::tRegular:
+                    obj.attr("type", "regular");
+                    obj.attr("size", st.fileSize);
+                    if (st.isExecutable)
+                        obj.attr("executable", true);
+                    break;
+                case FSAccessor::Type::tDirectory:
+                    obj.attr("type", "directory");
+                    {
+                        auto res2 = obj.object("entries");
+                        for (auto & name : accessor->readDirectory(path)) {
+                            auto res3 = res2.placeholder(name);
+                            recurse(path + "/" + name, res3);
+                        }
+                    }
+                    break;
+                case FSAccessor::Type::tSymlink:
+                    obj.attr("type", "symlink");
+                    obj.attr("target", accessor->readLink(path));
+                    break;
+                default:
+                    abort();
+                }
+            };
+
+            {
+                auto res = jsonRoot.placeholder("root");
+                recurse("", res);
+            }
+        }
+
+        upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()));
+    }
+
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, nar);
+    auto narCompressed = compress(compression, *nar);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();
@@ -118,7 +173,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
         % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0)
+        % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
 
     /* Atomically write the NAR file. */
@@ -132,7 +187,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar.size();
+    stats.narWriteBytes += nar->size();
     stats.narWriteCompressedBytes += narCompressed->size();
     stats.narWriteCompressionTimeMs += duration;
 
@@ -231,7 +286,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     ValidPathInfo info;
     info.path = makeFixedOutputPath(recursive, h, name);
 
-    addToStore(info, *sink.s, repair);
+    addToStore(info, sink.s, repair);
 
     return info.path;
 }
@@ -246,7 +301,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, *sink.s, repair);
+        addToStore(info, sink.s, repair);
     }
 
     return info.path;
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 574696cf3a1f..333cf08562a0 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -19,12 +19,16 @@ private:
 
     std::string compression;
 
+    bool writeNARListing;
+
 protected:
 
     BinaryCacheStore(const Params & params);
 
     [[noreturn]] void notImpl();
 
+public:
+
     virtual bool fileExists(const std::string & path) = 0;
 
     virtual void upsertFile(const std::string & path, const std::string & data) = 0;
@@ -37,6 +41,8 @@ protected:
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
+protected:
+
     bool wantMassQuery_ = false;
     int priority = 50;
 
@@ -86,7 +92,7 @@ public:
 
     bool wantMassQuery() override { return wantMassQuery_; }
 
-    void addToStore(const ValidPathInfo & info, const std::string & nar,
+    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         bool repair = false, bool dontCheckSigs = false) override;
 
     Path addToStore(const string & name, const Path & srcPath,
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index c01ba63ef3c2..954044c2344f 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -558,7 +558,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
                 Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data);
                 info.path = store->makeFixedOutputPath(false, hash, name);
                 info.narHash = hashString(htSHA256, *sink.s);
-                store->addToStore(info, *sink.s, false, true);
+                store->addToStore(info, sink.s, false, true);
                 storePath = info.path;
             }
 
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index b7f43acf1dc2..883b87217346 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -117,7 +117,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
         if (readInt(source) == 1)
             readString(source);
 
-        addToStore(info, *tee.data, false, dontCheckSigs);
+        addToStore(info, tee.data, false, dontCheckSigs);
 
         if (accessor)
             addPathToAccessor(ref<FSAccessor>(accessor), info.path, tee.data);
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index ed5747f3132a..2c2475afc5a7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -909,10 +909,10 @@ void LocalStore::invalidatePath(State & state, const Path & path)
 }
 
 
-void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
+void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     bool repair, bool dontCheckSigs)
 {
-    Hash h = hashString(htSHA256, nar);
+    Hash h = hashString(htSHA256, *nar);
     if (h != info.narHash)
         throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") %
             info.path % info.narHash.to_string() % h.to_string());
@@ -939,7 +939,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
 
             deletePath(realPath);
 
-            StringSource source(nar);
+            StringSource source(*nar);
             restorePath(realPath, source);
 
             canonicalisePathMetaData(realPath, -1);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 24188130db78..86f03526d181 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -125,7 +125,7 @@ public:
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos) override;
 
-    void addToStore(const ValidPathInfo & info, const std::string & nar,
+    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         bool repair, bool dontCheckSigs) override;
 
     Path addToStore(const string & name, const Path & srcPath,
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 38af145f9856..de51716db2ef 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -332,7 +332,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
 }
 
 
-void RemoteStore::addToStore(const ValidPathInfo & info, const std::string & nar,
+void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     bool repair, bool dontCheckSigs)
 {
     throw Error("RemoteStore::addToStore() not implemented");
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 9879337d65a6..5932fda95626 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -53,7 +53,7 @@ public:
     void querySubstitutablePathInfos(const PathSet & paths,
         SubstitutablePathInfos & infos) override;
 
-    void addToStore(const ValidPathInfo & info, const std::string & nar,
+    void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         bool repair, bool dontCheckSigs) override;
 
     Path addToStore(const string & name, const Path & srcPath,
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b6ac1c3b987e..f365406cb961 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -456,7 +456,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
     StringSink sink;
     srcStore->narFromPath({storePath}, sink);
 
-    dstStore->addToStore(*info, *sink.s, repair);
+    dstStore->addToStore(*info, sink.s, repair);
 }
 
 
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index ce1583b0c6fa..d6e181196349 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -365,7 +365,7 @@ public:
     virtual bool wantMassQuery() { return false; }
 
     /* Import a path into the store. */
-    virtual void addToStore(const ValidPathInfo & info, const std::string & nar,
+    virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         bool repair = false, bool dontCheckSigs = false) = 0;
 
     /* Copy the contents of a path to the store and register the