BinaryCacheStore: Optionally write a NAR listing

The store parameter "write-nar-listing=1" will cause BinaryCacheStore
to write a file ‘<store-hash>.ls.xz’ for each ‘<store-hash>.narinfo’
added to the binary cache. This file contains an XZ-compressed JSON
file describing the contents of the NAR, excluding the contents of
regular files.

E.g.

  {
    "version": 1,
    "root": {
      "type": "directory",
      "entries": {
        "lib": {
          "type": "directory",
          "entries": {
            "Mcrt1.o": {
              "type": "regular",
              "size": 1288
            },
            "Scrt1.o": {
              "type": "regular",
              "size": 3920
            },
          }
        }
      }
      ...
    }
  }

(The actual file has no indentation.)

This is intended to speed up the NixOS channels programs index
generator [1], since fetching gazillions of large NARs from
cache.nixos.org is currently a bottleneck for updating the regular
(non-small) channel.

[1] https://github.com/NixOS/nixos-channel-scripts/blob/master/generate-programs-index.cc

1 files changed, 64 insertions, 9 deletions

diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 01d1a7e246..7692da742f 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -9,6 +9,7 @@
 #include "worker-protocol.hh"
 #include "nar-accessor.hh"
 #include "nar-info-disk-cache.hh"
+#include "json.hh"
 
 #include <chrono>
 
@@ -19,6 +20,7 @@ namespace nix {
 BinaryCacheStore::BinaryCacheStore(const Params & params)
     : Store(params)
     , compression(get(params, "compression", "xz"))
+    , writeNARListing(get(params, "write-nar-listing", "0") == "1")
 {
     auto secretKeyFile = get(params, "secret-key", "");
     if (secretKeyFile != "")
@@ -79,7 +81,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
     return storePathToHash(storePath) + ".narinfo";
 }
 
-void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar,
+void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
     bool repair, bool dontCheckSigs)
 {
     if (!repair && isValidPath(info.path)) return;
@@ -97,20 +99,73 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
 
     auto narInfoFile = narInfoFileFor(info.path);
 
-    assert(nar.compare(0, narMagic.size(), narMagic) == 0);
+    assert(nar->compare(0, narMagic.size(), narMagic) == 0);
 
     auto narInfo = make_ref<NarInfo>(info);
 
-    narInfo->narSize = nar.size();
-    narInfo->narHash = hashString(htSHA256, nar);
+    narInfo->narSize = nar->size();
+    narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
         throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
 
+    /* Optionally write a JSON file containing a listing of the
+       contents of the NAR. */
+    if (writeNARListing) {
+        std::ostringstream jsonOut;
+
+        {
+            JSONObject jsonRoot(jsonOut);
+            jsonRoot.attr("version", 1);
+
+            auto accessor = makeNarAccessor(nar);
+
+            std::function<void(const Path &, JSONPlaceholder &)> recurse;
+
+            recurse = [&](const Path & path, JSONPlaceholder & res) {
+                auto st = accessor->stat(path);
+
+                auto obj = res.object();
+
+                switch (st.type) {
+                case FSAccessor::Type::tRegular:
+                    obj.attr("type", "regular");
+                    obj.attr("size", st.fileSize);
+                    if (st.isExecutable)
+                        obj.attr("executable", true);
+                    break;
+                case FSAccessor::Type::tDirectory:
+                    obj.attr("type", "directory");
+                    {
+                        auto res2 = obj.object("entries");
+                        for (auto & name : accessor->readDirectory(path)) {
+                            auto res3 = res2.placeholder(name);
+                            recurse(path + "/" + name, res3);
+                        }
+                    }
+                    break;
+                case FSAccessor::Type::tSymlink:
+                    obj.attr("type", "symlink");
+                    obj.attr("target", accessor->readLink(path));
+                    break;
+                default:
+                    abort();
+                }
+            };
+
+            {
+                auto res = jsonRoot.placeholder("root");
+                recurse("", res);
+            }
+        }
+
+        upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()));
+    }
+
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
-    auto narCompressed = compress(compression, nar);
+    auto narCompressed = compress(compression, *nar);
     auto now2 = std::chrono::steady_clock::now();
     narInfo->fileHash = hashString(htSHA256, *narCompressed);
     narInfo->fileSize = narCompressed->size();
@@ -118,7 +173,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
     printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
         % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0)
+        % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
 
     /* Atomically write the NAR file. */
@@ -132,7 +187,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
     } else
         stats.narWriteAverted++;
 
-    stats.narWriteBytes += nar.size();
+    stats.narWriteBytes += nar->size();
     stats.narWriteCompressedBytes += narCompressed->size();
     stats.narWriteCompressionTimeMs += duration;
 
@@ -231,7 +286,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     ValidPathInfo info;
     info.path = makeFixedOutputPath(recursive, h, name);
 
-    addToStore(info, *sink.s, repair);
+    addToStore(info, sink.s, repair);
 
     return info.path;
 }
@@ -246,7 +301,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
         dumpString(s, sink);
-        addToStore(info, *sink.s, repair);
+        addToStore(info, sink.s, repair);
     }
 
     return info.path;