about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/binary-cache-store.cc12
-rw-r--r--src/libstore/nar-accessor.cc193
-rw-r--r--src/libstore/nar-accessor.hh10
-rw-r--r--src/libstore/remote-fs-accessor.cc82
-rw-r--r--src/libstore/remote-fs-accessor.hh5
5 files changed, 198 insertions, 104 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 68af85bf16d9..ab971dd8b6d9 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -129,10 +129,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
             auto narAccessor = makeNarAccessor(nar);
 
-            if (accessor_) {
-                accessor_->nars.emplace(info.path, narAccessor);
-                accessor_->addToCache(info.path, *nar);
-            }
+            if (accessor_)
+                accessor_->addToCache(info.path, *nar, narAccessor);
 
             {
                 auto res = jsonRoot.placeholder("root");
@@ -144,10 +142,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     }
 
     else {
-        if (accessor_) {
-            accessor_->nars.emplace(info.path, makeNarAccessor(nar));
-            accessor_->addToCache(info.path, *nar);
-        }
+        if (accessor_)
+            accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
     }
 
     /* Compress the NAR. */
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index d6e9757c2cbf..b74480684f2a 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -6,6 +6,8 @@
 #include <stack>
 #include <algorithm>
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 struct NarMember
@@ -24,83 +26,127 @@ struct NarMember
     std::map<std::string, NarMember> children;
 };
 
-struct NarIndexer : ParseSink, StringSource
+struct NarAccessor : public FSAccessor
 {
-    NarMember root;
-    std::stack<NarMember*> parents;
+    std::shared_ptr<const std::string> nar;
 
-    std::string currentStart;
-    bool isExec = false;
+    GetNarBytes getNarBytes;
 
-    NarIndexer(const std::string & nar) : StringSource(nar)
+    NarMember root;
+
+    struct NarIndexer : ParseSink, StringSource
     {
-    }
+        NarAccessor & acc;
 
-    void createMember(const Path & path, NarMember member) {
-        size_t level = std::count(path.begin(), path.end(), '/');
-        while(parents.size() > level) {
-            parents.pop();
-        }
+        std::stack<NarMember *> parents;
 
-        if(parents.empty()) {
-            root = std::move(member);
-            parents.push(&root);
-        } else {
-            if(parents.top()->type != FSAccessor::Type::tDirectory) {
-                throw Error(format("NAR file missing parent directory of path '%1%'") % path);
+        std::string currentStart;
+        bool isExec = false;
+
+        NarIndexer(NarAccessor & acc, const std::string & nar)
+            : StringSource(nar), acc(acc)
+        { }
+
+        void createMember(const Path & path, NarMember member) {
+            size_t level = std::count(path.begin(), path.end(), '/');
+            while (parents.size() > level) parents.pop();
+
+            if (parents.empty()) {
+                acc.root = std::move(member);
+                parents.push(&acc.root);
+            } else {
+                if (parents.top()->type != FSAccessor::Type::tDirectory)
+                    throw Error("NAR file missing parent directory of path '%s'", path);
+                auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
+                parents.push(&result.first->second);
             }
-            auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
-            parents.push(&result.first->second);
         }
-    }
 
-    void createDirectory(const Path & path) override
-    {
-        createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0 });
-    }
+        void createDirectory(const Path & path) override
+        {
+            createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0});
+        }
 
-    void createRegularFile(const Path & path) override
-    {
-        createMember(path, {FSAccessor::Type::tRegular, false, 0, 0 });
-    }
+        void createRegularFile(const Path & path) override
+        {
+            createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
+        }
 
-    void isExecutable() override
-    {
-        parents.top()->isExecutable = true;
-    }
+        void isExecutable() override
+        {
+            parents.top()->isExecutable = true;
+        }
 
-    void preallocateContents(unsigned long long size) override
-    {
-        currentStart = string(s, pos, 16);
-        assert(size <= std::numeric_limits<size_t>::max());
-        parents.top()->size = (size_t)size;
-        parents.top()->start = pos;
-    }
+        void preallocateContents(unsigned long long size) override
+        {
+            currentStart = string(s, pos, 16);
+            assert(size <= std::numeric_limits<size_t>::max());
+            parents.top()->size = (size_t)size;
+            parents.top()->start = pos;
+        }
 
-    void receiveContents(unsigned char * data, unsigned int len) override
-    {
-        // Sanity check
-        if (!currentStart.empty()) {
-            assert(len < 16 || currentStart == string((char *) data, 16));
-            currentStart.clear();
+        void receiveContents(unsigned char * data, unsigned int len) override
+        {
+            // Sanity check
+            if (!currentStart.empty()) {
+                assert(len < 16 || currentStart == string((char *) data, 16));
+                currentStart.clear();
+            }
+        }
+
+        void createSymlink(const Path & path, const string & target) override
+        {
+            createMember(path,
+                NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
         }
+    };
+
+    NarAccessor(ref<const std::string> nar) : nar(nar)
+    {
+        NarIndexer indexer(*this, *nar);
+        parseDump(indexer, indexer);
     }
 
-    void createSymlink(const Path & path, const string & target) override
+    NarAccessor(const std::string & listing, GetNarBytes getNarBytes)
+        : getNarBytes(getNarBytes)
     {
-        createMember(path,
-            NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
+        using json = nlohmann::json;
+
+        std::function<void(NarMember &, json &)> recurse;
+
+        recurse = [&](NarMember & member, json & v) {
+            std::string type = v["type"];
+
+            if (type == "directory") {
+                member.type = FSAccessor::Type::tDirectory;
+                for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) {
+                    std::string name = i.key();
+                    recurse(member.children[name], i.value());
+                }
+            } else if (type == "regular") {
+                member.type = FSAccessor::Type::tRegular;
+                member.size = v["size"];
+                member.isExecutable = v.value("executable", false);
+                member.start = v["narOffset"];
+            } else if (type == "symlink") {
+                member.type = FSAccessor::Type::tSymlink;
+                member.target = v.value("target", "");
+            } else return;
+        };
+
+        json v = json::parse(listing);
+        recurse(root, v);
     }
 
-    NarMember* find(const Path & path)
+    NarMember * find(const Path & path)
     {
         Path canon = path == "" ? "" : canonPath(path);
-        NarMember* current = &root;
+        NarMember * current = &root;
         auto end = path.end();
-        for(auto it = path.begin(); it != end; ) {
+        for (auto it = path.begin(); it != end; ) {
             // because it != end, the remaining component is non-empty so we need
             // a directory
-            if(current->type != FSAccessor::Type::tDirectory) return nullptr;
+            if (current->type != FSAccessor::Type::tDirectory) return nullptr;
 
             // skip slash (canonPath above ensures that this is always a slash)
             assert(*it == '/');
@@ -109,7 +155,7 @@ struct NarIndexer : ParseSink, StringSource
             // lookup current component
             auto next = std::find(it, end, '/');
             auto child = current->children.find(std::string(it, next));
-            if(child == current->children.end()) return nullptr;
+            if (child == current->children.end()) return nullptr;
             current = &child->second;
 
             it = next;
@@ -118,28 +164,16 @@ struct NarIndexer : ParseSink, StringSource
         return current;
     }
 
-    NarMember& at(const Path & path) {
+    NarMember & get(const Path & path) {
         auto result = find(path);
-        if(result == nullptr) {
-            throw Error(format("NAR file does not contain path '%1%'") % path);
-        }
+        if (result == nullptr)
+            throw Error("NAR file does not contain path '%1%'", path);
         return *result;
     }
-};
-
-struct NarAccessor : public FSAccessor
-{
-    ref<const std::string> nar;
-    NarIndexer indexer;
-
-    NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar)
-    {
-        parseDump(indexer, indexer);
-    }
 
     Stat stat(const Path & path) override
     {
-        auto i = indexer.find(path);
+        auto i = find(path);
         if (i == nullptr)
             return {FSAccessor::Type::tMissing, 0, false};
         return {i->type, i->size, i->isExecutable, i->start};
@@ -147,30 +181,33 @@ struct NarAccessor : public FSAccessor
 
     StringSet readDirectory(const Path & path) override
     {
-        auto i = indexer.at(path);
+        auto i = get(path);
 
         if (i.type != FSAccessor::Type::tDirectory)
             throw Error(format("path '%1%' inside NAR file is not a directory") % path);
 
         StringSet res;
-        for(auto&& child : i.children) {
+        for (auto & child : i.children)
             res.insert(child.first);
 
-        }
         return res;
     }
 
     std::string readFile(const Path & path) override
     {
-        auto i = indexer.at(path);
+        auto i = get(path);
         if (i.type != FSAccessor::Type::tRegular)
             throw Error(format("path '%1%' inside NAR file is not a regular file") % path);
+
+        if (getNarBytes) return getNarBytes(i.start, i.size);
+
+        assert(nar);
         return std::string(*nar, i.start, i.size);
     }
 
     std::string readLink(const Path & path) override
     {
-        auto i = indexer.at(path);
+        auto i = get(path);
         if (i.type != FSAccessor::Type::tSymlink)
             throw Error(format("path '%1%' inside NAR file is not a symlink") % path);
         return i.target;
@@ -182,6 +219,12 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
     return make_ref<NarAccessor>(nar);
 }
 
+ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
+    GetNarBytes getNarBytes)
+{
+    return make_ref<NarAccessor>(listing, getNarBytes);
+}
+
 void listNar(JSONPlaceholder & res, ref<FSAccessor> accessor,
     const Path & path, bool recurse)
 {
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index ed8fe15cad23..1903355a236e 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -8,6 +8,16 @@ namespace nix {
    file. */
 ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
 
+/* Create a NAR accessor from a NAR listing (in the format produced by
+   listNar()). The callback getNarBytes(offset, length) is used by the
+   readFile() method of the accessor to get the contents of files
+   inside the NAR. */
+typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes;
+
+ref<FSAccessor> makeLazyNarAccessor(
+    const std::string & listing,
+    GetNarBytes getNarBytes);
+
 class JSONPlaceholder;
 
 /* Write a JSON representation of the contents of a NAR (except file
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index ba9620a175bb..5233fb2c239b 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -1,5 +1,10 @@
 #include "remote-fs-accessor.hh"
 #include "nar-accessor.hh"
+#include "json.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 namespace nix {
 
@@ -11,20 +16,30 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
         createDirs(cacheDir);
 }
 
-Path RemoteFSAccessor::makeCacheFile(const Path & storePath)
+Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
 {
     assert(cacheDir != "");
-    return fmt("%s/%s.nar", cacheDir, storePathToHash(storePath));
+    return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
 }
 
-void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar)
+void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
+    ref<FSAccessor> narAccessor)
 {
-    try {
-        if (cacheDir == "") return;
-        /* FIXME: do this asynchronously. */
-        writeFile(makeCacheFile(storePath), nar);
-    } catch (...) {
-        ignoreException();
+    nars.emplace(storePath, narAccessor);
+
+    if (cacheDir != "") {
+        try {
+            std::ostringstream str;
+            JSONPlaceholder jsonRoot(str);
+            listNar(jsonRoot, narAccessor, "", true);
+            writeFile(makeCacheFile(storePath, "ls"), str.str());
+
+            /* FIXME: do this asynchronously. */
+            writeFile(makeCacheFile(storePath, "nar"), nar);
+
+        } catch (...) {
+            ignoreException();
+        }
     }
 }
 
@@ -42,20 +57,49 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
     if (i != nars.end()) return {i->second, restPath};
 
     StringSink sink;
+    std::string listing;
+    Path cacheFile;
+
+    if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
+
+        try {
+            listing = nix::readFile(makeCacheFile(storePath, "ls"));
+
+            auto narAccessor = makeLazyNarAccessor(listing,
+                [cacheFile](uint64_t offset, uint64_t length) {
+
+                    AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
+                    if (!fd)
+                        throw SysError("opening NAR cache file '%s'", cacheFile);
+
+                    if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset)
+                        throw SysError("seeking in '%s'", cacheFile);
+
+                    std::string buf(length, 0);
+                    readFull(fd.get(), (unsigned char *) buf.data(), length);
+
+                    return buf;
+                });
+
+            nars.emplace(storePath, narAccessor);
+            return {narAccessor, restPath};
+
+        } catch (SysError &) { }
+
+        try {
+            *sink.s = nix::readFile(cacheFile);
 
-    try {
-        if (cacheDir != "")
-            *sink.s = nix::readFile(makeCacheFile(storePath));
-    } catch (SysError &) { }
+            auto narAccessor = makeNarAccessor(sink.s);
+            nars.emplace(storePath, narAccessor);
+            return {narAccessor, restPath};
 
-    if (sink.s->empty()) {
-        store->narFromPath(storePath, sink);
-        addToCache(storePath, *sink.s);
+        } catch (SysError &) { }
     }
 
-    auto accessor = makeNarAccessor(sink.s);
-    nars.emplace(storePath, accessor);
-    return {accessor, restPath};
+    store->narFromPath(storePath, sink);
+    auto narAccessor = makeNarAccessor(sink.s);
+    addToCache(storePath, *sink.s, narAccessor);
+    return {narAccessor, restPath};
 }
 
 FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 2a3fc01eff58..4afb3be95736 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -18,9 +18,10 @@ class RemoteFSAccessor : public FSAccessor
 
     friend class BinaryCacheStore;
 
-    Path makeCacheFile(const Path & storePath);
+    Path makeCacheFile(const Path & storePath, const std::string & ext);
 
-    void addToCache(const Path & storePath, const std::string & nar);
+    void addToCache(const Path & storePath, const std::string & nar,
+        ref<FSAccessor> narAccessor);
 
 public:
generated by cgit-pink 1.4.1 (git 2.44.2) at 2024-11-09T19ยท05+0000