diff options
Diffstat (limited to 'src/libstore/binary-cache-store.cc')
| -rw-r--r-- | src/libstore/binary-cache-store.cc | 178 |
1 files changed, 162 insertions, 16 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ab80e4032f04..3e07a2aa2b60 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -8,14 +8,79 @@ #include "sync.hh" #include "remote-fs-accessor.hh" #include "nar-info-disk-cache.hh" +#include "nar-accessor.hh" +#include "json.hh" #include <chrono> +#include <future> + namespace nix { +/* Given requests for a path /nix/store/<x>/<y>, this accessor will + first download the NAR for /nix/store/<x> from the binary cache, + build a NAR accessor for that NAR, and use that to access <y>. */ +struct BinaryCacheStoreAccessor : public FSAccessor +{ + ref<BinaryCacheStore> store; + + std::map<Path, ref<FSAccessor>> nars; + + BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) + : store(store) + { + } + + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) + { + auto path = canonPath(path_); + + auto storePath = store->toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + store->narFromPath(storePath, sink); + + auto accessor = makeNarAccessor(sink.s); + nars.emplace(storePath, accessor); + return {accessor, restPath}; + } + + Stat stat(const Path & path) override + { + auto res = fetch(path); + return res.first->stat(res.second); + } + + StringSet readDirectory(const Path & path) override + { + auto res = fetch(path); + return res.first->readDirectory(res.second); + } + + std::string readFile(const Path & path) override + { + auto res = fetch(path); + return res.first->readFile(res.second); + } + + std::string readLink(const Path & path) override + { + auto res = fetch(path); + return res.first->readLink(res.second); + } +}; + BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) , compression(get(params, "compression", "xz")) + , writeNARListing(get(params, "write-nar-listing", "0") == "1") { auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") @@ -57,14 +122,27 @@ void BinaryCacheStore::notImpl() throw Error("operation not implemented for binary cache stores"); } +std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) +{ + std::promise<std::shared_ptr<std::string>> promise; + getFile(path, + [&](std::shared_ptr<std::string> result) { + promise.set_value(result); + }, + [&](std::exception_ptr exc) { + promise.set_exception(exc); + }); + return promise.get_future().get(); +} + Path BinaryCacheStore::narInfoFileFor(const Path & storePath) { assertStorePath(storePath); return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar, - bool repair, bool dontCheckSigs) +void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { if (!repair && isValidPath(info.path)) return; @@ -81,20 +159,83 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string auto narInfoFile = narInfoFileFor(info.path); - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref<NarInfo>(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); + + /* Optionally write a JSON file containing a listing of the + contents of the NAR. */ + if (writeNARListing) { + std::ostringstream jsonOut; + + { + JSONObject jsonRoot(jsonOut); + jsonRoot.attr("version", 1); + + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->nars.emplace(info.path, narAccessor); + + std::function<void(const Path &, JSONPlaceholder &)> recurse; + + recurse = [&](const Path & path, JSONPlaceholder & res) { + auto st = narAccessor->stat(path); + + auto obj = res.object(); + + switch (st.type) { + case FSAccessor::Type::tRegular: + obj.attr("type", "regular"); + obj.attr("size", st.fileSize); + if (st.isExecutable) + obj.attr("executable", true); + break; + case FSAccessor::Type::tDirectory: + obj.attr("type", "directory"); + { + auto res2 = obj.object("entries"); + for (auto & name : narAccessor->readDirectory(path)) { + auto res3 = res2.placeholder(name); + recurse(path + "/" + name, res3); + } + } + break; + case FSAccessor::Type::tSymlink: + obj.attr("type", "symlink"); + obj.attr("target", narAccessor->readLink(path)); + break; + default: + abort(); + } + }; + + { + auto res = jsonRoot.placeholder("root"); + recurse("", res); + } + } + + upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str())); + } + + else { + if (accessor_) + accessor_->nars.emplace(info.path, makeNarAccessor(nar)); + } + /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - auto narCompressed = compress(compression, nar); + auto narCompressed = compress(compression, *nar); auto now2 = std::chrono::steady_clock::now(); narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); @@ -102,7 +243,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ @@ -116,7 +257,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); + stats.narWriteBytes += nar->size(); stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; @@ -175,17 +316,22 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) sink((unsigned char *) nar->c_str(), nar->size()); } -std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Path & storePath) +void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, + std::function<void(std::shared_ptr<ValidPathInfo>)> success, + std::function<void(std::exception_ptr exc)> failure) { auto narInfoFile = narInfoFileFor(storePath); - auto data = getFile(narInfoFile); - if (!data) return 0; - auto narInfo = make_ref<NarInfo>(*this, *data, narInfoFile); + getFile(narInfoFile, + [=](std::shared_ptr<std::string> data) { + if (!data) return success(0); - stats.narInfoRead++; + stats.narInfoRead++; - return std::shared_ptr<NarInfo>(narInfo); + callSuccess(success, failure, (std::shared_ptr<ValidPathInfo>) + std::make_shared<NarInfo>(*this, *data, narInfoFile)); + }, + failure); } Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, @@ -210,7 +356,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, ValidPathInfo info; info.path = makeFixedOutputPath(recursive, h, name); - addToStore(info, *sink.s, repair); + addToStore(info, sink.s, repair, false, 0); return info.path; } @@ -225,7 +371,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, *sink.s, repair); + addToStore(info, sink.s, repair, false, 0); } return info.path; |