diff options
Diffstat (limited to 'src/libstore/binary-cache-store.cc')
| -rw-r--r-- | src/libstore/binary-cache-store.cc | 180 |
1 files changed, 74 insertions, 106 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7016eedfa701..411d10130a31 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -8,20 +8,20 @@ #include "sync.hh" #include "worker-protocol.hh" #include "nar-accessor.hh" +#include "nar-info-disk-cache.hh" #include <chrono> namespace nix { BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile) + const StoreParams & params) : localStore(localStore) + , compression(get(params, "compression", "xz")) { - if (secretKeyFile != "") { + auto secretKeyFile = get(params, "secret-key", ""); + if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); - publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys); - publicKeys->emplace(secretKey->name, secretKey->toPublicKey()); - } StringSink sink; sink << narVersionMagic1; @@ -40,57 +40,65 @@ void BinaryCacheStore::notImpl() throw Error("operation not implemented for binary cache stores"); } -const BinaryCacheStore::Stats & BinaryCacheStore::getStats() -{ - return stats; -} - Path BinaryCacheStore::narInfoFileFor(const Path & storePath) { assertStorePath(storePath); return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToCache(const ValidPathInfo & info, - const string & nar) +void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref<std::string> nar) { + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) + try { + if (ref != info.path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error(format("cannot add ‘%s’ to the binary cache because the reference ‘%s’ is not valid") + % info.path % ref); + } + auto narInfoFile = narInfoFileFor(info.path); if (fileExists(narInfoFile)) return; - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref<NarInfo>(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); - if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash) + if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); /* Compress the NAR. */ - narInfo->compression = "xz"; + narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - string narXz = compressXZ(nar); + auto narCompressed = compress(compression, nar); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, narXz); - narInfo->fileSize = narXz.size(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ - narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + ""); if (!fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, narXz); + upsertFile(narInfo->url, *narCompressed); } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); - stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -98,69 +106,21 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, upsertFile(narInfoFile, narInfo->to_string()); - { - auto state_(state.lock()); - state_->narInfoCache.upsert(narInfo->path, narInfo); - stats.narInfoCacheSize = state_->narInfoCache.size(); - } + auto hashPart = storePathToHash(narInfo->path); - stats.narInfoWrite++; -} - -NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) -{ { auto state_(state.lock()); - auto res = state_->narInfoCache.get(storePath); - if (res) { - stats.narInfoReadAverted++; - if (!*res) - throw InvalidPath(format("path ‘%s’ is not valid") % storePath); - return **res; - } - } - - auto narInfoFile = narInfoFileFor(storePath); - auto data = getFile(narInfoFile); - if (!data) { - stats.narInfoMissing++; - auto state_(state.lock()); - state_->narInfoCache.upsert(storePath, 0); - stats.narInfoCacheSize = state_->narInfoCache.size(); - throw InvalidPath(format("path ‘%s’ is not valid") % storePath); - } - - auto narInfo = make_ref<NarInfo>(*data, narInfoFile); - if (narInfo->path != storePath) - throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath); - - stats.narInfoRead++; - - if (publicKeys) { - if (!narInfo->checkSignatures(*publicKeys)) - throw Error(format("no good signature on NAR info file ‘%1%’") % narInfoFile); + state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); } - { - auto state_(state.lock()); - state_->narInfoCache.upsert(storePath, narInfo); - stats.narInfoCacheSize = state_->narInfoCache.size(); - } + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); - return *narInfo; + stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPath(const Path & storePath) +bool BinaryCacheStore::isValidPathUncached(const Path & storePath) { - { - auto state_(state.lock()); - auto res = state_->narInfoCache.get(storePath); - if (res) { - stats.narInfoReadAverted++; - return *res != 0; - } - } - // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. @@ -169,23 +129,23 @@ bool BinaryCacheStore::isValidPath(const Path & storePath) void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) { - auto res = readNarInfo(storePath); + auto info = queryPathInfo(storePath).cast<const NarInfo>(); - auto nar = getFile(res.url); + auto nar = getFile(info->url); - if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % res.url); + if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % info->url); stats.narRead++; stats.narReadCompressedBytes += nar->size(); /* Decompress the NAR. FIXME: would be nice to have the remote side do this. */ - if (res.compression == "none") - ; - else if (res.compression == "xz") - nar = decompressXZ(*nar); - else - throw Error(format("unknown NAR compression type ‘%1%’") % nar); + try { + nar = decompress(info->compression, ref<std::string>(nar)); + } catch (UnknownCompressionMethod &) { + throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’") + % storePath % info->compression); + } stats.narReadBytes += nar->size(); @@ -200,13 +160,13 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink { assert(!sign); - auto res = readNarInfo(storePath); + auto res = queryPathInfo(storePath); narFromPath(storePath, sink); // FIXME: check integrity of NAR. - sink << exportMagic << storePath << res.references << res.deriver << 0; + sink << exportMagic << storePath << res->references << res->deriver << 0; } Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, @@ -244,9 +204,17 @@ struct NopSink : ParseSink { }; -ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) +std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Path & storePath) { - return ValidPathInfo(readNarInfo(storePath)); + auto narInfoFile = narInfoFileFor(storePath); + auto data = getFile(narInfoFile); + if (!data) return 0; + + auto narInfo = make_ref<NarInfo>(*data, narInfoFile); + + stats.narInfoRead++; + + return std::shared_ptr<NarInfo>(narInfo); } void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, @@ -257,16 +225,16 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, if (!localStore) return; for (auto & storePath : paths) { - if (!localStore->isValidPath(storePath)) { + try { + auto info = localStore->queryPathInfo(storePath); + SubstitutablePathInfo sub; + sub.references = info->references; + sub.downloadSize = 0; + sub.narSize = info->narSize; + infos.emplace(storePath, sub); + } catch (InvalidPath &) { left.insert(storePath); - continue; } - ValidPathInfo info = localStore->queryPathInfo(storePath); - SubstitutablePathInfo sub; - sub.references = info.references; - sub.downloadSize = 0; - sub.narSize = info.narSize; - infos.emplace(storePath, sub); } if (settings.useSubstitutes) @@ -296,7 +264,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); if (repair || !isValidPath(info.path)) - addToCache(info, *sink.s); + addToCache(info, sink.s); return info.path; } @@ -311,7 +279,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToCache(info, *sink.s); + addToCache(info, sink.s); } return info.path; @@ -332,16 +300,16 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) if (!localStore->isValidPath(storePath)) localStore->ensurePath(storePath); - ValidPathInfo info = localStore->queryPathInfo(storePath); + auto info = localStore->queryPathInfo(storePath); - for (auto & ref : info.references) + for (auto & ref : info->references) if (ref != storePath) ensurePath(ref); StringSink sink; dumpPath(storePath, sink); - addToCache(info, *sink.s); + addToCache(*info, sink.s); } } @@ -439,7 +407,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> a bool haveSignature = readInt(source) == 1; assert(!haveSignature); - addToCache(info, *tee.data); + addToCache(info, tee.data); auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); if (accessor_) |