From 5acb6914020a94be8468d7670b564388c59794ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Apr 2016 17:02:57 +0200 Subject: BinaryCacheStore: Support "none" compression method --- src/libstore/binary-cache-store.cc | 41 +++++++++++++++++--------------------- src/libstore/binary-cache-store.hh | 4 +++- src/libstore/builtins.cc | 2 +- 3 files changed, 22 insertions(+), 25 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 063d1cce267f..90d7fa18b94b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -17,6 +17,7 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(std::shared_ptr localStore, const StoreParams & params) : localStore(localStore) + , compression(get(params, "compression", "xz")) { auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") @@ -45,8 +46,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath) return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToCache(const ValidPathInfo & info, - const string & nar) +void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref nar) { /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -62,40 +62,40 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, auto narInfoFile = narInfoFileFor(info.path); if (fileExists(narInfoFile)) return; - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); /* Compress the NAR. */ - narInfo->compression = "xz"; + narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - string narXz = compressXZ(nar); + auto narCompressed = compress(compression, nar); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, narXz); - narInfo->fileSize = narXz.size(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; if (!fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, narXz); + upsertFile(narInfo->url, *narCompressed); } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); - stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -137,12 +137,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) /* Decompress the NAR. FIXME: would be nice to have the remote side do this. */ - if (info->compression == "none") - ; - else if (info->compression == "xz") - nar = decompressXZ(*nar); - else - throw Error(format("unknown NAR compression type ‘%1%’") % info->compression); + nar = decompress(info->compression, ref(nar)); stats.narReadBytes += nar->size(); @@ -261,7 +256,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); if (repair || !isValidPath(info.path)) - addToCache(info, *sink.s); + addToCache(info, sink.s); return info.path; } @@ -276,7 +271,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToCache(info, *sink.s); + addToCache(info, sink.s); } return info.path; @@ -306,7 +301,7 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) StringSink sink; dumpPath(storePath, sink); - addToCache(*info, *sink.s); + addToCache(*info, sink.s); } } @@ -404,7 +399,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr a bool haveSignature = readInt(source) == 1; assert(!haveSignature); - addToCache(info, *tee.data); + addToCache(info, tee.data); auto accessor_ = std::dynamic_pointer_cast(accessor); if (accessor_) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index f6fa0cac03db..46a38a1e0fc3 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -19,6 +19,8 @@ private: std::shared_ptr localStore; + std::string compression; + protected: BinaryCacheStore(std::shared_ptr localStore, @@ -44,7 +46,7 @@ private: std::string narInfoFileFor(const Path & storePath); - void addToCache(const ValidPathInfo & info, const string & nar); + void addToCache(const ValidPathInfo & info, ref nar); public: diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index 50417b644a02..a4785d6905bb 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -31,7 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv) auto unpack = drv.env.find("unpack"); if (unpack != drv.env.end() && unpack->second == "1") { if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data.data = decompressXZ(*data.data); + data.data = decompress("xz", ref(data.data)); StringSource source(*data.data); restorePath(storePath, source); } else -- cgit 1.4.1