From 5acb6914020a94be8468d7670b564388c59794ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Apr 2016 17:02:57 +0200 Subject: BinaryCacheStore: Support "none" compression method --- src/libstore/binary-cache-store.cc | 41 +++++++++++++++++--------------------- src/libstore/binary-cache-store.hh | 4 +++- src/libstore/builtins.cc | 2 +- src/libutil/compression.cc | 28 ++++++++++++++++++++++---- src/libutil/compression.hh | 7 +++++-- 5 files changed, 51 insertions(+), 31 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 063d1cce267f..90d7fa18b94b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -17,6 +17,7 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(std::shared_ptr localStore, const StoreParams & params) : localStore(localStore) + , compression(get(params, "compression", "xz")) { auto secretKeyFile = get(params, "secret-key", ""); if (secretKeyFile != "") @@ -45,8 +46,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath) return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToCache(const ValidPathInfo & info, - const string & nar) +void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref nar) { /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -62,40 +62,40 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, auto narInfoFile = narInfoFileFor(info.path); if (fileExists(narInfoFile)) return; - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); /* Compress the NAR. */ - narInfo->compression = "xz"; + narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - string narXz = compressXZ(nar); + auto narCompressed = compress(compression, nar); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, narXz); - narInfo->fileSize = narXz.size(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; if (!fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, narXz); + upsertFile(narInfo->url, *narCompressed); } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); - stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -137,12 +137,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) /* Decompress the NAR. FIXME: would be nice to have the remote side do this. */ - if (info->compression == "none") - ; - else if (info->compression == "xz") - nar = decompressXZ(*nar); - else - throw Error(format("unknown NAR compression type ‘%1%’") % info->compression); + nar = decompress(info->compression, ref(nar)); stats.narReadBytes += nar->size(); @@ -261,7 +256,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); if (repair || !isValidPath(info.path)) - addToCache(info, *sink.s); + addToCache(info, sink.s); return info.path; } @@ -276,7 +271,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToCache(info, *sink.s); + addToCache(info, sink.s); } return info.path; @@ -306,7 +301,7 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) StringSink sink; dumpPath(storePath, sink); - addToCache(*info, *sink.s); + addToCache(*info, sink.s); } } @@ -404,7 +399,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr a bool haveSignature = readInt(source) == 1; assert(!haveSignature); - addToCache(info, *tee.data); + addToCache(info, tee.data); auto accessor_ = std::dynamic_pointer_cast(accessor); if (accessor_) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index f6fa0cac03db..46a38a1e0fc3 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -19,6 +19,8 @@ private: std::shared_ptr localStore; + std::string compression; + protected: BinaryCacheStore(std::shared_ptr localStore, @@ -44,7 +46,7 @@ private: std::string narInfoFileFor(const Path & storePath); - void addToCache(const ValidPathInfo & info, const string & nar); + void addToCache(const ValidPathInfo & info, ref nar); public: diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index 50417b644a02..a4785d6905bb 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -31,7 +31,7 @@ void builtinFetchurl(const BasicDerivation & drv) auto unpack = drv.env.find("unpack"); if (unpack != drv.env.end() && unpack->second == "1") { if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data.data = decompressXZ(*data.data); + data.data = decompress("xz", ref(data.data)); StringSource source(*data.data); restorePath(storePath, source); } else diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index beede13211fa..e8a820d30429 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -15,7 +15,7 @@ struct LzmaStream lzma_stream & operator()() { return strm; } }; -std::string compressXZ(const std::string & in) +static ref compressXZ(const std::string & in) { LzmaStream strm; @@ -28,7 +28,7 @@ std::string compressXZ(const std::string & in) lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; - string res; + ref res = make_ref(); strm().next_in = (uint8_t *) in.c_str(); strm().avail_in = in.size(); strm().next_out = outbuf; @@ -43,7 +43,7 @@ std::string compressXZ(const std::string & in) lzma_ret ret = lzma_code(&strm(), action); if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out); + res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); strm().next_out = outbuf; strm().avail_out = sizeof(outbuf); } @@ -56,7 +56,7 @@ std::string compressXZ(const std::string & in) } } -ref decompressXZ(const std::string & in) +static ref decompressXZ(const std::string & in) { LzmaStream strm; @@ -95,4 +95,24 @@ ref decompressXZ(const std::string & in) } } +ref compress(const std::string & method, ref in) +{ + if (method == "none") + return in; + else if (method == "xz") + return compressXZ(*in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +ref decompress(const std::string & method, ref in) +{ + if (method == "none") + return in; + else if (method == "xz") + return decompressXZ(*in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 79a796db7756..33c465df8455 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,13 +1,16 @@ #pragma once #include "ref.hh" +#include "types.hh" #include namespace nix { -std::string compressXZ(const std::string & in); +ref compress(const std::string & method, ref in); -ref decompressXZ(const std::string & in); +ref decompress(const std::string & method, ref in); + +MakeError(UnknownCompressionMethod, Error); } -- cgit 1.4.1