From 4dde0b0562115525f76943653357410835910a99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Apr 2016 17:43:37 +0200 Subject: BinaryCacheStore: Support bzip2 compression --- src/libutil/compression.cc | 150 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 116 insertions(+), 34 deletions(-) (limited to 'src/libutil/compression.cc') diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index e8a820d30429..4d15d2acdd4e 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,90 +1,88 @@ #include "compression.hh" #include "util.hh" +#include "finally.hh" #include +#include #include +#include namespace nix { -/* RAII wrapper around lzma_stream. */ -struct LzmaStream -{ - lzma_stream strm; - LzmaStream() : strm(LZMA_STREAM_INIT) { }; - ~LzmaStream() { lzma_end(&strm); }; - lzma_stream & operator()() { return strm; } -}; - static ref compressXZ(const std::string & in) { - LzmaStream strm; + lzma_stream strm; // FIXME: apply the x86 BCJ filter? lzma_ret ret = lzma_easy_encoder( - &strm(), 6, LZMA_CHECK_CRC64); + &strm, 6, LZMA_CHECK_CRC64); if (ret != LZMA_OK) throw Error("unable to initialise lzma encoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; ref res = make_ref(); - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) return res; if (ret != LZMA_OK) - throw Error("error while decompressing xz file"); + throw Error("error while compressing xz file"); } } static ref decompressXZ(const std::string & in) { - LzmaStream strm; + lzma_stream strm; lzma_ret ret = lzma_stream_decoder( - &strm(), UINT64_MAX, LZMA_CONCATENATED); + &strm, UINT64_MAX, LZMA_CONCATENATED); if (ret != LZMA_OK) throw Error("unable to initialise lzma decoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; ref res = make_ref(); - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res->append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) @@ -95,12 +93,94 @@ static ref decompressXZ(const std::string & in) } } +static ref compressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 encoder"); + + Finally free([&]() { BZ2_bzCompressEnd(&strm); }); + + int action = BZ_RUN; + char outbuf[BUFSIZ]; + ref res = make_ref(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + if (strm.avail_in == 0) + action = BZ_FINISH; + + int ret = BZ2_bzCompress(&strm, action); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK && ret != BZ_FINISH_OK) + Error("error while compressing bzip2 file"); + } + + return res; +} + +static ref decompressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 decoder"); + + Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + + char outbuf[BUFSIZ]; + ref res = make_ref(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK) + throw Error("error while decompressing bzip2 file"); + } +} + ref compress(const std::string & method, ref in) { if (method == "none") return in; else if (method == "xz") return compressXZ(*in); + else if (method == "bzip2") + return compressBzip2(*in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } @@ -111,6 +191,8 @@ ref decompress(const std::string & method, ref in) return in; else if (method == "xz") return decompressXZ(*in); + else if (method == "bzip2") + return decompressBzip2(*in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } -- cgit 1.4.1