diff options
| -rw-r--r-- | Makefile.config.in | 1 | ||||
| -rw-r--r-- | configure.ac | 1 | ||||
| -rw-r--r-- | release.nix | 8 | ||||
| -rw-r--r-- | shell.nix | 3 | ||||
| -rw-r--r-- | src/libstore/download.cc | 21 | ||||
| -rw-r--r-- | src/libutil/compression.cc | 7 | ||||
| -rw-r--r-- | src/libutil/local.mk | 2 |
7 files changed, 38 insertions, 5 deletions
diff --git a/Makefile.config.in b/Makefile.config.in index d4953b521c48..fccf63b3627d 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -14,6 +14,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +bro = @bro@ datadir = @datadir@ datarootdir = @datarootdir@ docdir = @docdir@ diff --git a/configure.ac b/configure.ac index 34dcd6b2a7dc..21ca78af0fdb 100644 --- a/configure.ac +++ b/configure.ac @@ -128,6 +128,7 @@ NEED_PROG(gzip, gzip) NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(pv, pv, pv) +NEED_PROG(bro, bro) # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). diff --git a/release.nix b/release.nix index e61e81bdf37e..a266af7c2e14 100644 --- a/release.nix +++ b/release.nix @@ -24,7 +24,8 @@ let inherit officialRelease; buildInputs = - [ curl bison flex perl libxml2 libxslt bzip2 xz + [ curl bison flex perl libxml2 libxslt + bzip2 xz brotli pkgconfig sqlite libsodium boehmgc docbook5 docbook5_xsl autoconf-archive @@ -73,7 +74,10 @@ let src = tarball; buildInputs = - [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ] + [ curl perl + bzip2 xz brotli + openssl pkgconfig sqlite boehmgc + ] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) (aws-sdk-cpp.override { diff --git a/shell.nix b/shell.nix index 4c1608230cee..df0ad01df583 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,8 @@ with import <nixpkgs> {}; name = "nix"; buildInputs = - [ curl bison flex perl libxml2 libxslt bzip2 xz + [ curl bison flex perl libxml2 libxslt + bzip2 xz brotli pkgconfig sqlite libsodium boehmgc docbook5 docbook5_xsl autoconf-archive diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 6567a4dc4754..d9b8fbc08080 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -5,6 +5,8 @@ #include "store-api.hh" #include "archive.hh" #include "s3.hh" +#include "compression.hh" + #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> #endif @@ -70,6 +72,8 @@ struct CurlDownloader : public Downloader struct curl_slist * requestHeaders = 0; + std::string encoding; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request) : downloader(downloader), request(request) { @@ -127,6 +131,7 @@ struct CurlDownloader : public Downloader auto ss = tokenizeString<vector<string>>(line, " "); status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); + encoding = ""; } else { auto i = line.find(':'); if (i != string::npos) { @@ -142,7 +147,8 @@ struct CurlDownloader : public Downloader debug(format("shutting down on 200 HTTP response with expected ETag")); return 0; } - } + } else if (name == "content-encoding") + encoding = trim(string(line, i + 1));; } } return realSize; @@ -268,7 +274,18 @@ struct CurlDownloader : public Downloader { result.cached = httpStatus == 304; done = true; - callSuccess(success, failure, const_cast<const DownloadResult &>(result)); + + /* Ad hoc support for brotli, since curl doesn't do + this yet. */ + try { + if (encoding == "br") + result.data = decompress("br", *result.data); + + callSuccess(success, failure, const_cast<const DownloadResult &>(result)); + } catch (...) { + done = true; + callFailure(failure, std::current_exception()); + } } else { Error err = (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound : diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a3bbb5170d9f..723b072af968 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -89,6 +89,11 @@ static ref<std::string> decompressBzip2(const std::string & in) } } +static ref<std::string> decompressBrotli(const std::string & in) +{ + return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in)); +} + ref<std::string> compress(const std::string & method, const std::string & in) { StringSink ssink; @@ -106,6 +111,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in) return decompressXZ(in); else if (method == "bzip2") return decompressBzip2(in); + else if (method == "br") + return decompressBrotli(in); else throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index cac5c8795db7..0721b21c2089 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat + +libutil_CXXFLAGS = -DBRO=\"$(bro)\" |