about summary refs log tree commit diff
diff options
context:
space:
mode:
authorEelco Dolstra <edolstra@gmail.com>2017-03-13T13·40+0100
committerEelco Dolstra <edolstra@gmail.com>2017-03-15T15·49+0100
commite8186085e07104d4b844208613c2d704b5b57dec (patch)
treecce5074dd45b109c0edd5427fa06c4f8ec668bbe
parent73d7a51ee6942f681db468dc8e3c631b4d3daa4e (diff)
Add support for brotli compression
Build logs on cache.nixos.org are compressed using Brotli (since this
allows them to be decompressed automatically by Chrome and Firefox),
so it's handy if "nix log" can decompress them.
-rw-r--r--Makefile.config.in1
-rw-r--r--configure.ac1
-rw-r--r--release.nix8
-rw-r--r--shell.nix3
-rw-r--r--src/libstore/download.cc21
-rw-r--r--src/libutil/compression.cc7
-rw-r--r--src/libutil/local.mk2
7 files changed, 38 insertions, 5 deletions
diff --git a/Makefile.config.in b/Makefile.config.in
index d4953b521c48..fccf63b3627d 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -14,6 +14,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 bash = @bash@
 bindir = @bindir@
+bro = @bro@
 datadir = @datadir@
 datarootdir = @datarootdir@
 docdir = @docdir@
diff --git a/configure.ac b/configure.ac
index 34dcd6b2a7dc..21ca78af0fdb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -128,6 +128,7 @@ NEED_PROG(gzip, gzip)
 NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(pv, pv, pv)
+NEED_PROG(bro, bro)
 
 
 # Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).
diff --git a/release.nix b/release.nix
index e61e81bdf37e..a266af7c2e14 100644
--- a/release.nix
+++ b/release.nix
@@ -24,7 +24,8 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison flex perl libxml2 libxslt bzip2 xz
+          [ curl bison flex perl libxml2 libxslt
+            bzip2 xz brotli
             pkgconfig sqlite libsodium boehmgc
             docbook5 docbook5_xsl
             autoconf-archive
@@ -73,7 +74,10 @@ let
         src = tarball;
 
         buildInputs =
-          [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ]
+          [ curl perl
+            bzip2 xz brotli
+            openssl pkgconfig sqlite boehmgc
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
             (aws-sdk-cpp.override {
diff --git a/shell.nix b/shell.nix
index 4c1608230cee..df0ad01df583 100644
--- a/shell.nix
+++ b/shell.nix
@@ -6,7 +6,8 @@ with import <nixpkgs> {};
   name = "nix";
 
   buildInputs =
-    [ curl bison flex perl libxml2 libxslt bzip2 xz
+    [ curl bison flex perl libxml2 libxslt
+      bzip2 xz brotli
       pkgconfig sqlite libsodium boehmgc
       docbook5 docbook5_xsl
       autoconf-archive
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 6567a4dc4754..d9b8fbc08080 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -5,6 +5,8 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "s3.hh"
+#include "compression.hh"
+
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
 #endif
@@ -70,6 +72,8 @@ struct CurlDownloader : public Downloader
 
         struct curl_slist * requestHeaders = 0;
 
+        std::string encoding;
+
         DownloadItem(CurlDownloader & downloader, const DownloadRequest & request)
             : downloader(downloader), request(request)
         {
@@ -127,6 +131,7 @@ struct CurlDownloader : public Downloader
                 auto ss = tokenizeString<vector<string>>(line, " ");
                 status = ss.size() >= 2 ? ss[1] : "";
                 result.data = std::make_shared<std::string>();
+                encoding = "";
             } else {
                 auto i = line.find(':');
                 if (i != string::npos) {
@@ -142,7 +147,8 @@ struct CurlDownloader : public Downloader
                             debug(format("shutting down on 200 HTTP response with expected ETag"));
                             return 0;
                         }
-                    }
+                    } else if (name == "content-encoding")
+                        encoding = trim(string(line, i + 1));;
                 }
             }
             return realSize;
@@ -268,7 +274,18 @@ struct CurlDownloader : public Downloader
             {
                 result.cached = httpStatus == 304;
                 done = true;
-                callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+
+                /* Ad hoc support for brotli, since curl doesn't do
+                   this yet. */
+                try {
+                    if (encoding == "br")
+                        result.data = decompress("br", *result.data);
+
+                    callSuccess(success, failure, const_cast<const DownloadResult &>(result));
+                } catch (...) {
+                    done = true;
+                    callFailure(failure, std::current_exception());
+                }
             } else {
                 Error err =
                     (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) ? NotFound :
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index a3bbb5170d9f..723b072af968 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -89,6 +89,11 @@ static ref<std::string> decompressBzip2(const std::string & in)
     }
 }
 
+static ref<std::string> decompressBrotli(const std::string & in)
+{
+    return make_ref<std::string>(runProgram(BRO, true, {"-d"}, in));
+}
+
 ref<std::string> compress(const std::string & method, const std::string & in)
 {
     StringSink ssink;
@@ -106,6 +111,8 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
         return decompressXZ(in);
     else if (method == "bzip2")
         return decompressBzip2(in);
+    else if (method == "br")
+        return decompressBrotli(in);
     else
         throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method);
 }
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index cac5c8795db7..0721b21c2089 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -9,3 +9,5 @@ libutil_SOURCES := $(wildcard $(d)/*.cc)
 libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS)
 
 libutil_LIBS = libformat
+
+libutil_CXXFLAGS = -DBRO=\"$(bro)\"