about summary refs log tree commit diff
diff options
context:
space:
mode:
authorEelco Dolstra <edolstra@gmail.com>2018-05-02T10·54+0200
committerEelco Dolstra <edolstra@gmail.com>2018-05-02T10·54+0200
commit4a2c948943e02af7829a758104044c72b49f9b64 (patch)
tree27995f22f56a3650b74ebf3d28dbed3652b1743e
parent3560654e6aaaba6c87ecc7591a2e6d107dbf8b69 (diff)
Fix bzip2 compression of files > 4 GiB
Bzip2's 'avail_in' parameter is declared as an unsigned int, so
assigning a size_t length to it led to silent truncation.

Fixes #2111.
-rw-r--r--src/libutil/compression.cc17
1 files changed, 14 insertions, 3 deletions
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 81cb5e98c763..e1782f8c4bd9 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -369,7 +369,20 @@ struct BzipSink : CompressionSink
 
     void write(const unsigned char * data, size_t len) override
     {
+        /* Bzip2's 'avail_in' parameter is an unsigned int, so we need
+           to split the input into chunks of at most 4 GiB. */
+        while (len) {
+            auto n = std::min((size_t) std::numeric_limits<decltype(strm.avail_in)>::max(), len);
+            writeInternal(data, n);
+            data += n;
+            len -= n;
+        }
+    }
+
+    void writeInternal(const unsigned char * data, size_t len)
+    {
         assert(!finished);
+        assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
 
         strm.next_in = (char *) data;
         strm.avail_in = len;
@@ -475,8 +488,6 @@ struct BrotliSink : CompressionSink
 
     void write(const unsigned char * data, size_t len) override
     {
-        assert(!finished);
-
         // Don't feed brotli too much at once
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
         while (len) {
@@ -486,7 +497,7 @@ struct BrotliSink : CompressionSink
           len -= n;
         }
     }
-  private:
+
     void writeInternal(const unsigned char * data, size_t len)
     {
         assert(!finished);