Make LocalBinaryCacheStore::narFromPath() run in constant memory

This reduces memory consumption of

  nix copy --from file://... --to ~/my-nix /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79

from 514 MiB to 18 MiB for an uncompressed binary cache, and from 192
MiB to 53 MiB for a bzipped binary cache. It may also be faster
because fetching can happen concurrently with decompression/writing.

Continuation of 48662d151bdf4a38670897beacea9d1bd750376a.

Issue https://github.com/NixOS/nix/issues/1681.

2 files changed, 10 insertions, 1 deletions

diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 15962236ec65..22753f4b7eca 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -311,6 +311,14 @@ string readFile(const Path & path, bool drain)
 }
 
 
+void readFile(const Path & path, Sink & sink)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd) throw SysError("opening file '%s'", path);
+    drainFD(fd.get(), sink);
+}
+
+
 void writeFile(const Path & path, const string & s, mode_t mode)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
@@ -593,7 +601,7 @@ void drainFD(int fd, Sink & sink, bool block)
             throw SysError("making file descriptor non-blocking");
     }
 
-    std::vector<unsigned char> buf(4096);
+    std::vector<unsigned char> buf(64 * 1024);
     while (1) {
         checkInterrupt();
         ssize_t rd = read(fd, buf.data(), buf.size());
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 215c7cecafef..af97b76d90bc 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -98,6 +98,7 @@ unsigned char getFileType(const Path & path);
 /* Read the contents of a file into a string. */
 string readFile(int fd);
 string readFile(const Path & path, bool drain = false);
+void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
 void writeFile(const Path & path, const string & s, mode_t mode = 0666);