From 08ec757726e5ef47e71bf16ed0b252b288bcf0f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Mar 2018 23:12:31 +0200 Subject: Make LocalBinaryCacheStore::narFromPath() run in constant memory This reduces memory consumption of nix copy --from file://... --to ~/my-nix /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79 from 514 MiB to 18 MiB for an uncompressed binary cache, and from 192 MiB to 53 MiB for a bzipped binary cache. It may also be faster because fetching can happen concurrently with decompression/writing. Continuation of 48662d151bdf4a38670897beacea9d1bd750376a. Issue https://github.com/NixOS/nix/issues/1681. --- src/libutil/util.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/libutil/util.cc') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 15962236ec65..22753f4b7eca 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -311,6 +311,14 @@ string readFile(const Path & path, bool drain) } +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + void writeFile(const Path & path, const string & s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); @@ -593,7 +601,7 @@ void drainFD(int fd, Sink & sink, bool block) throw SysError("making file descriptor non-blocking"); } - std::vector buf(4096); + std::vector buf(64 * 1024); while (1) { checkInterrupt(); ssize_t rd = read(fd, buf.data(), buf.size()); -- cgit 1.4.1