nix-prefetch-url: Download file in constant memory

Before:

  $ command time nix-prefetch-url https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.17.6.tar.xz
  1.19user 1.02system 0:41.96elapsed 5%CPU (0avgtext+0avgdata 182720maxresident)k

After:

  1.38user 1.05system 0:39.73elapsed 6%CPU (0avgtext+0avgdata 16204maxresident)k

Note however that addToStore() can still take a lot of memory
(e.g. RemoteStore::addToStore() is constant space, but
LocalStore::addToStore() isn't; that's fixed by
https://github.com/edolstra/nix/commit/c94b4fc7ee0c7b322a5f3c7ee784063b47a11d98
though).

Fixes #1400.

1 files changed, 17 insertions, 7 deletions

diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 50b2c2803ec9..a3b025723cf1 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -9,6 +9,10 @@
 
 #include <iostream>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
 using namespace nix;
 
 
@@ -160,14 +164,20 @@ int main(int argc, char * * argv)
 
             auto actualUri = resolveMirrorUri(*state, uri);
 
-            /* Download the file. */
-            DownloadRequest req(actualUri);
-            req.decompress = false;
-            auto result = getDownloader()->download(req);
-
             AutoDelete tmpDir(createTempDir(), true);
             Path tmpFile = (Path) tmpDir + "/tmp";
-            writeFile(tmpFile, *result.data);
+
+            /* Download the file. */
+            {
+                AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+                if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
+
+                FdSink sink(fd.get());
+
+                DownloadRequest req(actualUri);
+                req.decompress = false;
+                getDownloader()->download(std::move(req), sink);
+            }
 
             /* Optionally unpack the file. */
             if (unpack) {
@@ -191,7 +201,7 @@ int main(int argc, char * * argv)
 
             /* FIXME: inefficient; addToStore() will also hash
                this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data);
+            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
             if (expectedHash != Hash(ht) && expectedHash != hash)
                 throw Error(format("hash mismatch for '%1%'") % uri);