From 63116d8c21afdc50725ae93d13839fe1915b06b7 Mon Sep 17 00:00:00 2001 From: Connor Brewster Date: Fri, 22 Mar 2024 18:52:21 -0500 Subject: fix(tvix): Avoid buffering file into memory in builtins.hashFile Right now `builtins.hashFile` always reads the entire file into memory before hashing, which is not ideal for large files. This replaces `read_to_string` with `open_file` which allows calculating the hash of the file without buffering it entirely into memory. Other callers can continue to buffer into memory if they choose, but they still use the `open_file` VM request and then call `read_to_string` or `read_to_end` on the `std::io::Reader`. Fixes b/380 Change-Id: Ifa1c8324bcee8f751604b0b449feab875c632fda Reviewed-on: https://cl.tvl.fyi/c/depot/+/11236 Reviewed-by: flokli Tested-by: BuildkiteCI --- tvix/glue/src/tvix_io.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'tvix/glue/src/tvix_io.rs') diff --git a/tvix/glue/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs index 95146df7287e..0e5f23b99093 100644 --- a/tvix/glue/src/tvix_io.rs +++ b/tvix/glue/src/tvix_io.rs @@ -8,7 +8,7 @@ //! otherwise fundamental features like nixpkgs bootstrapping and hash //! calculation will not work. -use std::io; +use std::io::{self, Cursor}; use std::path::{Path, PathBuf}; use tvix_eval::{EvalIO, FileType}; @@ -44,7 +44,7 @@ where self.actual.as_ref().path_exists(path) } - fn read_to_end(&self, path: &Path) -> io::Result> { + fn open(&self, path: &Path) -> io::Result> { // Bundled version of corepkgs/fetchurl.nix. The counterpart // of this happens in [crate::configure_nix_path], where the `nix_path` // of the evaluation has `nix=/__corepkgs__` added to it. @@ -52,13 +52,12 @@ where // This workaround is similar to what cppnix does for passing // the path through. // - // TODO: this comparison is bad and allocates, we should use - // the sane path library. + // TODO: this comparison is bad we should use the sane path library. if path.starts_with("/__corepkgs__/fetchurl.nix") { - return Ok(include_bytes!("fetchurl.nix").to_vec()); + return Ok(Box::new(Cursor::new(include_bytes!("fetchurl.nix")))); } - self.actual.as_ref().read_to_end(path) + self.actual.as_ref().open(path) } fn read_dir(&self, path: &Path) -> io::Result> { -- cgit 1.4.1