From 63116d8c21afdc50725ae93d13839fe1915b06b7 Mon Sep 17 00:00:00 2001 From: Connor Brewster Date: Fri, 22 Mar 2024 18:52:21 -0500 Subject: fix(tvix): Avoid buffering file into memory in builtins.hashFile Right now `builtins.hashFile` always reads the entire file into memory before hashing, which is not ideal for large files. This replaces `read_to_string` with `open_file` which allows calculating the hash of the file without buffering it entirely into memory. Other callers can continue to buffer into memory if they choose, but they still use the `open_file` VM request and then call `read_to_string` or `read_to_end` on the `std::io::Reader`. Fixes b/380 Change-Id: Ifa1c8324bcee8f751604b0b449feab875c632fda Reviewed-on: https://cl.tvl.fyi/c/depot/+/11236 Reviewed-by: flokli Tested-by: BuildkiteCI --- tvix/eval/src/compiler/import.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'tvix/eval/src/compiler') diff --git a/tvix/eval/src/compiler/import.rs b/tvix/eval/src/compiler/import.rs index c56909e958fb..9036eec81731 100644 --- a/tvix/eval/src/compiler/import.rs +++ b/tvix/eval/src/compiler/import.rs @@ -6,7 +6,6 @@ //! instance, or observers). use super::GlobalsMap; -use bstr::ByteSlice; use genawaiter::rc::Gen; use std::rc::Weak; @@ -39,9 +38,11 @@ async fn import_impl( return Ok(cached); } - // TODO(tazjin): make this return a string directly instead - let contents: Value = generators::request_read_to_string(&co, path.clone()).await; - let contents = contents.to_str()?.to_str()?.to_owned(); + let mut reader = generators::request_open_file(&co, path.clone()).await; + // We read to a String instead of a Vec because rnix only supports + // string source files. + let mut contents = String::new(); + reader.read_to_string(&mut contents)?; let parsed = rnix::ast::Root::parse(&contents); let errors = parsed.errors(); -- cgit 1.4.1