about summary refs log tree commit diff
path: root/tvix/eval/src/compiler
diff options
context:
space:
mode:
authorConnor Brewster <cbrewster@hey.com>2024-03-22T23·52-0500
committerConnor Brewster <cbrewster@hey.com>2024-04-09T17·31+0000
commit63116d8c21afdc50725ae93d13839fe1915b06b7 (patch)
tree4997838251dac809c2917b35e5d32224030ba595 /tvix/eval/src/compiler
parent17849c5c0033fa1909f0403b5d5e6a5e018b7fee (diff)
fix(tvix): Avoid buffering file into memory in builtins.hashFile r/7882
Right now `builtins.hashFile` always reads the entire file into memory
before hashing, which is not ideal for large files. This replaces
`read_to_string` with `open_file` which allows calculating the hash of
the file without buffering it entirely into memory. Other callers can
continue to buffer into memory if they choose, but they still use the
`open_file` VM request and then call `read_to_string` or `read_to_end`
on the `std::io::Reader`.

Fixes b/380

Change-Id: Ifa1c8324bcee8f751604b0b449feab875c632fda
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11236
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Diffstat (limited to 'tvix/eval/src/compiler')
-rw-r--r--tvix/eval/src/compiler/import.rs9
1 files changed, 5 insertions, 4 deletions
diff --git a/tvix/eval/src/compiler/import.rs b/tvix/eval/src/compiler/import.rs
index c56909e958fb..9036eec81731 100644
--- a/tvix/eval/src/compiler/import.rs
+++ b/tvix/eval/src/compiler/import.rs
@@ -6,7 +6,6 @@
 //! instance, or observers).
 
 use super::GlobalsMap;
-use bstr::ByteSlice;
 use genawaiter::rc::Gen;
 use std::rc::Weak;
 
@@ -39,9 +38,11 @@ async fn import_impl(
         return Ok(cached);
     }
 
-    // TODO(tazjin): make this return a string directly instead
-    let contents: Value = generators::request_read_to_string(&co, path.clone()).await;
-    let contents = contents.to_str()?.to_str()?.to_owned();
+    let mut reader = generators::request_open_file(&co, path.clone()).await;
+    // We read to a String instead of a Vec<u8> because rnix only supports
+    // string source files.
+    let mut contents = String::new();
+    reader.read_to_string(&mut contents)?;
 
     let parsed = rnix::ast::Root::parse(&contents);
     let errors = parsed.errors();