fix(tvix): Avoid buffering file into memory in builtins.hashFile r/7882

Right now `builtins.hashFile` always reads the entire file into memory
before hashing, which is not ideal for large files. This replaces
`read_to_string` with `open_file` which allows calculating the hash of
the file without buffering it entirely into memory. Other callers can
continue to buffer into memory if they choose, but they still use the
`open_file` VM request and then call `read_to_string` or `read_to_end`
on the `std::io::Reader`.

Fixes b/380

Change-Id: Ifa1c8324bcee8f751604b0b449feab875c632fda
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11236
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI

1 files changed, 5 insertions, 4 deletions

diff --git a/tvix/eval/src/compiler/import.rs b/tvix/eval/src/compiler/import.rs
index c56909e958fb..9036eec81731 100644
--- a/tvix/eval/src/compiler/import.rs
+++ b/tvix/eval/src/compiler/import.rs
@@ -6,7 +6,6 @@
 //! instance, or observers).
 
 use super::GlobalsMap;
-use bstr::ByteSlice;
 use genawaiter::rc::Gen;
 use std::rc::Weak;
 
@@ -39,9 +38,11 @@ async fn import_impl(
         return Ok(cached);
     }
 
-    // TODO(tazjin): make this return a string directly instead
-    let contents: Value = generators::request_read_to_string(&co, path.clone()).await;
-    let contents = contents.to_str()?.to_str()?.to_owned();
+    let mut reader = generators::request_open_file(&co, path.clone()).await;
+    // We read to a String instead of a Vec<u8> because rnix only supports
+    // string source files.
+    let mut contents = String::new();
+    reader.read_to_string(&mut contents)?;
 
     let parsed = rnix::ast::Root::parse(&contents);
     let errors = parsed.errors();