From 771200df7c311fc8b87a0a65a02e22a11d80cd66 Mon Sep 17 00:00:00 2001
From: Florian Klink <flokli@flokli.de>
Date: Wed, 21 Feb 2024 20:39:53 +0700
Subject: fix(tvix/eval): allow reading non-UTF8 files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With our values using bstr now, we're not restricted to only reading
files that contain valid UTF-8.

Update our `read_to_string` function to `read_to_end`
(named like `std::io::Read::read_to_end`), and have it return a Vec<u8>.

Change-Id: I87f0291dc855a132689576559c891d66c30ddf2b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11003
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: Pádraic Ó Mhuiris <patrick.morris.310@gmail.com>
Reviewed-by: flokli <flokli@flokli.de>
---
 tvix/eval/src/io.rs            | 10 +++++-----
 tvix/eval/src/vm/generators.rs |  2 +-
 tvix/glue/src/tvix_io.rs       |  6 +++---
 tvix/glue/src/tvix_store_io.rs | 10 +++++-----
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tvix/eval/src/io.rs b/tvix/eval/src/io.rs
index ccbc7dfdbcb0..6589c0dc5cae 100644
--- a/tvix/eval/src/io.rs
+++ b/tvix/eval/src/io.rs
@@ -48,13 +48,13 @@ pub trait EvalIO {
     /// * `builtins.pathExists :: path -> bool`
     fn path_exists(&self, path: &Path) -> io::Result<bool>;
 
-    /// Read the file at the specified path to a string.
+    /// Read the file at the specified path to a Vec<u8>.
     ///
     /// This is used for the following language evaluation cases:
     ///
     /// * `builtins.readFile :: path -> string`
     /// * `builtins.import :: path -> any`
-    fn read_to_string(&self, path: &Path) -> io::Result<String>;
+    fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>>;
 
     /// Read the directory at the specified path and return the names
     /// of its entries associated with their [`FileType`].
@@ -99,8 +99,8 @@ impl EvalIO for StdIO {
         path.try_exists()
     }
 
-    fn read_to_string(&self, path: &Path) -> io::Result<String> {
-        std::fs::read_to_string(path)
+    fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>> {
+        std::fs::read(path)
     }
 
     fn read_dir(&self, path: &Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> {
@@ -145,7 +145,7 @@ impl EvalIO for DummyIO {
         ))
     }
 
-    fn read_to_string(&self, _: &Path) -> io::Result<String> {
+    fn read_to_end(&self, _: &Path) -> io::Result<Vec<u8>> {
         Err(io::Error::new(
             io::ErrorKind::Unsupported,
             "I/O methods are not implemented in DummyIO",
diff --git a/tvix/eval/src/vm/generators.rs b/tvix/eval/src/vm/generators.rs
index fba4e7cd90d2..2a2710dc34af 100644
--- a/tvix/eval/src/vm/generators.rs
+++ b/tvix/eval/src/vm/generators.rs
@@ -429,7 +429,7 @@ where
                             let content = self
                                 .io_handle
                                 .as_ref()
-                                .read_to_string(&path)
+                                .read_to_end(&path)
                                 .map_err(|e| ErrorKind::IO {
                                     path: Some(path),
                                     error: e.into(),
diff --git a/tvix/glue/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs
index 9fb9fbc37590..95146df7287e 100644
--- a/tvix/glue/src/tvix_io.rs
+++ b/tvix/glue/src/tvix_io.rs
@@ -44,7 +44,7 @@ where
         self.actual.as_ref().path_exists(path)
     }
 
-    fn read_to_string(&self, path: &Path) -> io::Result<String> {
+    fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>> {
         // Bundled version of corepkgs/fetchurl.nix. The counterpart
         // of this happens in [crate::configure_nix_path], where the `nix_path`
         // of the evaluation has `nix=/__corepkgs__` added to it.
@@ -55,10 +55,10 @@ where
         // TODO: this comparison is bad and allocates, we should use
         // the sane path library.
         if path.starts_with("/__corepkgs__/fetchurl.nix") {
-            return Ok(include_str!("fetchurl.nix").to_string());
+            return Ok(include_bytes!("fetchurl.nix").to_vec());
         }
 
-        self.actual.as_ref().read_to_string(path)
+        self.actual.as_ref().read_to_end(path)
     }
 
     fn read_dir(&self, path: &Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> {
diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs
index 333b04b170c7..c09f0098e43e 100644
--- a/tvix/glue/src/tvix_store_io.rs
+++ b/tvix/glue/src/tvix_store_io.rs
@@ -371,7 +371,7 @@ impl EvalIO for TvixStoreIO {
     }
 
     #[instrument(skip(self), err)]
-    fn read_to_string(&self, path: &Path) -> io::Result<String> {
+    fn read_to_end(&self, path: &Path) -> io::Result<Vec<u8>> {
         if let Ok((store_path, sub_path)) =
             StorePath::from_absolute_path_full(&path.to_string_lossy())
         {
@@ -416,9 +416,9 @@ impl EvalIO for TvixStoreIO {
                                 }
                             };
 
-                            let mut buf = String::new();
+                            let mut buf = Vec::new();
 
-                            reader.read_to_string(&mut buf).await?;
+                            reader.read_to_end(&mut buf).await?;
                             Ok(buf)
                         })
                     }
@@ -430,11 +430,11 @@ impl EvalIO for TvixStoreIO {
             } else {
                 // As tvix-store doesn't manage /nix/store on the filesystem,
                 // we still need to also ask self.std_io here.
-                self.std_io.read_to_string(path)
+                self.std_io.read_to_end(path)
             }
         } else {
             // The store path is no store path, so do regular StdIO.
-            self.std_io.read_to_string(path)
+            self.std_io.read_to_end(path)
         }
     }
 
-- 
cgit 1.4.1