refactor(tvix/nix-compat): support non-unicode Derivations r/6449

Derivations can have non-unicode strings in their env values, so the
ATerm representations are not necessarily String anymore, but Vec<u8>.

Change-Id: Ic23839471eb7f68d9c3c30667c878830946b6607
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8990
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Autosubmit: flokli <flokli@flokli.de>

2 files changed, 18 insertions, 4 deletions

diff --git a/tvix/cli/src/derivation.rs b/tvix/cli/src/derivation.rs
index fa246cc74f..f8afe19461 100644
--- a/tvix/cli/src/derivation.rs
+++ b/tvix/cli/src/derivation.rs
@@ -283,7 +283,7 @@ mod derivation_builtins {
             // Most of these are also added to the builder's environment in "raw" form.
             if drv
                 .environment
-                .insert(name.as_str().to_string(), val_str)
+                .insert(name.as_str().to_string(), val_str.into())
                 .is_some()
             {
                 return Err(Error::DuplicateEnvVar(name.as_str().to_string()).into());
@@ -312,7 +312,7 @@ mod derivation_builtins {
             } else {
                 let mut refscan = state.reference_scanner();
                 drv.arguments.iter().for_each(|s| refscan.scan_str(s));
-                drv.environment.values().for_each(|s| refscan.scan_str(s));
+                drv.environment.values().for_each(|s| refscan.scan_bytes(s));
                 refscan.scan_str(&drv.builder);
                 refscan.finalise()
             }
@@ -324,7 +324,7 @@ mod derivation_builtins {
         for output in drv.outputs.keys() {
             if drv
                 .environment
-                .insert(output.to_string(), String::new())
+                .insert(output.to_string(), String::new().into())
                 .is_some()
             {
                 emit_warning_kind(&co, WarningKind::ShadowedOutput(output.to_string())).await;
diff --git a/tvix/cli/src/refscan.rs b/tvix/cli/src/refscan.rs
index b391781f92..25ada4c865 100644
--- a/tvix/cli/src/refscan.rs
+++ b/tvix/cli/src/refscan.rs
@@ -37,7 +37,21 @@ impl<P: Clone + Ord + AsRef<[u8]>> ReferenceScanner<P> {
         }
     }
 
-    /// Scan the given string for all non-overlapping matches and collect them
+    /// If the given &[u8] is also a valid UTF-8 string, scan for all non-
+    /// overlapping matches and collect them in the scanner.
+    /// TODO: ideally, wu-manber would just work with &[u8] directly.
+    pub fn scan_bytes(&mut self, haystack: &[u8]) {
+        if haystack.len() < STORE_PATH_LEN {
+            return;
+        }
+
+        match std::str::from_utf8(haystack) {
+            Ok(s) => self.scan_str(s),
+            Err(_) => {}
+        }
+    }
+
+    /// Scan the given str for all non-overlapping matches and collect them
     /// in the scanner.
     pub fn scan_str(&mut self, haystack: &str) {
         if haystack.len() < STORE_PATH_LEN {