feat(tvix/glue): implement builtins.toFile r/8313

We currently only had a dummy implementation that didn't actually
persist the files in the store(s).

Copy the contents to the BlobService, and do the output path calculation
as part of the upload.
Use the plain context elements to construct the references.

Change-Id: Ibdaf7a645ddc31e847faa4b87a79f2f95116a7ab
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11879
Reviewed-by: Ilan Joselevich <personal@ilanjoselevich.com>
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI

4 files changed, 94 insertions, 9 deletions

diff --git a/tvix/glue/src/builtins/derivation.rs b/tvix/glue/src/builtins/derivation.rs
index f266141cb6e7..8400dcea13fe 100644
--- a/tvix/glue/src/builtins/derivation.rs
+++ b/tvix/glue/src/builtins/derivation.rs
@@ -168,15 +168,23 @@ fn handle_fixed_output(
 #[builtins(state = "Rc<TvixStoreIO>")]
 pub(crate) mod derivation_builtins {
     use std::collections::BTreeMap;
+    use std::io::Cursor;
 
     use crate::builtins::utils::{select_string, strong_importing_coerce_to_string};
     use crate::fetchurl::fetchurl_derivation_to_fetch;
 
     use super::*;
     use bstr::ByteSlice;
-    use nix_compat::store_path::hash_placeholder;
+    use md5::Digest;
+    use nix_compat::nixhash::CAHash;
+    use nix_compat::store_path::{build_ca_path, hash_placeholder};
+    use sha2::Sha256;
+    use tvix_castore::proto as castorepb;
+    use tvix_castore::proto::node::Node;
+    use tvix_castore::proto::FileNode;
     use tvix_eval::generators::Gen;
     use tvix_eval::{NixContext, NixContextElement, NixString};
+    use tvix_store::proto::{NarInfo, PathInfo};
 
     #[builtin("placeholder")]
     async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
@@ -525,7 +533,12 @@ pub(crate) mod derivation_builtins {
     }
 
     #[builtin("toFile")]
-    async fn builtin_to_file(co: GenCo, name: Value, content: Value) -> Result<Value, ErrorKind> {
+    async fn builtin_to_file(
+        state: Rc<TvixStoreIO>,
+        co: GenCo,
+        name: Value,
+        content: Value,
+    ) -> Result<Value, ErrorKind> {
         if name.is_catchable() {
             return Ok(name);
         }
@@ -545,20 +558,77 @@ pub(crate) mod derivation_builtins {
             return Err(ErrorKind::UnexpectedContext);
         }
 
-        let path =
-            nix_compat::store_path::build_text_path(name.to_str()?, &content, content.iter_plain())
+        let store_path = state.tokio_handle.block_on(async {
+            // upload contents to the blobservice and create a root node
+            let mut blob_writer = state.blob_service.open_write().await;
+
+            let mut r = Cursor::new(&content);
+
+            let blob_size = tokio::io::copy(&mut r, &mut blob_writer).await?;
+            let blob_digest = blob_writer.close().await?;
+            let ca_hash = CAHash::Text(Sha256::digest(&content).into());
+
+            let store_path = build_ca_path(name.to_str()?, &ca_hash, content.iter_plain(), false)
                 .map_err(|_e| {
                     nix_compat::derivation::DerivationError::InvalidOutputName(
                         name.to_str_lossy().into_owned(),
                     )
                 })
-                .map_err(DerivationError::InvalidDerivation)?
-                .to_absolute_path();
+                .map_err(DerivationError::InvalidDerivation)?;
+
+            let root_node = Node::File(FileNode {
+                name: store_path.to_string().into(),
+                digest: blob_digest.into(),
+                size: blob_size,
+                executable: false,
+            });
+
+            // calculate the nar hash
+            let (nar_size, nar_sha256) = state
+                .nar_calculation_service
+                .calculate_nar(&root_node)
+                .await
+                .map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
+
+            // assemble references from plain context.
+            let reference_paths: Vec<StorePathRef> = content
+                .iter_plain()
+                .map(|elem| StorePathRef::from_absolute_path(elem.as_bytes()))
+                .collect::<Result<_, _>>()
+                .map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
+
+            // persist via pathinfo service.
+            state
+                .path_info_service
+                .put(PathInfo {
+                    node: Some(castorepb::Node {
+                        node: Some(root_node),
+                    }),
+                    references: reference_paths
+                        .iter()
+                        .map(|x| bytes::Bytes::copy_from_slice(x.digest()))
+                        .collect(),
+                    narinfo: Some(NarInfo {
+                        nar_size,
+                        nar_sha256: nar_sha256.to_vec().into(),
+                        signatures: vec![],
+                        reference_names: reference_paths
+                            .into_iter()
+                            .map(|x| x.to_string())
+                            .collect(),
+                        deriver: None,
+                        ca: Some(ca_hash.into()),
+                    }),
+                })
+                .await
+                .map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
 
-        let context: NixContext = NixContextElement::Plain(path.clone()).into();
+            Ok::<_, ErrorKind>(store_path)
+        })?;
 
-        // TODO: actually persist the file in the store at that path ...
+        let abs_path = store_path.to_absolute_path();
+        let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
 
-        Ok(Value::from(NixString::new_context_from(context, path)))
+        Ok(Value::from(NixString::new_context_from(context, abs_path)))
     }
 }
diff --git a/tvix/glue/src/tests/tvix_tests/eval-fail-tofile-wrongctxtype.nix b/tvix/glue/src/tests/tvix_tests/eval-fail-tofile-wrongctxtype.nix
new file mode 100644
index 000000000000..60c94818ed25
--- /dev/null
+++ b/tvix/glue/src/tests/tvix_tests/eval-fail-tofile-wrongctxtype.nix
@@ -0,0 +1,3 @@
+# in 'toFile': the file 'foo' cannot refer to derivation outputs, at (string):1:1
+builtins.toFile "foo" "${(builtins.derivation {name = "foo"; builder = ":"; system = ":";})}"
+
diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-tofile.exp b/tvix/glue/src/tests/tvix_tests/eval-okay-tofile.exp
new file mode 100644
index 000000000000..c8e5b8fab5d9
--- /dev/null
+++ b/tvix/glue/src/tests/tvix_tests/eval-okay-tofile.exp
@@ -0,0 +1 @@
+[ "/nix/store/vxjiwkjkn7x4079qvh1jkl5pn05j2aw0-foo" "/nix/store/i7liwr52956m86kxp7dgbcwsk56r27v6-foo" "/nix/store/yw8k7ixk1zvb113p4y0bl3ahjxd5h9sr-foo" { "/nix/store/yw8k7ixk1zvb113p4y0bl3ahjxd5h9sr-foo" = { path = true; }; } ]
diff --git a/tvix/glue/src/tests/tvix_tests/eval-okay-tofile.nix b/tvix/glue/src/tests/tvix_tests/eval-okay-tofile.nix
new file mode 100644
index 000000000000..141bbc38ec11
--- /dev/null
+++ b/tvix/glue/src/tests/tvix_tests/eval-okay-tofile.nix
@@ -0,0 +1,11 @@
+let
+  noContext = (builtins.toFile "foo" "bar");
+  someContext = (builtins.toFile "foo" "bar${noContext}");
+  moreContext = (builtins.toFile "foo" "bar${someContext}");
+in
+[
+  noContext
+  someContext
+  moreContext
+  (builtins.getContext moreContext)
+]