feat(tvix/eval): allow extending builtins outside of tvix_eval r/6344

The change allows applications that use tvix_serde for parsing
nix-based configuration to extend the language with domain-specific
set of features.

Change-Id: Ia86612308a167c456ecf03e93fe0fbae55b876a6
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8848
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

9 files changed, 55 insertions, 9 deletions

diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index 7bcf60dceca3..78758fa99822 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -2755,6 +2755,7 @@ dependencies = [
 name = "tvix-serde"
 version = "0.1.0"
 dependencies = [
+ "genawaiter",
  "serde",
  "tvix-eval",
 ]
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index df2b4495ae69..6227178d88fd 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -8169,6 +8169,13 @@ rec {
             packageId = "tvix-eval";
           }
         ];
+        devDependencies = [
+          {
+            name = "genawaiter";
+            packageId = "genawaiter";
+            usesDefaultFeatures = false;
+          }
+        ];
 
       };
       "tvix-store" = rec {
diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs
index 3613859eede3..7c980da78fbc 100644
--- a/tvix/cli/src/main.rs
+++ b/tvix/cli/src/main.rs
@@ -14,7 +14,7 @@ use clap::Parser;
 use known_paths::KnownPaths;
 use rustyline::{error::ReadlineError, Editor};
 use tvix_eval::observer::{DisassemblingObserver, TracingObserver};
-use tvix_eval::{Builtin, Value};
+use tvix_eval::Value;
 use tvix_store::blobservice::MemoryBlobService;
 use tvix_store::directoryservice::MemoryDirectoryService;
 use tvix_store::pathinfoservice::MemoryPathInfoService;
diff --git a/tvix/eval/builtin-macros/src/lib.rs b/tvix/eval/builtin-macros/src/lib.rs
index dfd0948c7d9f..e2cd51b92451 100644
--- a/tvix/eval/builtin-macros/src/lib.rs
+++ b/tvix/eval/builtin-macros/src/lib.rs
@@ -116,10 +116,8 @@ fn parse_module_args(args: TokenStream) -> Option<Type> {
 ///
 /// # Examples
 /// ```ignore
+/// # use tvix_eval;
 /// # use tvix_eval_builtin_macros::builtins;
-/// # mod value {
-/// #     pub use tvix_eval::Builtin;
-/// # }
 ///
 /// #[builtins]
 /// mod builtins {
@@ -270,7 +268,7 @@ pub fn builtins(args: TokenStream, item: TokenStream) -> TokenStream {
 
                     if arg.strict {
                         f.block = Box::new(parse_quote_spanned! {arg.span=> {
-                            let #ident: #ty = generators::request_force(&co, values.pop()
+                            let #ident: #ty = tvix_eval::generators::request_force(&co, values.pop()
                               .expect("Tvix bug: builtin called with incorrect number of arguments")).await;
 
                             #block
@@ -295,20 +293,20 @@ pub fn builtins(args: TokenStream, item: TokenStream) -> TokenStream {
                 if captures_state {
                     builtins.push(quote_spanned! { builtin_attr.span() => {
                         let inner_state = state.clone();
-                        crate::Builtin::new(
+                        tvix_eval::Builtin::new(
                             #name,
                             #docstring,
                             #arg_count,
-                            move |values| Gen::new(|co| generators::pin_generator(#fn_name(inner_state.clone(), co, values))),
+                            move |values| Gen::new(|co| tvix_eval::generators::pin_generator(#fn_name(inner_state.clone(), co, values))),
                         )
                     }});
                 } else {
                     builtins.push(quote_spanned! { builtin_attr.span() => {
-                        crate::Builtin::new(
+                        tvix_eval::Builtin::new(
                             #name,
                             #docstring,
                             #arg_count,
-                            |values| Gen::new(|co| generators::pin_generator(#fn_name(co, values))),
+                            |values| Gen::new(|co| tvix_eval::generators::pin_generator(#fn_name(co, values))),
                         )
                     }});
                 }
diff --git a/tvix/eval/src/builtins/impure.rs b/tvix/eval/src/builtins/impure.rs
index adf43e1a2ea9..cf3186ce5087 100644
--- a/tvix/eval/src/builtins/impure.rs
+++ b/tvix/eval/src/builtins/impure.rs
@@ -7,6 +7,7 @@ use std::{
 };
 
 use crate::{
+    self as tvix_eval,
     errors::ErrorKind,
     io::FileType,
     value::NixAttrs,
diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs
index b5c7931768e9..c841f9fc6a29 100644
--- a/tvix/eval/src/builtins/mod.rs
+++ b/tvix/eval/src/builtins/mod.rs
@@ -16,6 +16,7 @@ use crate::value::PointerEquality;
 use crate::vm::generators::{self, GenCo};
 use crate::warnings::WarningKind;
 use crate::{
+    self as tvix_eval,
     errors::ErrorKind,
     value::{CoercionKind, NixAttrs, NixList, NixString, SharedThunkSet, Thunk, Value},
 };
diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index f800baf05018..02227a7e9a2d 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -10,6 +10,7 @@ mod one_offs;
 mod mock_builtins {
     //! Builtins which are required by language tests, but should not
     //! actually exist in //tvix/eval.
+    use crate as tvix_eval;
     use crate::generators::GenCo;
     use crate::*;
     use genawaiter::rc::Gen;
diff --git a/tvix/serde/Cargo.toml b/tvix/serde/Cargo.toml
index 8ffc11a4eb6e..774214eab667 100644
--- a/tvix/serde/Cargo.toml
+++ b/tvix/serde/Cargo.toml
@@ -6,3 +6,6 @@ edition = "2021"
 [dependencies]
 tvix-eval = { path = "../eval" }
 serde = { version = "1.0", features = ["derive"] }
+
+[dev-dependencies]
+genawaiter = { version = "0.99.1", default_features = false }
\ No newline at end of file
diff --git a/tvix/serde/src/de_tests.rs b/tvix/serde/src/de_tests.rs
index 50a078cc0998..a77a59f2b5cd 100644
--- a/tvix/serde/src/de_tests.rs
+++ b/tvix/serde/src/de_tests.rs
@@ -1,5 +1,6 @@
 use serde::Deserialize;
 use std::collections::HashMap;
+use tvix_eval::builtin_macros::builtins;
 
 use crate::de::{from_str, from_str_with_config};
 
@@ -209,3 +210,36 @@ fn deserialize_with_config() {
 
     assert_eq!(result, "ok");
 }
+
+#[builtins]
+mod test_builtins {
+    use genawaiter::rc::Gen;
+    use tvix_eval::generators::GenCo;
+    use tvix_eval::{ErrorKind, NixString, Value};
+
+    #[builtin("prependHello")]
+    pub async fn builtin_prepend_hello(co: GenCo, x: Value) -> Result<Value, ErrorKind> {
+        match x {
+            Value::String(s) => {
+                let new_string = NixString::from(format!("hello {}", s.as_str()));
+                Ok(Value::String(new_string))
+            }
+            _ => Err(ErrorKind::TypeError {
+                expected: "string",
+                actual: "not string",
+            }),
+        }
+    }
+}
+
+#[test]
+fn deserialize_with_extra_builtin() {
+    let code = "builtins.prependHello \"world\"";
+
+    let result: String = from_str_with_config(code, |eval| {
+        eval.builtins.append(&mut test_builtins::builtins());
+    })
+    .expect("should deserialize");
+
+    assert_eq!(result, "hello world");
+}