feat(tvix/eval): implement builtins.fromTOML r/5754

This allows parsing TOML from Tvix. We can enable the eval-okay-fromTOML
testcase from nix_tests. It uses the `toml` crate, and the serde
integration it brings with it.

Change-Id: Ic6f95aacf2aeb890116629b409752deac49dd655
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7920
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

8 files changed, 273 insertions, 3 deletions

diff --git a/corp/tvixbolt/Cargo.lock b/corp/tvixbolt/Cargo.lock
index 033335e84a..4ac5887a82 100644
--- a/corp/tvixbolt/Cargo.lock
+++ b/corp/tvixbolt/Cargo.lock
@@ -428,6 +428,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "nom8"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
 name = "once_cell"
 version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -629,6 +638,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "serde_spanned"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -720,6 +738,40 @@ dependencies = [
 ]
 
 [[package]]
+name = "toml"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "729bfd096e40da9c001f778f5cdecbd2957929a24e10e5883d9392220a751581"
+dependencies = [
+ "indexmap",
+ "nom8",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+]
+
+[[package]]
 name = "tvix-eval"
 version = "0.1.0"
 dependencies = [
@@ -737,6 +789,7 @@ dependencies = [
  "serde_json",
  "smol_str",
  "tabwriter",
+ "toml",
  "tvix-eval-builtin-macros",
  "xml-rs",
 ]
diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock
index 4f4ebac0d3..c0b82eff86 100644
--- a/tvix/Cargo.lock
+++ b/tvix/Cargo.lock
@@ -1236,6 +1236,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "nom8"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1894,6 +1903,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "serde_spanned"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "sha2"
 version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2242,6 +2260,40 @@ dependencies = [
 ]
 
 [[package]]
+name = "toml"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "729bfd096e40da9c001f778f5cdecbd2957929a24e10e5883d9392220a751581"
+dependencies = [
+ "indexmap",
+ "nom8",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+]
+
+[[package]]
 name = "tonic"
 version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2495,6 +2547,7 @@ dependencies = [
  "tempdir",
  "test-generator",
  "test-strategy",
+ "toml",
  "tvix-eval-builtin-macros",
  "xml-rs",
 ]
diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix
index d435d5870f..27cc40d2bb 100644
--- a/tvix/Cargo.nix
+++ b/tvix/Cargo.nix
@@ -3593,6 +3593,25 @@ rec {
         features = { };
         resolvedDefaultFeatures = [ "integration_tests" ];
       };
+      "nom8" = rec {
+        crateName = "nom8";
+        version = "0.2.0";
+        edition = "2018";
+        sha256 = "1y6jzabxyrl05vxnh63r66ac2fh0symg5fnynxm4ii3zkif580df";
+        dependencies = [
+          {
+            name = "memchr";
+            packageId = "memchr";
+            usesDefaultFeatures = false;
+          }
+        ];
+        features = {
+          "default" = [ "std" ];
+          "std" = [ "alloc" "memchr/std" ];
+          "unstable-doc" = [ "alloc" "std" ];
+        };
+        resolvedDefaultFeatures = [ "alloc" "default" "std" ];
+      };
       "nu-ansi-term" = rec {
         crateName = "nu-ansi-term";
         version = "0.46.0";
@@ -5478,6 +5497,23 @@ rec {
         };
         resolvedDefaultFeatures = [ "default" "std" ];
       };
+      "serde_spanned" = rec {
+        crateName = "serde_spanned";
+        version = "0.6.0";
+        edition = "2021";
+        sha256 = "1s1qvxk2h1i3p3b47p1vc7cr3f694zfvlajwjaw42f7mrqhyjs1c";
+        dependencies = [
+          {
+            name = "serde";
+            packageId = "serde";
+            optional = true;
+          }
+        ];
+        features = {
+          "serde" = [ "dep:serde" ];
+        };
+        resolvedDefaultFeatures = [ "serde" ];
+      };
       "sha2" = rec {
         crateName = "sha2";
         version = "0.10.6";
@@ -6472,6 +6508,107 @@ rec {
         };
         resolvedDefaultFeatures = [ "codec" "default" "tracing" ];
       };
+      "toml" = rec {
+        crateName = "toml";
+        version = "0.6.0";
+        edition = "2021";
+        sha256 = "05zjz69wjymp9yrgccg5vhvxpf855rgn23vl1yvri4nwwj8difag";
+        authors = [
+          "Alex Crichton <alex@alexcrichton.com>"
+        ];
+        dependencies = [
+          {
+            name = "serde";
+            packageId = "serde";
+          }
+          {
+            name = "serde_spanned";
+            packageId = "serde_spanned";
+            features = [ "serde" ];
+          }
+          {
+            name = "toml_datetime";
+            packageId = "toml_datetime";
+            features = [ "serde" ];
+          }
+          {
+            name = "toml_edit";
+            packageId = "toml_edit";
+            optional = true;
+            features = [ "serde" ];
+          }
+        ];
+        features = {
+          "default" = [ "parse" "display" ];
+          "display" = [ "dep:toml_edit" ];
+          "indexmap" = [ "dep:indexmap" ];
+          "parse" = [ "dep:toml_edit" ];
+          "preserve_order" = [ "indexmap" ];
+        };
+        resolvedDefaultFeatures = [ "default" "display" "parse" ];
+      };
+      "toml_datetime" = rec {
+        crateName = "toml_datetime";
+        version = "0.5.1";
+        edition = "2021";
+        sha256 = "1xcw3kyklh3s2gxp65ma26rgkl7505la4xx1r55kfgcfmikz8ls5";
+        authors = [
+          "Alex Crichton <alex@alexcrichton.com>"
+        ];
+        dependencies = [
+          {
+            name = "serde";
+            packageId = "serde";
+            optional = true;
+          }
+        ];
+        features = {
+          "serde" = [ "dep:serde" ];
+        };
+        resolvedDefaultFeatures = [ "serde" ];
+      };
+      "toml_edit" = rec {
+        crateName = "toml_edit";
+        version = "0.18.0";
+        edition = "2021";
+        sha256 = "108mfl5254lk7n4fa42fl8lpk5fjrgg5r3vp3w09rnj0dq4zv6vj";
+        authors = [
+          "Andronik Ordian <write@reusable.software>"
+          "Ed Page <eopage@gmail.com>"
+        ];
+        dependencies = [
+          {
+            name = "indexmap";
+            packageId = "indexmap";
+          }
+          {
+            name = "nom8";
+            packageId = "nom8";
+          }
+          {
+            name = "serde";
+            packageId = "serde";
+            optional = true;
+            features = [ "derive" ];
+          }
+          {
+            name = "serde_spanned";
+            packageId = "serde_spanned";
+            optional = true;
+            features = [ "serde" ];
+          }
+          {
+            name = "toml_datetime";
+            packageId = "toml_datetime";
+          }
+        ];
+        features = {
+          "easy" = [ "serde" ];
+          "perf" = [ "dep:kstring" ];
+          "serde" = [ "dep:serde" "toml_datetime/serde" "dep:serde_spanned" ];
+        };
+        resolvedDefaultFeatures = [ "default" "serde" ];
+      };
       "tonic" = rec {
         crateName = "tonic";
         version = "0.8.3";
@@ -7471,6 +7608,10 @@ rec {
             optional = true;
           }
           {
+            name = "toml";
+            packageId = "toml";
+          }
+          {
             name = "tvix-eval-builtin-macros";
             packageId = "tvix-eval-builtin-macros";
             rename = "builtin-macros";
diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml
index d47ad8c397..cf46d4bbd4 100644
--- a/tvix/eval/Cargo.toml
+++ b/tvix/eval/Cargo.toml
@@ -27,6 +27,7 @@ serde_json = "1.0"
 smol_str = "0.1"
 tabwriter = "1.2"
 test-strategy = { version = "0.2.1", optional = true }
+toml = "0.6.0"
 xml-rs = "0.8.4"
 
 [dev-dependencies]
@@ -42,7 +43,7 @@ git = "https://github.com/JamesGuthrie/test-generator.git"
 rev = "82e799979980962aec1aa324ec6e0e4cad781f41"
 
 [features]
-default = [ "impure", "arbitrary", "nix_tests", "backtrace_overflow" ]
+default = ["impure", "arbitrary", "nix_tests", "backtrace_overflow"]
 
 # Enables running the Nix language test suite from the original C++
 # Nix implementation (at version 2.3) against Tvix.
@@ -52,10 +53,10 @@ nix_tests = []
 impure = []
 
 # Enables Arbitrary impls for internal types (required to run tests)
-arbitrary = [ "proptest", "test-strategy", "imbl/proptest" ]
+arbitrary = ["proptest", "test-strategy", "imbl/proptest"]
 
 # For debugging use only; not appropriate for production use.
-backtrace_overflow = [ "backtrace-on-stack-overflow" ]
+backtrace_overflow = ["backtrace-on-stack-overflow"]
 
 [[bench]]
 name = "eval"
diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs
index b1ea408a8c..4f3c93ec14 100644
--- a/tvix/eval/src/builtins/mod.rs
+++ b/tvix/eval/src/builtins/mod.rs
@@ -355,6 +355,13 @@ mod pure_builtins {
         Ok(json_str.into())
     }
 
+    #[builtin("fromTOML")]
+    fn builtin_from_toml(_: &mut VM, toml: Value) -> Result<Value, ErrorKind> {
+        let toml_str = toml.to_str()?;
+
+        toml::from_str(&toml_str).map_err(|err| err.into())
+    }
+
     #[builtin("genericClosure")]
     fn builtin_generic_closure(vm: &mut VM, input: Value) -> Result<Value, ErrorKind> {
         let attrs = input.to_attrs()?;
diff --git a/tvix/eval/src/errors.rs b/tvix/eval/src/errors.rs
index ccc2d89764..8c0d612420 100644
--- a/tvix/eval/src/errors.rs
+++ b/tvix/eval/src/errors.rs
@@ -134,6 +134,9 @@ pub enum ErrorKind {
     /// Errors converting JSON to a value
     FromJsonError(String),
 
+    /// Errors converting TOML to a value
+    FromTomlError(String),
+
     /// An unexpected argument was supplied to a function that takes formal parameters
     UnexpectedArgument {
         arg: NixString,
@@ -246,6 +249,12 @@ impl From<serde_json::Error> for ErrorKind {
     }
 }
 
+impl From<toml::de::Error> for ErrorKind {
+    fn from(err: toml::de::Error) -> Self {
+        Self::FromTomlError(format!("error in TOML serialization: {err}"))
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct Error {
     kind: ErrorKind,
@@ -434,6 +443,10 @@ to a missing value in the attribute set(s) included via `with`."#,
                 write!(f, "Error converting JSON to a Nix value: {msg}")
             }
 
+            ErrorKind::FromTomlError(msg) => {
+                write!(f, "Error converting TOML to a Nix value: {msg}")
+            }
+
             ErrorKind::UnexpectedArgument { arg, .. } => {
                 write!(
                     f,
@@ -749,6 +762,7 @@ impl Error {
             | ErrorKind::ImportCompilerError { .. }
             | ErrorKind::IO { .. }
             | ErrorKind::FromJsonError(_)
+            | ErrorKind::FromTomlError(_)
             | ErrorKind::Xml(_)
             | ErrorKind::TvixError(_)
             | ErrorKind::TvixBug { .. }
@@ -795,6 +809,7 @@ impl Error {
             ErrorKind::RelativePathResolution(_) => "E032",
             ErrorKind::DivisionByZero => "E033",
             ErrorKind::Xml(_) => "E034",
+            ErrorKind::FromTomlError(_) => "E035",
 
             // Special error code for errors from other Tvix
             // components. We may want to introduce a code namespacing
diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.exp b/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.exp
index d0dd3af2c8..d0dd3af2c8 100644
--- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.exp
+++ b/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.exp
diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.nix b/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.nix
index 9639326899..9639326899 100644
--- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.nix
+++ b/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.nix