feat(tvix/eval): Implement builtins.fromJSON r/5135

Using `serde_json` for parsing JSON here, plus an `impl FromJSON for
Value`. The latter is primarily to stay "dependency light" for now -
likely going with an actual serde `Deserialize` impl in the future is
going to be way better as it allows saving significantly on intermediary
allocations.

Change-Id: I152a0448ff7c87cf7ebaac927c38912b99de1c18
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6920
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>

12 files changed, 123 insertions, 13 deletions

diff --git a/corp/tvixbolt/Cargo.lock b/corp/tvixbolt/Cargo.lock
index a33fa4351d79..c402a18d4de3 100644
--- a/corp/tvixbolt/Cargo.lock
+++ b/corp/tvixbolt/Cargo.lock
@@ -478,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.85"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074"
 dependencies = [
  "itoa",
  "ryu",
@@ -582,6 +582,7 @@ dependencies = [
  "path-clean",
  "rnix",
  "rowan",
+ "serde_json",
  "smol_str",
  "tabwriter",
 ]
diff --git a/tvix/eval/Cargo.lock b/tvix/eval/Cargo.lock
index ec91d1c0520e..b9fa3d3a11da 100644
--- a/tvix/eval/Cargo.lock
+++ b/tvix/eval/Cargo.lock
@@ -997,9 +997,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.85"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074"
 dependencies = [
  "itoa 1.0.3",
  "ryu",
@@ -1211,6 +1211,7 @@ dependencies = [
  "rnix",
  "rowan",
  "rustyline",
+ "serde_json",
  "smol_str",
  "tabwriter",
  "tempdir",
diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml
index 2edeccb9d1ef..67ed42609deb 100644
--- a/tvix/eval/Cargo.toml
+++ b/tvix/eval/Cargo.toml
@@ -24,6 +24,7 @@ codemap-diagnostic = "0.1.1"
 proptest = { version = "1.0.0", default_features = false, features = ["std", "alloc", "break-dead-code", "tempfile"], optional = true }
 test-strategy = { version = "0.2.1", optional = true }
 clap = { version = "3.2.22", optional = true, features = ["derive", "env"] }
+serde_json = "1.0.86"
 
 # rnix has not been released in a while (as of 2022-09-23), we will
 # use it from git.
diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix
index 5147ebe8e678..2989e8348f15 100644
--- a/tvix/eval/default.nix
+++ b/tvix/eval/default.nix
@@ -106,11 +106,15 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: {
 
           base="$(dirname "$i")/$(basename "$i" ".nix")"
 
-          if [[ "$(basename "$i")" == "eval-okay-search-path.nix" ]]; then
-            # TODO(sterni): fix this test
-            echo "SKIPPED: $i"
-            continue
-          fi
+          case "$(basename $i)" in
+            eval-okay-search-path.nix) ;&
+            eval-okay-fromjson.nix)
+              # TODO(sterni,grfn): fix these tests
+              echo "SKIPPED: $i"
+              continue
+              ;;
+            *) ;;
+          esac
 
           if test -e $base.exp; then
               flags=
diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs
index 1ed59ffdc8ed..c0dce868c6fc 100644
--- a/tvix/eval/src/builtins/mod.rs
+++ b/tvix/eval/src/builtins/mod.rs
@@ -271,6 +271,11 @@ fn pure_builtins() -> Vec<Builtin> {
                 Ok(res)
             },
         ),
+        Builtin::new("fromJSON", &[true], |args: Vec<Value>, _: &mut VM| {
+            let json_str = args[0].to_str()?;
+            let json: serde_json::Value = serde_json::from_str(&json_str)?;
+            json.try_into()
+        }),
         Builtin::new("genList", &[true, true], |args: Vec<Value>, vm: &mut VM| {
             let len = args[1].as_int()?;
             (0..len)
diff --git a/tvix/eval/src/errors.rs b/tvix/eval/src/errors.rs
index 1c0d71f6188f..33b12daa5d82 100644
--- a/tvix/eval/src/errors.rs
+++ b/tvix/eval/src/errors.rs
@@ -129,6 +129,9 @@ pub enum ErrorKind {
         error: Rc<io::Error>,
     },
 
+    /// Errors converting JSON to a value
+    FromJsonError(String),
+
     /// Tvix internal warning for features triggered by users that are
     /// not actually implemented yet, and without which eval can not
     /// proceed.
@@ -176,6 +179,13 @@ impl ErrorKind {
     }
 }
 
+impl From<serde_json::Error> for ErrorKind {
+    fn from(err: serde_json::Error) -> Self {
+        // Can't just put the `serde_json::Error` in the ErrorKind since it doesn't impl `Clone`
+        Self::FromJsonError(format!("Error parsing JSON: {err}"))
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct Error {
     pub kind: ErrorKind,
@@ -343,6 +353,10 @@ to a missing value in the attribute set(s) included via `with`."#,
                 write!(f, "{error}")
             }
 
+            ErrorKind::FromJsonError(msg) => {
+                write!(f, "Error converting JSON to a Nix value: {msg}")
+            }
+
             ErrorKind::NotImplemented(feature) => {
                 write!(f, "feature not yet implemented in Tvix: {}", feature)
             }
@@ -621,6 +635,7 @@ impl Error {
             | ErrorKind::ImportParseError { .. }
             | ErrorKind::ImportCompilerError { .. }
             | ErrorKind::IO { .. }
+            | ErrorKind::FromJsonError(_)
             | ErrorKind::NotImplemented(_) => return None,
         };
 
@@ -659,6 +674,7 @@ impl Error {
             ErrorKind::ImportParseError { .. } => "E027",
             ErrorKind::ImportCompilerError { .. } => "E028",
             ErrorKind::IO { .. } => "E029",
+            ErrorKind::FromJsonError { .. } => "E030",
 
             // Placeholder error while Tvix is under construction.
             ErrorKind::NotImplemented(_) => "E999",
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp
new file mode 100644
index 000000000000..add5505a8287
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp
@@ -0,0 +1 @@
+"quote \" reverse solidus \\ solidus / backspace  formfeed  newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace  1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →"
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix
new file mode 100644
index 000000000000..f00713507732
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix
@@ -0,0 +1,3 @@
+# This string contains all supported escapes in a JSON string, per json.org
+# \b and \f are not supported by Nix
+builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"''
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp
new file mode 100644
index 000000000000..4f75c09231b6
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp
@@ -0,0 +1 @@
+[ { Image = { Animated = false; Height = 600; IDs = [ 116 943 234 38793 true false null -100 ]; Latitude = 37.7668; Longitude = -122.3959; Thumbnail = { Height = 125; Url = "http://www.example.com/image/481989943"; Width = 100; }; Title = "View from 15th Floor"; Width = 800; }; } { name = "a"; value = "b"; } ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix
new file mode 100644
index 000000000000..ccb83fd0bd72
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix
@@ -0,0 +1,23 @@
+[
+# RFC 7159, section 13.
+  (builtins.fromJSON
+    ''
+    {
+      "Image": {
+          "Width":  800,
+          "Height": 600,
+          "Title":  "View from 15th Floor",
+          "Thumbnail": {
+              "Url":    "http://www.example.com/image/481989943",
+              "Height": 125,
+              "Width":  100
+          },
+          "Animated" : false,
+          "IDs": [116, 943, 234, 38793, true  ,false,null, -100],
+          "Latitude":  37.7668,
+          "Longitude": -122.3959
+      }
+    }
+  '')
+  (builtins.fromJSON ''{"name": "a", "value": "b"}'')
+]
diff --git a/tvix/eval/src/value/attrs.rs b/tvix/eval/src/value/attrs.rs
index 318a8cfa8209..e9d5a239a3cf 100644
--- a/tvix/eval/src/value/attrs.rs
+++ b/tvix/eval/src/value/attrs.rs
@@ -274,6 +274,12 @@ impl NixAttrs {
         NixAttrs(AttrsRep::Map(map))
     }
 
+    /// Construct an optimized "KV"-style attribute set given the value for the
+    /// `"name"` key, and the value for the `"value"` key
+    pub(crate) fn from_kv(name: Value, value: Value) -> Self {
+        NixAttrs(AttrsRep::KV { name, value })
+    }
+
     /// Compare `self` against `other` for equality using Nix equality semantics
     pub fn nix_eq(&self, other: &Self, vm: &mut VM) -> Result<bool, ErrorKind> {
         match (&self.0, &other.0) {
@@ -376,10 +382,10 @@ fn attempt_optimise_kv(slice: &mut [Value]) -> Option<NixAttrs> {
         }
     };
 
-    Some(NixAttrs(AttrsRep::KV {
-        name: slice[name_idx].clone(),
-        value: slice[value_idx].clone(),
-    }))
+    Some(NixAttrs::from_kv(
+        slice[name_idx].clone(),
+        slice[value_idx].clone(),
+    ))
 }
 
 /// Set an attribute on an in-construction attribute set, while
diff --git a/tvix/eval/src/value/mod.rs b/tvix/eval/src/value/mod.rs
index 8672ffc1bb89..175b33bfa2e8 100644
--- a/tvix/eval/src/value/mod.rs
+++ b/tvix/eval/src/value/mod.rs
@@ -390,6 +390,54 @@ impl From<PathBuf> for Value {
     }
 }
 
+impl From<Vec<Value>> for Value {
+    fn from(val: Vec<Value>) -> Self {
+        Self::List(NixList::from(val))
+    }
+}
+
+impl TryFrom<serde_json::Value> for Value {
+    type Error = ErrorKind;
+
+    fn try_from(value: serde_json::Value) -> Result<Self, Self::Error> {
+        // TODO(grfn): Replace with a real serde::Deserialize impl (for perf)
+        match value {
+            serde_json::Value::Null => Ok(Self::Null),
+            serde_json::Value::Bool(b) => Ok(Self::Bool(b)),
+            serde_json::Value::Number(n) => {
+                if let Some(i) = n.as_i64() {
+                    Ok(Self::Integer(i))
+                } else if let Some(f) = n.as_f64() {
+                    Ok(Self::Float(f))
+                } else {
+                    Err(ErrorKind::FromJsonError(format!(
+                        "JSON number not representable as Nix value: {n}"
+                    )))
+                }
+            }
+            serde_json::Value::String(s) => Ok(s.into()),
+            serde_json::Value::Array(a) => Ok(a
+                .into_iter()
+                .map(Value::try_from)
+                .collect::<Result<Vec<_>, _>>()?
+                .into()),
+            serde_json::Value::Object(obj) => {
+                match (obj.len(), obj.get("name"), obj.get("value")) {
+                    (2, Some(name), Some(value)) => Ok(Self::attrs(NixAttrs::from_kv(
+                        name.clone().try_into()?,
+                        value.clone().try_into()?,
+                    ))),
+                    _ => Ok(Self::attrs(NixAttrs::from_map(
+                        obj.into_iter()
+                            .map(|(k, v)| Ok((k.into(), v.try_into()?)))
+                            .collect::<Result<_, ErrorKind>>()?,
+                    ))),
+                }
+            }
+        }
+    }
+}
+
 fn type_error(expected: &'static str, actual: &Value) -> ErrorKind {
     ErrorKind::TypeError {
         expected,