From 5eb89be68246f1e5a8cd28e48d5cec75921ca97a Mon Sep 17 00:00:00 2001 From: Griffin Smith Date: Mon, 10 Oct 2022 00:32:57 -0400 Subject: feat(tvix/eval): Implement builtins.fromJSON Using `serde_json` for parsing JSON here, plus an `impl FromJSON for Value`. The latter is primarily to stay "dependency light" for now - likely going with an actual serde `Deserialize` impl in the future is going to be way better as it allows saving significantly on intermediary allocations. Change-Id: I152a0448ff7c87cf7ebaac927c38912b99de1c18 Reviewed-on: https://cl.tvl.fyi/c/depot/+/6920 Tested-by: BuildkiteCI Reviewed-by: tazjin --- tvix/eval/Cargo.lock | 5 ++- tvix/eval/Cargo.toml | 1 + tvix/eval/default.nix | 14 ++++--- tvix/eval/src/builtins/mod.rs | 5 +++ tvix/eval/src/errors.rs | 16 ++++++++ .../tvix_tests/eval-okay-fromjson-escapes.exp | 1 + .../tvix_tests/eval-okay-fromjson-escapes.nix | 3 ++ .../src/tests/tvix_tests/eval-okay-fromjson.exp | 1 + .../src/tests/tvix_tests/eval-okay-fromjson.nix | 23 +++++++++++ tvix/eval/src/value/attrs.rs | 14 +++++-- tvix/eval/src/value/mod.rs | 48 ++++++++++++++++++++++ 11 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp create mode 100644 tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix create mode 100644 tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp create mode 100644 tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix (limited to 'tvix/eval') diff --git a/tvix/eval/Cargo.lock b/tvix/eval/Cargo.lock index ec91d1c052..b9fa3d3a11 100644 --- a/tvix/eval/Cargo.lock +++ b/tvix/eval/Cargo.lock @@ -997,9 +997,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" dependencies = [ "itoa 1.0.3", "ryu", @@ -1211,6 +1211,7 @@ dependencies = [ "rnix", "rowan", "rustyline", + "serde_json", "smol_str", "tabwriter", "tempdir", diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml index 2edeccb9d1..67ed42609d 100644 --- a/tvix/eval/Cargo.toml +++ b/tvix/eval/Cargo.toml @@ -24,6 +24,7 @@ codemap-diagnostic = "0.1.1" proptest = { version = "1.0.0", default_features = false, features = ["std", "alloc", "break-dead-code", "tempfile"], optional = true } test-strategy = { version = "0.2.1", optional = true } clap = { version = "3.2.22", optional = true, features = ["derive", "env"] } +serde_json = "1.0.86" # rnix has not been released in a while (as of 2022-09-23), we will # use it from git. diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix index 5147ebe8e6..2989e8348f 100644 --- a/tvix/eval/default.nix +++ b/tvix/eval/default.nix @@ -106,11 +106,15 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: { base="$(dirname "$i")/$(basename "$i" ".nix")" - if [[ "$(basename "$i")" == "eval-okay-search-path.nix" ]]; then - # TODO(sterni): fix this test - echo "SKIPPED: $i" - continue - fi + case "$(basename $i)" in + eval-okay-search-path.nix) ;& + eval-okay-fromjson.nix) + # TODO(sterni,grfn): fix these tests + echo "SKIPPED: $i" + continue + ;; + *) ;; + esac if test -e $base.exp; then flags= diff --git a/tvix/eval/src/builtins/mod.rs b/tvix/eval/src/builtins/mod.rs index 1ed59ffdc8..c0dce868c6 100644 --- a/tvix/eval/src/builtins/mod.rs +++ b/tvix/eval/src/builtins/mod.rs @@ -271,6 +271,11 @@ fn pure_builtins() -> Vec { Ok(res) }, ), + Builtin::new("fromJSON", &[true], |args: Vec, _: &mut VM| { + let json_str = args[0].to_str()?; + let json: serde_json::Value = serde_json::from_str(&json_str)?; + json.try_into() + }), Builtin::new("genList", &[true, true], |args: Vec, vm: &mut VM| { let len = args[1].as_int()?; (0..len) diff --git a/tvix/eval/src/errors.rs b/tvix/eval/src/errors.rs index 1c0d71f618..33b12daa5d 100644 --- a/tvix/eval/src/errors.rs +++ b/tvix/eval/src/errors.rs @@ -129,6 +129,9 @@ pub enum ErrorKind { error: Rc, }, + /// Errors converting JSON to a value + FromJsonError(String), + /// Tvix internal warning for features triggered by users that are /// not actually implemented yet, and without which eval can not /// proceed. @@ -176,6 +179,13 @@ impl ErrorKind { } } +impl From for ErrorKind { + fn from(err: serde_json::Error) -> Self { + // Can't just put the `serde_json::Error` in the ErrorKind since it doesn't impl `Clone` + Self::FromJsonError(format!("Error parsing JSON: {err}")) + } +} + #[derive(Clone, Debug)] pub struct Error { pub kind: ErrorKind, @@ -343,6 +353,10 @@ to a missing value in the attribute set(s) included via `with`."#, write!(f, "{error}") } + ErrorKind::FromJsonError(msg) => { + write!(f, "Error converting JSON to a Nix value: {msg}") + } + ErrorKind::NotImplemented(feature) => { write!(f, "feature not yet implemented in Tvix: {}", feature) } @@ -621,6 +635,7 @@ impl Error { | ErrorKind::ImportParseError { .. } | ErrorKind::ImportCompilerError { .. } | ErrorKind::IO { .. } + | ErrorKind::FromJsonError(_) | ErrorKind::NotImplemented(_) => return None, }; @@ -659,6 +674,7 @@ impl Error { ErrorKind::ImportParseError { .. } => "E027", ErrorKind::ImportCompilerError { .. } => "E028", ErrorKind::IO { .. } => "E029", + ErrorKind::FromJsonError { .. } => "E030", // Placeholder error while Tvix is under construction. ErrorKind::NotImplemented(_) => "E999", diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp new file mode 100644 index 0000000000..add5505a82 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.exp @@ -0,0 +1 @@ +"quote \" reverse solidus \\ solidus / backspace  formfeed newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace  1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →" diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix new file mode 100644 index 0000000000..f007135077 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson-escapes.nix @@ -0,0 +1,3 @@ +# This string contains all supported escapes in a JSON string, per json.org +# \b and \f are not supported by Nix +builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"'' diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp new file mode 100644 index 0000000000..4f75c09231 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp @@ -0,0 +1 @@ +[ { Image = { Animated = false; Height = 600; IDs = [ 116 943 234 38793 true false null -100 ]; Latitude = 37.7668; Longitude = -122.3959; Thumbnail = { Height = 125; Url = "http://www.example.com/image/481989943"; Width = 100; }; Title = "View from 15th Floor"; Width = 800; }; } { name = "a"; value = "b"; } ] diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix new file mode 100644 index 0000000000..ccb83fd0bd --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix @@ -0,0 +1,23 @@ +[ +# RFC 7159, section 13. + (builtins.fromJSON + '' + { + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Latitude": 37.7668, + "Longitude": -122.3959 + } + } + '') + (builtins.fromJSON ''{"name": "a", "value": "b"}'') +] diff --git a/tvix/eval/src/value/attrs.rs b/tvix/eval/src/value/attrs.rs index 318a8cfa82..e9d5a239a3 100644 --- a/tvix/eval/src/value/attrs.rs +++ b/tvix/eval/src/value/attrs.rs @@ -274,6 +274,12 @@ impl NixAttrs { NixAttrs(AttrsRep::Map(map)) } + /// Construct an optimized "KV"-style attribute set given the value for the + /// `"name"` key, and the value for the `"value"` key + pub(crate) fn from_kv(name: Value, value: Value) -> Self { + NixAttrs(AttrsRep::KV { name, value }) + } + /// Compare `self` against `other` for equality using Nix equality semantics pub fn nix_eq(&self, other: &Self, vm: &mut VM) -> Result { match (&self.0, &other.0) { @@ -376,10 +382,10 @@ fn attempt_optimise_kv(slice: &mut [Value]) -> Option { } }; - Some(NixAttrs(AttrsRep::KV { - name: slice[name_idx].clone(), - value: slice[value_idx].clone(), - })) + Some(NixAttrs::from_kv( + slice[name_idx].clone(), + slice[value_idx].clone(), + )) } /// Set an attribute on an in-construction attribute set, while diff --git a/tvix/eval/src/value/mod.rs b/tvix/eval/src/value/mod.rs index 8672ffc1bb..175b33bfa2 100644 --- a/tvix/eval/src/value/mod.rs +++ b/tvix/eval/src/value/mod.rs @@ -390,6 +390,54 @@ impl From for Value { } } +impl From> for Value { + fn from(val: Vec) -> Self { + Self::List(NixList::from(val)) + } +} + +impl TryFrom for Value { + type Error = ErrorKind; + + fn try_from(value: serde_json::Value) -> Result { + // TODO(grfn): Replace with a real serde::Deserialize impl (for perf) + match value { + serde_json::Value::Null => Ok(Self::Null), + serde_json::Value::Bool(b) => Ok(Self::Bool(b)), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + Ok(Self::Integer(i)) + } else if let Some(f) = n.as_f64() { + Ok(Self::Float(f)) + } else { + Err(ErrorKind::FromJsonError(format!( + "JSON number not representable as Nix value: {n}" + ))) + } + } + serde_json::Value::String(s) => Ok(s.into()), + serde_json::Value::Array(a) => Ok(a + .into_iter() + .map(Value::try_from) + .collect::, _>>()? + .into()), + serde_json::Value::Object(obj) => { + match (obj.len(), obj.get("name"), obj.get("value")) { + (2, Some(name), Some(value)) => Ok(Self::attrs(NixAttrs::from_kv( + name.clone().try_into()?, + value.clone().try_into()?, + ))), + _ => Ok(Self::attrs(NixAttrs::from_map( + obj.into_iter() + .map(|(k, v)| Ok((k.into(), v.try_into()?))) + .collect::>()?, + ))), + } + } + } + } +} + fn type_error(expected: &'static str, actual: &Value) -> ErrorKind { ErrorKind::TypeError { expected, -- cgit 1.4.1