fix(tvix/eval): implement cppnix JSON-serialisation semantics r/5979

This drops the usage of serde::Serialize, as the trait can not be used
to implement the correct semantics (function colouring!).

Instead, a manual JSON serialisation function is written which
correctly handles toString, outPath and other similar weirdnesses.

Unexpectedly, the eval-okay-tojson test from the C++ Nix test suite
now passes, too.

This fixes an issue where serialising data structures containing
derivations to JSON would fail.

Change-Id: I5c39e3d8356ee93a07eda481410f88610f6dd9f8
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8209
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI

5 files changed, 100 insertions, 38 deletions

diff --git a/tvix/eval/src/value/attrs.rs b/tvix/eval/src/value/attrs.rs
index e4584e3aa2f7..bacfd22217bd 100644
--- a/tvix/eval/src/value/attrs.rs
+++ b/tvix/eval/src/value/attrs.rs
@@ -10,8 +10,7 @@ use std::iter::FromIterator;
 use imbl::{ordmap, OrdMap};
 use lazy_static::lazy_static;
 use serde::de::{Deserializer, Error, Visitor};
-use serde::ser::SerializeMap;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 
 use super::string::NixString;
 use super::thunk::ThunkSet;
@@ -151,24 +150,6 @@ impl TotalDisplay for NixAttrs {
     }
 }
 
-impl Serialize for NixAttrs {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        match &self.0 {
-            AttrsRep::Empty => serializer.serialize_map(Some(0))?.end(),
-            AttrsRep::KV { name, value } => {
-                let mut map = serializer.serialize_map(Some(2))?;
-                map.serialize_entry("name", name)?;
-                map.serialize_entry("value", value)?;
-                map.end()
-            }
-            AttrsRep::Im(map) => map.serialize(serializer),
-        }
-    }
-}
-
 impl<'de> Deserialize<'de> for NixAttrs {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
diff --git a/tvix/eval/src/value/json.rs b/tvix/eval/src/value/json.rs
new file mode 100644
index 000000000000..33e16ebffcd0
--- /dev/null
+++ b/tvix/eval/src/value/json.rs
@@ -0,0 +1,84 @@
+/// Implementation of Value serialisation *to* JSON.
+///
+/// This can not be implemented through standard serde-derive methods,
+/// as there is internal Nix logic that must happen within the
+/// serialisation methods.
+use super::{CoercionKind, Value};
+use crate::generators::{self, GenCo};
+use crate::ErrorKind;
+
+use serde_json::value::to_value;
+use serde_json::Value as Json;
+use serde_json::{Map, Number}; // name clash with *our* `Value`
+
+impl Value {
+    pub(crate) async fn to_json(self, co: &GenCo) -> Result<Json, ErrorKind> {
+        let self_forced = generators::request_force(co, self).await;
+
+        let value = match self_forced {
+            Value::Null => Json::Null,
+            Value::Bool(b) => Json::Bool(b),
+            Value::Integer(i) => Json::Number(Number::from(i)),
+            Value::Float(f) => to_value(f)?,
+            Value::String(s) => Json::String(s.as_str().into()),
+
+            Value::Path(p) => {
+                let imported = generators::request_path_import(co, *p).await;
+                Json::String(imported.to_string_lossy().to_string())
+            }
+
+            Value::List(l) => {
+                let mut out = vec![];
+
+                for val in l.into_iter() {
+                    out.push(generators::request_to_json(co, val).await);
+                }
+
+                Json::Array(out)
+            }
+
+            Value::Attrs(attrs) => {
+                // Attribute sets with a callable `__toString` attribute
+                // serialise to the string-coerced version of the result of
+                // calling that.
+                if let Some(s) = attrs.try_to_string(&co, CoercionKind::Weak).await {
+                    return Ok(Json::String(s.as_str().to_string()));
+                }
+
+                // Attribute sets with an `outPath` attribute
+                // serialise to a JSON serialisation of that inner
+                // value (regardless of what it is!).
+                if let Some(out_path) = attrs.select("outPath") {
+                    return Ok(generators::request_to_json(co, out_path.clone()).await);
+                }
+
+                let mut out = Map::with_capacity(attrs.len());
+                for (name, value) in attrs.into_iter_sorted() {
+                    out.insert(
+                        name.as_str().to_string(),
+                        generators::request_to_json(co, value).await,
+                    );
+                }
+
+                Json::Object(out)
+            }
+
+            val @ Value::Closure(_)
+            | val @ Value::Thunk(_)
+            | val @ Value::Builtin(_)
+            | val @ Value::AttrNotFound
+            | val @ Value::Blueprint(_)
+            | val @ Value::DeferredUpvalue(_)
+            | val @ Value::UnresolvedPath(_)
+            | val @ Value::Json(_) => return Err(ErrorKind::NotSerialisableToJson(val.type_of())),
+        };
+
+        Ok(value)
+    }
+
+    /// Generator version of the above, which wraps responses in
+    /// Value::Json.
+    pub(crate) async fn to_json_generator(self, co: GenCo) -> Result<Value, ErrorKind> {
+        Ok(Value::Json(self.to_json(&co).await?))
+    }
+}
diff --git a/tvix/eval/src/value/list.rs b/tvix/eval/src/value/list.rs
index 0bdc61c91d68..cfaefff82195 100644
--- a/tvix/eval/src/value/list.rs
+++ b/tvix/eval/src/value/list.rs
@@ -4,7 +4,7 @@ use std::rc::Rc;
 
 use imbl::{vector, Vector};
 
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 
 use crate::generators;
 use crate::generators::GenCo;
@@ -16,7 +16,7 @@ use super::TotalDisplay;
 use super::Value;
 
 #[repr(transparent)]
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize)]
 pub struct NixList(Rc<Vector<Value>>);
 
 impl TotalDisplay for NixList {
diff --git a/tvix/eval/src/value/mod.rs b/tvix/eval/src/value/mod.rs
index 4e93b36d55d3..41b1e5ac8a16 100644
--- a/tvix/eval/src/value/mod.rs
+++ b/tvix/eval/src/value/mod.rs
@@ -9,13 +9,14 @@ use std::pin::Pin;
 use std::rc::Rc;
 
 use lexical_core::format::CXX_LITERAL;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 
 #[cfg(feature = "arbitrary")]
 mod arbitrary;
 mod attrs;
 mod builtin;
 mod function;
+mod json;
 mod list;
 mod path;
 mod string;
@@ -39,7 +40,7 @@ pub use self::thunk::{SharedThunkSet, ThunkSet};
 use lazy_static::lazy_static;
 
 #[warn(variant_size_differences)]
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 #[serde(untagged)]
 pub enum Value {
     Null,
@@ -76,6 +77,8 @@ pub enum Value {
     DeferredUpvalue(StackIdx),
     #[serde(skip)]
     UnresolvedPath(Box<PathBuf>),
+    #[serde(skip)]
+    Json(serde_json::Value),
 }
 
 lazy_static! {
@@ -231,7 +234,8 @@ impl Value {
             Value::AttrNotFound
             | Value::Blueprint(_)
             | Value::DeferredUpvalue(_)
-            | Value::UnresolvedPath(_) => panic!(
+            | Value::UnresolvedPath(_)
+            | Value::Json(_) => panic!(
                 "Tvix bug: internal value left on stack: {}",
                 value.type_of()
             ),
@@ -322,7 +326,8 @@ impl Value {
             (Value::AttrNotFound, _)
             | (Value::Blueprint(_), _)
             | (Value::DeferredUpvalue(_), _)
-            | (Value::UnresolvedPath(_), _) => {
+            | (Value::UnresolvedPath(_), _)
+            | (Value::Json(_), _) => {
                 panic!("tvix bug: .coerce_to_string() called on internal value")
             }
         }
@@ -512,6 +517,7 @@ impl Value {
             Value::Blueprint(_) => "internal[blueprint]",
             Value::DeferredUpvalue(_) => "internal[deferred_upvalue]",
             Value::UnresolvedPath(_) => "internal[unresolved_path]",
+            Value::Json(_) => "internal[json]",
         }
     }
 
@@ -643,7 +649,8 @@ impl Value {
             Value::AttrNotFound
             | Value::Blueprint(_)
             | Value::DeferredUpvalue(_)
-            | Value::UnresolvedPath(_) => "an internal Tvix evaluator value".into(),
+            | Value::UnresolvedPath(_)
+            | Value::Json(_) => "an internal Tvix evaluator value".into(),
         }
     }
 }
@@ -756,6 +763,7 @@ impl TotalDisplay for Value {
             Value::Blueprint(_) => f.write_str("internal[blueprint]"),
             Value::DeferredUpvalue(_) => f.write_str("internal[deferred_upvalue]"),
             Value::UnresolvedPath(_) => f.write_str("internal[unresolved_path]"),
+            Value::Json(_) => f.write_str("internal[json]"),
 
             // Delegate thunk display to the type, as it must handle
             // the case of already evaluated or cyclic thunks.
diff --git a/tvix/eval/src/value/thunk.rs b/tvix/eval/src/value/thunk.rs
index 42e8ec869ac5..e990a5cad5cc 100644
--- a/tvix/eval/src/value/thunk.rs
+++ b/tvix/eval/src/value/thunk.rs
@@ -25,8 +25,6 @@ use std::{
     rc::Rc,
 };
 
-use serde::Serialize;
-
 use crate::{
     errors::ErrorKind,
     spans::LightSpan,
@@ -269,15 +267,6 @@ impl TotalDisplay for Thunk {
     }
 }
 
-impl Serialize for Thunk {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        self.value().serialize(serializer)
-    }
-}
-
 /// A wrapper type for tracking which thunks have already been seen in a
 /// context. This is necessary for cycle detection.
 ///