From 67e796b2e128f7fcbf7e41e6403b0fad455e6f9c Mon Sep 17 00:00:00 2001
From: Griffin Smith <root@gws.fyi>
Date: Sat, 3 Sep 2022 11:42:27 -0400
Subject: test(tvix/eval): Add the start of a nix oracle test suite

Add the start of a test suite that compares tvix eval results against
nix, using the string repr of the value as the comparison. This shells
out to a nix-instantiate binary, which is configurable as an environment
variable, to eval - there's some extra machinery there to setup a new
nix store as a tempdir to allow running this test inside the nix build
for tvix-eval itself.

Currently this has a macro that'll allow writing lots and lots of
hardcoded tests, but going forward I'm also going to be looking into
adding proptest-based generation of expressions to compare.

Change-Id: I9f4895fab1e668ed2b7dfd6f92f8c80de1bbb16b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6307
Autosubmit: grfn <grfn@gws.fyi>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
---
 tvix/eval/tests/nix_oracle.rs | 67 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 tvix/eval/tests/nix_oracle.rs

(limited to 'tvix/eval/tests/nix_oracle.rs')

diff --git a/tvix/eval/tests/nix_oracle.rs b/tvix/eval/tests/nix_oracle.rs
new file mode 100644
index 0000000000..1091d6adf0
--- /dev/null
+++ b/tvix/eval/tests/nix_oracle.rs
@@ -0,0 +1,67 @@
+//! Tests which use upstream nix as an oracle to test evaluation against
+
+use std::{env, path::PathBuf, process::Command};
+
+use pretty_assertions::assert_eq;
+use tempdir::TempDir;
+
+fn nix_binary_path() -> PathBuf {
+    env::var("NIX_INSTANTIATE_BINARY_PATH")
+        .unwrap_or_else(|_| "nix-instantiate".to_owned())
+        .into()
+}
+
+fn nix_eval(expr: &str) -> String {
+    let store_dir = TempDir::new("store-dir").unwrap();
+
+    let output = Command::new(nix_binary_path())
+        .args(["--eval", "-E"])
+        .arg(format!("({expr})"))
+        .env(
+            "NIX_REMOTE",
+            format!("local?root={}", store_dir.path().display()),
+        )
+        .output()
+        .unwrap();
+    if !output.status.success() {
+        panic!(
+            "nix eval {expr} failed!\n    stdout: {}\n    stderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        )
+    }
+
+    String::from_utf8(output.stdout).unwrap()
+}
+
+/// Compare the evaluation of the given nix expression in nix (using the
+/// `NIX_INSTANTIATE_BINARY_PATH` env var to resolve the `nix-instantiate` binary) and tvix, and
+/// assert that the result is identical
+#[track_caller]
+fn compare_eval(expr: &str) {
+    let nix_result = nix_eval(expr);
+    let tvix_result = tvix_eval::interpret(expr, None).unwrap().to_string();
+
+    assert_eq!(nix_result.trim(), tvix_result);
+}
+
+/// Generate a suite of tests which call [`compare_eval`] on expressions, checking that nix and tvix
+/// return identical results.
+macro_rules! compare_eval_tests {
+    () => {};
+    ($(#[$meta:meta])* $test_name: ident($expr: expr); $($rest:tt)*) => {
+        #[test]
+        $(#[$meta])*
+        fn $test_name() {
+            compare_eval($expr);
+        }
+
+        compare_eval_tests!($($rest)*);
+    }
+}
+
+compare_eval_tests! {
+    literal_int("1");
+    add_ints("1 + 1");
+    add_lists("[1 2] ++ [3 4]");
+}
-- 
cgit 1.4.1