From 67e796b2e128f7fcbf7e41e6403b0fad455e6f9c Mon Sep 17 00:00:00 2001
From: Griffin Smith <root@gws.fyi>
Date: Sat, 3 Sep 2022 11:42:27 -0400
Subject: test(tvix/eval): Add the start of a nix oracle test suite

Add the start of a test suite that compares tvix eval results against
nix, using the string repr of the value as the comparison. This shells
out to a nix-instantiate binary, which is configurable as an environment
variable, to eval - there's some extra machinery there to setup a new
nix store as a tempdir to allow running this test inside the nix build
for tvix-eval itself.

Currently this has a macro that'll allow writing lots and lots of
hardcoded tests, but going forward I'm also going to be looking into
adding proptest-based generation of expressions to compare.

Change-Id: I9f4895fab1e668ed2b7dfd6f92f8c80de1bbb16b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6307
Autosubmit: grfn <grfn@gws.fyi>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
---
 tvix/eval/Cargo.lock          | 123 +++++++++++++++++++++++++++++++-----------
 tvix/eval/Cargo.toml          |   1 +
 tvix/eval/default.nix         |   4 ++
 tvix/eval/tests/nix_oracle.rs |  67 +++++++++++++++++++++++
 4 files changed, 165 insertions(+), 30 deletions(-)
 create mode 100644 tvix/eval/tests/nix_oracle.rs

(limited to 'tvix')

diff --git a/tvix/eval/Cargo.lock b/tvix/eval/Cargo.lock
index 83e0b25ee2..beba1a7bf6 100644
--- a/tvix/eval/Cargo.lock
+++ b/tvix/eval/Cargo.lock
@@ -320,6 +320,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
 [[package]]
 name = "getrandom"
 version = "0.2.7"
@@ -366,9 +372,9 @@ checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06"
 
 [[package]]
 name = "itertools"
-version = "0.10.3"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
+checksum = "d8bf247779e67a9082a4790b45e71ac7cfd1321331a5c856a74a9faebdab78d0"
 dependencies = [
  "either",
 ]
@@ -387,9 +393,9 @@ checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
 
 [[package]]
 name = "js-sys"
-version = "0.3.59"
+version = "0.3.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
+checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -504,9 +510,9 @@ checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd"
 
 [[package]]
 name = "plotters"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "716b4eeb6c4a1d3ecc956f75b43ec2e8e8ba80026413e70a3f41fd3313d3492b"
+checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
 dependencies = [
  "num-traits",
  "plotters-backend",
@@ -588,6 +594,34 @@ dependencies = [
  "nibble_vec",
 ]
 
+[[package]]
+name = "rand"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
+dependencies = [
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.1",
+ "rdrand",
+ "winapi",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
 [[package]]
 name = "rayon"
 version = "1.5.3"
@@ -612,6 +646,15 @@ dependencies = [
  "num_cpus",
 ]
 
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -653,6 +696,15 @@ version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "rnix"
 version = "0.11.0-dev"
@@ -663,9 +715,9 @@ dependencies = [
 
 [[package]]
 name = "rowan"
-version = "0.15.8"
+version = "0.15.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e88acf7b001007e9e8c989fe7449f6601d909e5dd2c56399fc158977ad6c56e8"
+checksum = "7aeb3c95eac7794161baf500b5d4647217ffb32b6660c04d3bb087c834cb58f4"
 dependencies = [
  "countme",
  "hashbrown",
@@ -828,6 +880,16 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "tempdir"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
+dependencies = [
+ "rand",
+ "remove_dir_all",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -866,18 +928,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.34"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c1b05ca9d106ba7d2e31a9dab4a64e7be2cce415321966ea3132c49a656e252"
+checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.34"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8f2591983642de85c921015f3f070c665a197ed69e417af436115e3a1407487"
+checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783"
 dependencies = [
  "proc-macro2 1.0.43",
  "quote 1.0.21",
@@ -910,26 +972,27 @@ dependencies = [
  "rustyline",
  "smol_str",
  "tabwriter",
+ "tempdir",
  "test-generator",
 ]
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
+checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
+checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
 
 [[package]]
 name = "unicode-width"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "unicode-xid"
@@ -962,9 +1025,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
+checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -972,9 +1035,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
+checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
 dependencies = [
  "bumpalo",
  "log",
@@ -987,9 +1050,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
+checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
 dependencies = [
  "quote 1.0.21",
  "wasm-bindgen-macro-support",
@@ -997,9 +1060,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
+checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
 dependencies = [
  "proc-macro2 1.0.43",
  "quote 1.0.21",
@@ -1010,15 +1073,15 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.82"
+version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
+checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
 
 [[package]]
 name = "web-sys"
-version = "0.3.59"
+version = "0.3.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1"
+checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
diff --git a/tvix/eval/Cargo.toml b/tvix/eval/Cargo.toml
index 60c0f12e6f..5ccfa63283 100644
--- a/tvix/eval/Cargo.toml
+++ b/tvix/eval/Cargo.toml
@@ -33,6 +33,7 @@ criterion = "0.3.6"
 test-generator = "0.3.0"
 pretty_assertions = "1.2.1"
 itertools = "0.10.3"
+tempdir = "0.3.7"
 
 [features]
 default = [ "repl" ]
diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix
index a3b4a01ee0..aa8d8a4af4 100644
--- a/tvix/eval/default.nix
+++ b/tvix/eval/default.nix
@@ -9,6 +9,10 @@ depot.third_party.naersk.buildPackage {
   doCheck = true;
   cargoBuildOptions = opts: opts ++ [ "--all-targets" ];
 
+  # Tell the test suite where to find upstream nix, to compare eval results
+  # against
+  NIX_INSTANTIATE_BINARY_PATH = "${pkgs.nix}/bin/nix-instantiate";
+
   meta.ci.targets = builtins.attrNames self.passthru;
 
   passthru.cpp-nix-run-lang-tests = pkgs.stdenv.mkDerivation {
diff --git a/tvix/eval/tests/nix_oracle.rs b/tvix/eval/tests/nix_oracle.rs
new file mode 100644
index 0000000000..1091d6adf0
--- /dev/null
+++ b/tvix/eval/tests/nix_oracle.rs
@@ -0,0 +1,67 @@
+//! Tests which use upstream nix as an oracle to test evaluation against
+
+use std::{env, path::PathBuf, process::Command};
+
+use pretty_assertions::assert_eq;
+use tempdir::TempDir;
+
+fn nix_binary_path() -> PathBuf {
+    env::var("NIX_INSTANTIATE_BINARY_PATH")
+        .unwrap_or_else(|_| "nix-instantiate".to_owned())
+        .into()
+}
+
+fn nix_eval(expr: &str) -> String {
+    let store_dir = TempDir::new("store-dir").unwrap();
+
+    let output = Command::new(nix_binary_path())
+        .args(["--eval", "-E"])
+        .arg(format!("({expr})"))
+        .env(
+            "NIX_REMOTE",
+            format!("local?root={}", store_dir.path().display()),
+        )
+        .output()
+        .unwrap();
+    if !output.status.success() {
+        panic!(
+            "nix eval {expr} failed!\n    stdout: {}\n    stderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr)
+        )
+    }
+
+    String::from_utf8(output.stdout).unwrap()
+}
+
+/// Compare the evaluation of the given nix expression in nix (using the
+/// `NIX_INSTANTIATE_BINARY_PATH` env var to resolve the `nix-instantiate` binary) and tvix, and
+/// assert that the result is identical
+#[track_caller]
+fn compare_eval(expr: &str) {
+    let nix_result = nix_eval(expr);
+    let tvix_result = tvix_eval::interpret(expr, None).unwrap().to_string();
+
+    assert_eq!(nix_result.trim(), tvix_result);
+}
+
+/// Generate a suite of tests which call [`compare_eval`] on expressions, checking that nix and tvix
+/// return identical results.
+macro_rules! compare_eval_tests {
+    () => {};
+    ($(#[$meta:meta])* $test_name: ident($expr: expr); $($rest:tt)*) => {
+        #[test]
+        $(#[$meta])*
+        fn $test_name() {
+            compare_eval($expr);
+        }
+
+        compare_eval_tests!($($rest)*);
+    }
+}
+
+compare_eval_tests! {
+    literal_int("1");
+    add_ints("1 + 1");
+    add_lists("[1 2] ++ [3 4]");
+}
-- 
cgit 1.4.1