feat(tvix/eval): enable the XFAIL tests r/5123

This commit adds support for running the "expected failure" tests in
both the nix and tvix test suites.

I have disabled the eval-fail-blackhole.nix test because it gets
stuck running forever.

Signed-off-by: Adam Joseph <adam@westernsemico.com>
Change-Id: Iba75ce6c8f2becab3c834fcfdd9f4fdc5a4bdb9f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6990
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: grfn <grfn@gws.fyi>

2 files changed, 45 insertions, 14 deletions

diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index 03e6f9f0220d..a0980cc029ab 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -4,24 +4,40 @@ use pretty_assertions::assert_eq;
 
 use test_generator::test_resources;
 
-fn eval_okay_test(code_path: &str) {
+fn eval_test(code_path: &str, expect_success: bool) {
     let base = code_path
         .strip_suffix("nix")
         .expect("test files always end in .nix");
     let exp_path = format!("{}exp", base);
 
     let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
-    let exp = std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
 
-    let result = interpret(&code, Some(code_path.into()), Options::test_options())
-        .expect("evaluation of eval-okay test should succeed");
-    let result_str = format!("{}", result);
-
-    assert_eq!(
-        result_str,
-        exp.trim(),
-        "result value representation (left) must match expectation (right)"
-    );
+    match interpret(&code, Some(code_path.into()), Options::test_options()) {
+        Ok(result) => {
+            if !expect_success {
+                panic!(
+                    "test should have failed, but succeeded with output {}",
+                    result
+                );
+            }
+            let result_str = format!("{}", result);
+            let exp =
+                std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
+            assert_eq!(
+                result_str,
+                exp.trim(),
+                "result value representation (left) must match expectation (right)"
+            );
+        }
+        Err(e) => {
+            if expect_success {
+                panic!(
+                    "evaluation of eval-okay test should succeed, but failed with {:?}",
+                    e
+                );
+            }
+        }
+    }
 }
 
 // identity-* tests contain Nix code snippets which should evaluate to
@@ -48,12 +64,27 @@ fn identity(code_path: &str) {
 // are guaranteed to be valid Nix code.
 #[test_resources("src/tests/tvix_tests/eval-okay-*.nix")]
 fn eval_okay(code_path: &str) {
-    eval_okay_test(code_path)
+    eval_test(code_path, true)
 }
 
-// eval-okay-* tests from the original Nix test suite.
+// eval-fail-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
 #[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
 fn nix_eval_okay(code_path: &str) {
-    eval_okay_test(code_path)
+    eval_test(code_path, true)
+}
+
+// eval-fail-* tests contain a snippet of Nix code, which is
+// expected to fail evaluation.  The exact type of failure
+// (assertion, parse error, etc) is not currently checked.
+#[test_resources("src/tests/tvix_tests/eval-fail-*.nix")]
+fn eval_fail(code_path: &str) {
+    eval_test(code_path, false)
+}
+
+// eval-fail-* tests from the original Nix test suite.
+#[cfg(feature = "nix_tests")]
+#[test_resources("src/tests/nix_tests/eval-fail-*.nix")]
+fn nix_eval_fail(code_path: &str) {
+    eval_test(code_path, false)
 }
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix b/tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix-disabled
index 81133b511c95..81133b511c95 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix
+++ b/tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix-disabled