feat(tvix/eval): expect not-yet-passing tests to fail r/5161

It is helpful to be able to use the test suite as a regression test:
make a change to the compiler/vm, re-run the tests, and if there are
any failures you know it's your fault.

Right now we can't do that, because the expected-to-fail tests are
mixed in with the expected-to-pass tests.  So we can't use them as a
regression test.

Change-Id: Ied606882b9835a7effd7e75bfcf3e5f827e0a2c8
Signed-off-by: Adam Joseph <adam@westernsemico.com>
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7036
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

1 files changed, 48 insertions, 14 deletions

diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index a0980cc029ab..8a58b3cc7973 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -14,20 +14,31 @@ fn eval_test(code_path: &str, expect_success: bool) {
 
     match interpret(&code, Some(code_path.into()), Options::test_options()) {
         Ok(result) => {
-            if !expect_success {
-                panic!(
-                    "test should have failed, but succeeded with output {}",
-                    result
-                );
-            }
             let result_str = format!("{}", result);
-            let exp =
-                std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
-            assert_eq!(
-                result_str,
-                exp.trim(),
-                "result value representation (left) must match expectation (right)"
-            );
+            if let Ok(exp) = std::fs::read_to_string(exp_path) {
+                if expect_success {
+                    assert_eq!(
+                        result_str,
+                        exp.trim(),
+                        "result value representation (left) must match expectation (right)"
+                    );
+                } else {
+                    assert_ne!(
+                        result_str,
+                        exp.trim(),
+                        "test passed unexpectedly!  consider moving it out of notyetpassing"
+                    );
+                }
+            } else {
+                if expect_success {
+                    panic!("should be able to read test expectation");
+                } else {
+                    panic!(
+                        "test should have failed, but succeeded with output {}",
+                        result
+                    );
+                }
+            }
         }
         Err(e) => {
             if expect_success {
@@ -67,13 +78,36 @@ fn eval_okay(code_path: &str) {
     eval_test(code_path, true)
 }
 
-// eval-fail-* tests from the original Nix test suite.
+// eval-okay-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
 #[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
 fn nix_eval_okay(code_path: &str) {
     eval_test(code_path, true)
 }
 
+// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix
+//
+// Eventually there will be none of these left, and this function
+// will disappear :) Until then, to run these tests, use `cargo test
+// --features expected_failures`.
+//
+// Please don't submit failing tests unless they're in
+// notyetpassing; this makes the test suite much more useful for
+// regression testing, since there should always be zero non-ignored
+// failing tests.
+//
+// Unfortunately test_generator is unmaintained, so the PRs to make
+// it understand #[ignored] has been sitting for two years, so we
+// can't use `cargo test --include-ignored`, which is the normal way
+// of handling this situation.
+//
+//   https://github.com/frehberg/test-generator/pull/10
+//   https://github.com/frehberg/test-generator/pull/8
+#[test_resources("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")]
+fn nix_eval_okay_currently_failing(code_path: &str) {
+    eval_test(code_path, false)
+}
+
 // eval-fail-* tests contain a snippet of Nix code, which is
 // expected to fail evaluation.  The exact type of failure
 // (assertion, parse error, etc) is not currently checked.