feat(tvix/eval): expect not-yet-passing tests to fail r/5161

It is helpful to be able to use the test suite as a regression test:
make a change to the compiler/vm, re-run the tests, and if there are
any failures you know it's your fault.

Right now we can't do that, because the expected-to-fail tests are
mixed in with the expected-to-pass tests.  So we can't use them as a
regression test.

Change-Id: Ied606882b9835a7effd7e75bfcf3e5f827e0a2c8
Signed-off-by: Adam Joseph <adam@westernsemico.com>
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7036
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

74 files changed, 49 insertions, 14 deletions

diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix
index 2989e8348f..7328ed7ee2 100644
--- a/tvix/eval/default.nix
+++ b/tvix/eval/default.nix
@@ -108,6 +108,7 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: {
 
           case "$(basename $i)" in
             eval-okay-search-path.nix) ;&
+            eval-okay-tail-call-1.nix | \
             eval-okay-fromjson.nix)
               # TODO(sterni,grfn): fix these tests
               echo "SKIPPED: $i"
diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index a0980cc029..8a58b3cc79 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -14,20 +14,31 @@ fn eval_test(code_path: &str, expect_success: bool) {
 
     match interpret(&code, Some(code_path.into()), Options::test_options()) {
         Ok(result) => {
-            if !expect_success {
-                panic!(
-                    "test should have failed, but succeeded with output {}",
-                    result
-                );
-            }
             let result_str = format!("{}", result);
-            let exp =
-                std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
-            assert_eq!(
-                result_str,
-                exp.trim(),
-                "result value representation (left) must match expectation (right)"
-            );
+            if let Ok(exp) = std::fs::read_to_string(exp_path) {
+                if expect_success {
+                    assert_eq!(
+                        result_str,
+                        exp.trim(),
+                        "result value representation (left) must match expectation (right)"
+                    );
+                } else {
+                    assert_ne!(
+                        result_str,
+                        exp.trim(),
+                        "test passed unexpectedly!  consider moving it out of notyetpassing"
+                    );
+                }
+            } else {
+                if expect_success {
+                    panic!("should be able to read test expectation");
+                } else {
+                    panic!(
+                        "test should have failed, but succeeded with output {}",
+                        result
+                    );
+                }
+            }
         }
         Err(e) => {
             if expect_success {
@@ -67,13 +78,36 @@ fn eval_okay(code_path: &str) {
     eval_test(code_path, true)
 }
 
-// eval-fail-* tests from the original Nix test suite.
+// eval-okay-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
 #[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
 fn nix_eval_okay(code_path: &str) {
     eval_test(code_path, true)
 }
 
+// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix
+//
+// Eventually there will be none of these left, and this function
+// will disappear :) Until then, to run these tests, use `cargo test
+// --features expected_failures`.
+//
+// Please don't submit failing tests unless they're in
+// notyetpassing; this makes the test suite much more useful for
+// regression testing, since there should always be zero non-ignored
+// failing tests.
+//
+// Unfortunately test_generator is unmaintained, so the PRs to make
+// it understand #[ignored] has been sitting for two years, so we
+// can't use `cargo test --include-ignored`, which is the normal way
+// of handling this situation.
+//
+//   https://github.com/frehberg/test-generator/pull/10
+//   https://github.com/frehberg/test-generator/pull/8
+#[test_resources("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")]
+fn nix_eval_okay_currently_failing(code_path: &str) {
+    eval_test(code_path, false)
+}
+
 // eval-fail-* tests contain a snippet of Nix code, which is
 // expected to fail evaluation.  The exact type of failure
 // (assertion, parse error, etc) is not currently checked.
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp-disabled b/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp
index f7393e847d..f7393e847d 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp-disabled
+++ b/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-antiquoted-path.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-antiquoted-path.nix
index f2f08107b5..f2f08107b5 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-antiquoted-path.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-antiquoted-path.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-bad-antiquote-2.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-bad-antiquote-2.nix
index 3745235ce9..3745235ce9 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-bad-antiquote-2.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-bad-antiquote-2.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-scope-5.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-scope-5.nix
index f89a65a99b..f89a65a99b 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-scope-5.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-scope-5.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-undeclared-arg.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-undeclared-arg.nix
index cafdf16362..cafdf16362 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-undeclared-arg.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-undeclared-arg.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.exp
index b46938032e..b46938032e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.nix
index 2e5c85483b..2e5c85483b 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.exp
index 7a8391786a..7a8391786a 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.flags b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.flags
index 217c7a5ae2..217c7a5ae2 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.flags
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.flags
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.nix
index 815f51b1d6..815f51b1d6 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.exp
index 0661686d61..0661686d61 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.nix
index e9d65e88a8..e9d65e88a8 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp
index dffc03a998..dffc03a998 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-closure.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.nix
index cccd4dc357..cccd4dc357 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-closure.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.exp
index 27ba77ddaf..27ba77ddaf 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.nix
index 43178bd2ee..43178bd2ee 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.exp
index 2f535bdbc4..2f535bdbc4 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.nix
index 7b9531cfe9..7b9531cfe9 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.exp
index 65fd65b4d0..65fd65b4d0 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.nix
index b79553df0b..b79553df0b 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.exp
index 8d38505c16..8d38505c16 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.nix
index 53aa4b1dc2..53aa4b1dc2 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.exp
index eaacb55c1a..eaacb55c1a 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.nix
index 84b388c271..84b388c271 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.exp
index 8e7c61ab8e..8e7c61ab8e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.nix
index 3fb023e1cd..3fb023e1cd 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.exp
index ec04aab6ae..ec04aab6ae 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.nix
index d526cb4a21..d526cb4a21 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq.exp.disabled b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.exp
index 2015847b65..2015847b65 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq.exp.disabled
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.nix
index 73d200b381..73d200b381 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.exp
index d0dd3af2c8..d0dd3af2c8 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.nix
index 9639326899..9639326899 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.exp
index 27ba77ddaf..27ba77ddaf 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.nix
index 102ee82b5e..102ee82b5e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp
index 651f54c363..651f54c363 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix
index 68dca62ee1..68dca62ee1 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.exp
index 19765bd501..19765bd501 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.nix
index 14dd38f773..14dd38f773 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.exp
index 469249bbc6..469249bbc6 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.nix
index ca6b079615..ca6b079615 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.exp
index 14e24d4190..14e24d4190 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.nix
index 4cfec5f553..4cfec5f553 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp
index ff1e8293ef..ff1e8293ef 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix
index aff5a18568..aff5a18568 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp
index d720a082dd..d720a082dd 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix
index b0f62b245c..b0f62b245c 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-import.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.exp
index c508125b55..c508125b55 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-import.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-import.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.nix
index 0b18d94131..0b18d94131 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-import.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.exp
index 3f113f17ba..3f113f17ba 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.nix
index f075b6275e..f075b6275e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.exp
index 0cfbf08886..0cfbf08886 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.nix
index 358742b36e..358742b36e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-path.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-path.nix
index e67168cf3e..e67168cf3e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-path.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-path.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.exp
index 27ba77ddaf..27ba77ddaf 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.nix
index 50c28ee0cd..50c28ee0cd 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.exp
index 27ba77ddaf..27ba77ddaf 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.nix
index 0073e05778..0073e05778 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.exp
index 72e8274d8c..72e8274d8c 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.nix
index bd8031fc00..bd8031fc00 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.exp
index 4519bc406d..4519bc406d 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.flags b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.flags
index a28e682100..a28e682100 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.flags
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.flags
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.nix
index cca41f821f..cca41f821f 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-sort.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.exp
index 148b935163..148b935163 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-sort.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-sort.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.nix
index 8299c3a4a3..8299c3a4a3 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-sort.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.exp
index e92aae3235..e92aae3235 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.nix
index ce67943bea..ce67943bea 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.exp
index 828220890e..828220890e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.nix
index 068c97a6c1..068c97a6c1 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.exp
index 634a841eb1..634a841eb1 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.nix
index ff1791b30e..ff1791b30e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-xml.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.exp
index 92b75e0b8b..92b75e0b8b 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-xml.exp.xml
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-xml.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.nix
index 9ee9f8a0b4..9ee9f8a0b4 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-xml.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.nix