feat(tvix/eval): expect not-yet-passing tests to fail r/5161

It is helpful to be able to use the test suite as a regression test:
make a change to the compiler/vm, re-run the tests, and if there are
any failures you know it's your fault.

Right now we can't do that, because the expected-to-fail tests are
mixed in with the expected-to-pass tests.  So we can't use them as a
regression test.

Change-Id: Ied606882b9835a7effd7e75bfcf3e5f827e0a2c8
Signed-off-by: Adam Joseph <adam@westernsemico.com>
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7036
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

74 files changed, 49 insertions, 14 deletions

diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix
index 2989e8348f15..7328ed7ee2c2 100644
--- a/tvix/eval/default.nix
+++ b/tvix/eval/default.nix
@@ -108,6 +108,7 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: {
 
           case "$(basename $i)" in
             eval-okay-search-path.nix) ;&
+            eval-okay-tail-call-1.nix | \
             eval-okay-fromjson.nix)
               # TODO(sterni,grfn): fix these tests
               echo "SKIPPED: $i"
diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index a0980cc029ab..8a58b3cc7973 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -14,20 +14,31 @@ fn eval_test(code_path: &str, expect_success: bool) {
 
     match interpret(&code, Some(code_path.into()), Options::test_options()) {
         Ok(result) => {
-            if !expect_success {
-                panic!(
-                    "test should have failed, but succeeded with output {}",
-                    result
-                );
-            }
             let result_str = format!("{}", result);
-            let exp =
-                std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
-            assert_eq!(
-                result_str,
-                exp.trim(),
-                "result value representation (left) must match expectation (right)"
-            );
+            if let Ok(exp) = std::fs::read_to_string(exp_path) {
+                if expect_success {
+                    assert_eq!(
+                        result_str,
+                        exp.trim(),
+                        "result value representation (left) must match expectation (right)"
+                    );
+                } else {
+                    assert_ne!(
+                        result_str,
+                        exp.trim(),
+                        "test passed unexpectedly!  consider moving it out of notyetpassing"
+                    );
+                }
+            } else {
+                if expect_success {
+                    panic!("should be able to read test expectation");
+                } else {
+                    panic!(
+                        "test should have failed, but succeeded with output {}",
+                        result
+                    );
+                }
+            }
         }
         Err(e) => {
             if expect_success {
@@ -67,13 +78,36 @@ fn eval_okay(code_path: &str) {
     eval_test(code_path, true)
 }
 
-// eval-fail-* tests from the original Nix test suite.
+// eval-okay-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
 #[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
 fn nix_eval_okay(code_path: &str) {
     eval_test(code_path, true)
 }
 
+// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix
+//
+// Eventually there will be none of these left, and this function
+// will disappear :) Until then, to run these tests, use `cargo test
+// --features expected_failures`.
+//
+// Please don't submit failing tests unless they're in
+// notyetpassing; this makes the test suite much more useful for
+// regression testing, since there should always be zero non-ignored
+// failing tests.
+//
+// Unfortunately test_generator is unmaintained, so the PRs to make
+// it understand #[ignored] has been sitting for two years, so we
+// can't use `cargo test --include-ignored`, which is the normal way
+// of handling this situation.
+//
+//   https://github.com/frehberg/test-generator/pull/10
+//   https://github.com/frehberg/test-generator/pull/8
+#[test_resources("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")]
+fn nix_eval_okay_currently_failing(code_path: &str) {
+    eval_test(code_path, false)
+}
+
 // eval-fail-* tests contain a snippet of Nix code, which is
 // expected to fail evaluation.  The exact type of failure
 // (assertion, parse error, etc) is not currently checked.
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp-disabled b/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp
index f7393e847d34..f7393e847d34 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp-disabled
+++ b/tvix/eval/src/tests/nix_tests/eval-okay-tail-call-1.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-antiquoted-path.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-antiquoted-path.nix
index f2f08107b516..f2f08107b516 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-antiquoted-path.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-antiquoted-path.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-bad-antiquote-2.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-bad-antiquote-2.nix
index 3745235ce95e..3745235ce95e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-bad-antiquote-2.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-bad-antiquote-2.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-scope-5.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-scope-5.nix
index f89a65a99be3..f89a65a99be3 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-scope-5.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-scope-5.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-undeclared-arg.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-undeclared-arg.nix
index cafdf1636272..cafdf1636272 100644
--- a/tvix/eval/src/tests/nix_tests/eval-fail-undeclared-arg.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-fail-undeclared-arg.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.exp
index b46938032e73..b46938032e73 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.nix
index 2e5c85483be6..2e5c85483be6 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-attrs6.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-attrs6.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.exp
index 7a8391786a09..7a8391786a09 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.flags b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.flags
index 217c7a5ae291..217c7a5ae291 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.flags
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.flags
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.nix
index 815f51b1d67a..815f51b1d67a 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-autoargs.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-autoargs.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.exp
index 0661686d611d..0661686d611d 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.nix
index e9d65e88a817..e9d65e88a817 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-builtins.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-builtins.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp
index dffc03a99891..dffc03a99891 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-closure.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.nix
index cccd4dc35730..cccd4dc35730 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-closure.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.exp
index 27ba77ddaf61..27ba77ddaf61 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.nix
index 43178bd2eef9..43178bd2eef9 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context-introspection.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context-introspection.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.exp
index 2f535bdbc454..2f535bdbc454 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-context.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.nix
index 7b9531cfe9e1..7b9531cfe9e1 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-context.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-context.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.exp
index 65fd65b4d01f..65fd65b4d01f 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.nix
index b79553df0bd3..b79553df0bd3 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-curpos.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-curpos.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.exp
index 8d38505c1686..8d38505c1686 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.nix
index 53aa4b1dc251..53aa4b1dc251 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-deepseq.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-deepseq.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.exp
index eaacb55c1aff..eaacb55c1aff 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.nix
index 84b388c27130..84b388c27130 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with-inherit.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with-inherit.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.exp
index 8e7c61ab8e77..8e7c61ab8e77 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.nix
index 3fb023e1cd42..3fb023e1cd42 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-delayed-with.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-delayed-with.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.exp
index ec04aab6aeec..ec04aab6aeec 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.nix
index d526cb4a2161..d526cb4a2161 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq-derivations.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq-derivations.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq.exp.disabled b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.exp
index 2015847b65e7..2015847b65e7 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq.exp.disabled
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-eq.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.nix
index 73d200b38141..73d200b38141 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-eq.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-eq.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.exp
index d0dd3af2c814..d0dd3af2c814 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.nix
index 963932689942..963932689942 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromTOML.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromTOML.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.exp
index 27ba77ddaf61..27ba77ddaf61 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.nix
index 102ee82b5e6b..102ee82b5e6b 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-fromjson.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-fromjson.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp
index 651f54c36341..651f54c36341 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix
index 68dca62ee18d..68dca62ee18d 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.exp
index 19765bd501b6..19765bd501b6 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.nix
index 14dd38f7734c..14dd38f7734c 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos-undefined.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos-undefined.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.exp
index 469249bbc646..469249bbc646 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.nix
index ca6b07961547..ca6b07961547 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getattrpos.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getattrpos.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.exp
index 14e24d419005..14e24d419005 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.nix
index 4cfec5f553d9..4cfec5f553d9 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-getenv.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-getenv.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp
index ff1e8293ef22..ff1e8293ef22 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix
index aff5a1856814..aff5a1856814 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashfile.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashfile.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp
index d720a082ddb3..d720a082ddb3 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix
index b0f62b245ca8..b0f62b245ca8 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-import.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.exp
index c508125b55be..c508125b55be 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-import.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-import.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.nix
index 0b18d9413122..0b18d9413122 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-import.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-import.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.exp
index 3f113f17bab1..3f113f17bab1 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.nix
index f075b6275e5a..f075b6275e5a 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-mapattrs.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-mapattrs.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.exp
index 0cfbf08886fc..0cfbf08886fc 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.nix
index 358742b36e22..358742b36e22 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-overrides.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-overrides.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-path.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-path.nix
index e67168cf3edf..e67168cf3edf 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-path.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-path.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.exp
index 27ba77ddaf61..27ba77ddaf61 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.nix
index 50c28ee0cd30..50c28ee0cd30 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-pathexists.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-pathexists.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.exp
index 27ba77ddaf61..27ba77ddaf61 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.nix
index 0073e057787d..0073e057787d 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-regex-split.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-regex-split.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.exp
index 72e8274d8c58..72e8274d8c58 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.nix
index bd8031fc004e..bd8031fc004e 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-replacestrings.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-replacestrings.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.exp
index 4519bc406db5..4519bc406db5 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.flags b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.flags
index a28e6821004a..a28e6821004a 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.flags
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.flags
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.nix
index cca41f821f83..cca41f821f83 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-search-path.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-search-path.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-sort.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.exp
index 148b93516394..148b93516394 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-sort.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-sort.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.nix
index 8299c3a4a3aa..8299c3a4a3aa 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-sort.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-sort.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.exp
index e92aae3235f2..e92aae3235f2 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.nix
index ce67943bead5..ce67943bead5 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-tojson.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-tojson.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.exp
index 828220890ecd..828220890ecd 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.nix
index 068c97a6c1b3..068c97a6c1b3 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.exp b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.exp
index 634a841eb190..634a841eb190 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.exp
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.nix
index ff1791b30eb5..ff1791b30eb5 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-toxml2.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-toxml2.nix
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-xml.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.exp
index 92b75e0b8b17..92b75e0b8b17 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-xml.exp.xml
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.exp
diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-xml.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.nix
index 9ee9f8a0b4f5..9ee9f8a0b4f5 100644
--- a/tvix/eval/src/tests/nix_tests/eval-okay-xml.nix
+++ b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-xml.nix