diff options
author | Florian Klink <flokli@flokli.de> | 2024-01-23T14·19+0200 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2024-01-25T11·37+0000 |
commit | 850e50cd3f9ce83e4e2d73e48a84f496c27b31b3 (patch) | |
tree | 37f29403131877787415aa6ef6a72f5f774fa981 /tvix/eval/src | |
parent | 023e372583b7bcc71b5c301c49501cc72204068f (diff) |
feat(tvix/eval/tests): compare .xml outputs r/7449
In case a .exp.xml file is provided alongside the test, compare its output with the desired state. Also, add some function .exp.xml that were presumably moved out of the way back to src/tests/nix_tests, as they now produce the correct XML output. Change-Id: Ibd8123f3e6ed7bae3a44407d2284a2b2c8ce9a28 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10687 Reviewed-by: sterni <sternenseemann@systemli.org> Autosubmit: flokli <flokli@flokli.de> Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su>
Diffstat (limited to '')
-rw-r--r-- | tvix/eval/src/tests/mod.rs | 49 | ||||
-rw-r--r-- | tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml (renamed from tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml) | 0 | ||||
-rw-r--r-- | tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml (renamed from tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml) | 0 | ||||
-rw-r--r-- | tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix (renamed from tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix) | 0 |
4 files changed, 29 insertions, 20 deletions
diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs index 856707119be0..5a7708e298e8 100644 --- a/tvix/eval/src/tests/mod.rs +++ b/tvix/eval/src/tests/mod.rs @@ -48,17 +48,9 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { "nix", "test files always end in .nix" ); - let exp_path = code_path.with_extension("exp"); - let exp_xml_path = code_path.with_extension("exp.xml"); let code = std::fs::read_to_string(&code_path).expect("should be able to read test code"); - if exp_xml_path.exists() { - // We can't test them at the moment because we don't have XML output yet. - // Checking for success / failure only is a bit disingenious. - return; - } - let mut eval = crate::Evaluation::new_impure(); eval.strict = true; eval.builtins.extend(mock_builtins::builtins()); @@ -79,37 +71,54 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { if !expect_success && failed { return; } + // !expect_success can also mean the output differs, so don't panic if the + // evaluation didn't fail. let value = result.value.unwrap(); let result_str = value.to_string(); - if let Ok(exp) = std::fs::read_to_string(exp_path) { + let exp_path = code_path.with_extension("exp"); + if exp_path.exists() { + // If there's an .exp file provided alongside, compare it with the + // output of the NixValue .to_string() method. + let exp_str = std::fs::read_to_string(&exp_path).expect("unable to read .exp file"); + if expect_success { assert_eq!( result_str, - exp.trim(), + exp_str.trim(), "{}: result value representation (left) must match expectation (right)", code_path.display() ); } else { assert_ne!( result_str, - exp.trim(), + exp_str.trim(), "{}: test passed unexpectedly! consider moving it out of notyetpassing", code_path.display() ); + + // Early return here, we don't compare .xml outputs if this is a ! + // expect_success test. + return; } - } else if expect_success { - panic!( - "{}: should be able to read test expectation", + } + + let exp_xml_path = code_path.with_extension("exp.xml"); + if exp_xml_path.exists() { + // If there's an XML file provided alongside, compare it with the + // output produced when serializing the Value as XML. + let exp_xml_str = std::fs::read_to_string(exp_xml_path).expect("unable to read .xml file"); + + let mut xml_actual_buf = Vec::new(); + crate::builtins::value_to_xml(&mut xml_actual_buf, &value).expect("value_to_xml failed"); + + assert_eq!( + String::from_utf8(xml_actual_buf).expect("to_xml produced invalid utf-8"), + exp_xml_str, + "{}: result value representation (left) must match expectation (right)", code_path.display() ); - } else { - panic!( - "{}: test should have failed, but succeeded with output {}", - code_path.display(), - result_str - ); } } diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml b/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml index dffc03a99891..dffc03a99891 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml +++ b/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml index 651f54c36341..651f54c36341 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml +++ b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix index 68dca62ee18d..68dca62ee18d 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix +++ b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix |