diff options
Diffstat (limited to 'tvix/eval/src/tests')
25 files changed, 245 insertions, 204 deletions
diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs index 5a7708e298e8..21b5d35e6af0 100644 --- a/tvix/eval/src/tests/mod.rs +++ b/tvix/eval/src/tests/mod.rs @@ -1,203 +1,6 @@ -use crate::{value::Value, EvalIO}; -use builtin_macros::builtins; -use pretty_assertions::assert_eq; -use rstest::rstest; -use std::path::PathBuf; - /// Module for one-off tests which do not follow the rest of the /// test layout. mod one_offs; -#[builtins] -mod mock_builtins { - //! Builtins which are required by language tests, but should not - //! actually exist in //tvix/eval. - use crate as tvix_eval; - use crate::generators::GenCo; - use crate::*; - use genawaiter::rc::Gen; - - #[builtin("derivation")] - async fn builtin_derivation(co: GenCo, input: Value) -> Result<Value, ErrorKind> { - let input = input.to_attrs()?; - let attrs = input.update(NixAttrs::from_iter( - [ - ( - "outPath", - "/nix/store/00000000000000000000000000000000-mock", - ), - ( - "drvPath", - "/nix/store/00000000000000000000000000000000-mock.drv", - ), - ("type", "derivation"), - ] - .into_iter(), - )); - - Ok(Value::Attrs(Box::new(attrs))) - } -} - -fn eval_test(code_path: PathBuf, expect_success: bool) { - std::env::set_var("TEST_VAR", "foo"); // for eval-okay-getenv.nix - - eprintln!("path: {}", code_path.display()); - assert_eq!( - code_path.extension().unwrap(), - "nix", - "test files always end in .nix" - ); - - let code = std::fs::read_to_string(&code_path).expect("should be able to read test code"); - - let mut eval = crate::Evaluation::new_impure(); - eval.strict = true; - eval.builtins.extend(mock_builtins::builtins()); - - let result = eval.evaluate(code, Some(code_path.clone())); - let failed = match result.value { - Some(Value::Catchable(_)) => true, - _ => !result.errors.is_empty(), - }; - if expect_success && failed { - panic!( - "{}: evaluation of eval-okay test should succeed, but failed with {:?}", - code_path.display(), - result.errors, - ); - } - - if !expect_success && failed { - return; - } - // !expect_success can also mean the output differs, so don't panic if the - // evaluation didn't fail. - - let value = result.value.unwrap(); - let result_str = value.to_string(); - - let exp_path = code_path.with_extension("exp"); - if exp_path.exists() { - // If there's an .exp file provided alongside, compare it with the - // output of the NixValue .to_string() method. - let exp_str = std::fs::read_to_string(&exp_path).expect("unable to read .exp file"); - - if expect_success { - assert_eq!( - result_str, - exp_str.trim(), - "{}: result value representation (left) must match expectation (right)", - code_path.display() - ); - } else { - assert_ne!( - result_str, - exp_str.trim(), - "{}: test passed unexpectedly! consider moving it out of notyetpassing", - code_path.display() - ); - - // Early return here, we don't compare .xml outputs if this is a ! - // expect_success test. - return; - } - } - - let exp_xml_path = code_path.with_extension("exp.xml"); - if exp_xml_path.exists() { - // If there's an XML file provided alongside, compare it with the - // output produced when serializing the Value as XML. - let exp_xml_str = std::fs::read_to_string(exp_xml_path).expect("unable to read .xml file"); - - let mut xml_actual_buf = Vec::new(); - crate::builtins::value_to_xml(&mut xml_actual_buf, &value).expect("value_to_xml failed"); - - assert_eq!( - String::from_utf8(xml_actual_buf).expect("to_xml produced invalid utf-8"), - exp_xml_str, - "{}: result value representation (left) must match expectation (right)", - code_path.display() - ); - } -} - -// identity-* tests contain Nix code snippets which should evaluate to -// themselves exactly (i.e. literals). -#[rstest] -fn identity(#[files("src/tests/tvix_tests/identity-*.nix")] code_path: PathBuf) { - let code = std::fs::read_to_string(code_path).expect("should be able to read test code"); - - let mut eval = crate::Evaluation::new(Box::new(crate::StdIO) as Box<dyn EvalIO>, false); - eval.strict = true; - - let result = eval.evaluate(&code, None); - assert!( - result.errors.is_empty(), - "evaluation of identity test failed: {:?}", - result.errors - ); - - let result_str = result.value.unwrap().to_string(); - - assert_eq!( - result_str, - code.trim(), - "result value representation (left) must match expectation (right)" - ) -} - -// eval-okay-* tests contain a snippet of Nix code, and an expectation -// of the produced string output of the evaluator. -// -// These evaluations are always supposed to succeed, i.e. all snippets -// are guaranteed to be valid Nix code. -#[rstest] -fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) { - eval_test(code_path, true) -} - -// eval-okay-* tests from the original Nix test suite. -#[cfg(feature = "nix_tests")] -#[rstest] -fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) { - eval_test(code_path, true) -} - -// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix -// -// Eventually there will be none of these left, and this function -// will disappear :) -// -// Please don't submit failing tests unless they're in -// notyetpassing; this makes the test suite much more useful for -// regression testing, since there should always be zero non-ignored -// failing tests. -#[rstest] -fn nix_eval_okay_currently_failing( - #[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, -) { - eval_test(code_path, false) -} - -#[rstest] -fn eval_okay_currently_failing( - #[files("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, -) { - eval_test(code_path, false) -} - -// eval-fail-* tests contain a snippet of Nix code, which is -// expected to fail evaluation. The exact type of failure -// (assertion, parse error, etc) is not currently checked. -#[rstest] -fn eval_fail(#[files("src/tests/tvix_tests/eval-fail-*.nix")] code_path: PathBuf) { - eval_test(code_path, false) -} - -// eval-fail-* tests from the original Nix test suite. #[cfg(feature = "nix_tests")] -#[rstest] -fn nix_eval_fail(#[files("src/tests/nix_tests/eval-fail-*.nix")] code_path: PathBuf) { - eval_test(code_path, false) -} +mod nix_tests; diff --git a/tvix/eval/src/tests/nix_tests.rs b/tvix/eval/src/tests/nix_tests.rs new file mode 100644 index 000000000000..b17fe98bc9e4 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests.rs @@ -0,0 +1,212 @@ +use crate::value::Value; +use builtin_macros::builtins; +use pretty_assertions::assert_eq; +use rstest::rstest; +use std::path::PathBuf; + +#[builtins] +mod mock_builtins { + //! Builtins which are required by language tests, but should not + //! actually exist in //tvix/eval. + use crate as tvix_eval; + use crate::generators::GenCo; + use crate::*; + use genawaiter::rc::Gen; + + #[builtin("derivation")] + async fn builtin_derivation(co: GenCo, input: Value) -> Result<Value, ErrorKind> { + let input = input.to_attrs()?; + let attrs = input.update(NixAttrs::from_iter( + [ + ( + "outPath", + "/nix/store/00000000000000000000000000000000-mock", + ), + ( + "drvPath", + "/nix/store/00000000000000000000000000000000-mock.drv", + ), + ("type", "derivation"), + ] + .into_iter(), + )); + + Ok(Value::Attrs(Box::new(attrs))) + } +} + +#[cfg(feature = "impure")] +fn eval_test(code_path: PathBuf, expect_success: bool) { + use crate::vm::EvalMode; + + std::env::set_var("TEST_VAR", "foo"); // for eval-okay-getenv.nix + + eprintln!("path: {}", code_path.display()); + assert_eq!( + code_path.extension().unwrap(), + "nix", + "test files always end in .nix" + ); + + let code = std::fs::read_to_string(&code_path).expect("should be able to read test code"); + + let eval = crate::Evaluation::builder_impure() + .mode(EvalMode::Strict) + .add_builtins(mock_builtins::builtins()) + .build(); + + let result = eval.evaluate(code, Some(code_path.clone())); + let failed = match result.value { + Some(Value::Catchable(_)) => true, + _ => !result.errors.is_empty(), + }; + if expect_success && failed { + panic!( + "{}: evaluation of eval-okay test should succeed, but failed with {:?}", + code_path.display(), + result.errors, + ); + } + + if !expect_success && failed { + return; + } + // !expect_success can also mean the output differs, so don't panic if the + // evaluation didn't fail. + + let value = result.value.unwrap(); + let result_str = value.to_string(); + + let exp_path = code_path.with_extension("exp"); + if exp_path.exists() { + // If there's an .exp file provided alongside, compare it with the + // output of the NixValue .to_string() method. + let exp_str = std::fs::read_to_string(&exp_path).expect("unable to read .exp file"); + + if expect_success { + assert_eq!( + result_str, + exp_str.trim(), + "{}: result value representation (left) must match expectation (right)", + code_path.display() + ); + } else { + assert_ne!( + result_str, + exp_str.trim(), + "{}: test passed unexpectedly! consider moving it out of notyetpassing", + code_path.display() + ); + + // Early return here, we don't compare .xml outputs if this is a ! + // expect_success test. + return; + } + } + + let exp_xml_path = code_path.with_extension("exp.xml"); + if exp_xml_path.exists() { + // If there's an XML file provided alongside, compare it with the + // output produced when serializing the Value as XML. + let exp_xml_str = std::fs::read_to_string(exp_xml_path).expect("unable to read .xml file"); + + let mut xml_actual_buf = Vec::new(); + crate::builtins::value_to_xml(&mut xml_actual_buf, &value).expect("value_to_xml failed"); + + assert_eq!( + String::from_utf8(xml_actual_buf).expect("to_xml produced invalid utf-8"), + exp_xml_str, + "{}: result value representation (left) must match expectation (right)", + code_path.display() + ); + } +} + +// identity-* tests contain Nix code snippets which should evaluate to +// themselves exactly (i.e. literals). +#[cfg(feature = "impure")] +#[rstest] +fn identity(#[files("src/tests/tvix_tests/identity-*.nix")] code_path: PathBuf) { + use crate::{vm::EvalMode, EvalIO}; + + let code = std::fs::read_to_string(code_path).expect("should be able to read test code"); + + let eval = crate::Evaluation::builder(Box::new(crate::StdIO) as Box<dyn EvalIO>) + .disable_import() + .mode(EvalMode::Strict) + .build(); + + let result = eval.evaluate(&code, None); + assert!( + result.errors.is_empty(), + "evaluation of identity test failed: {:?}", + result.errors + ); + + let result_str = result.value.unwrap().to_string(); + + assert_eq!( + result_str, + code.trim(), + "result value representation (left) must match expectation (right)" + ) +} + +// eval-okay-* tests contain a snippet of Nix code, and an expectation +// of the produced string output of the evaluator. +// +// These evaluations are always supposed to succeed, i.e. all snippets +// are guaranteed to be valid Nix code. +#[cfg(feature = "impure")] +#[rstest] +fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) { + eval_test(code_path, true) +} + +// eval-okay-* tests from the original Nix test suite. +#[cfg(feature = "impure")] +#[rstest] +fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) { + eval_test(code_path, true) +} + +// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix +// +// Eventually there will be none of these left, and this function +// will disappear :) +// +// Please don't submit failing tests unless they're in +// notyetpassing; this makes the test suite much more useful for +// regression testing, since there should always be zero non-ignored +// failing tests. +#[cfg(feature = "impure")] +#[rstest] +fn nix_eval_okay_currently_failing( + #[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, +) { + eval_test(code_path, false) +} + +#[cfg(feature = "impure")] +#[rstest] +fn eval_okay_currently_failing( + #[files("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, +) { + eval_test(code_path, false) +} + +// eval-fail-* tests contain a snippet of Nix code, which is +// expected to fail evaluation. The exact type of failure +// (assertion, parse error, etc) is not currently checked. +#[cfg(feature = "impure")] +#[rstest] +fn eval_fail(#[files("src/tests/tvix_tests/eval-fail-*.nix")] code_path: PathBuf) { + eval_test(code_path, false) +} + +// eval-fail-* tests from the original Nix test suite. +#[cfg(feature = "impure")] +#[rstest] +fn nix_eval_fail(#[files("src/tests/nix_tests/eval-fail-*.nix")] code_path: PathBuf) { + eval_test(code_path, false) +} diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readDir.exp b/tvix/eval/src/tests/nix_tests/eval-okay-readDir.exp index 6413f6d4f9ec..6413f6d4f9ec 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readDir.exp +++ b/tvix/eval/src/tests/nix_tests/eval-okay-readDir.exp diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readDir.nix.disabled b/tvix/eval/src/tests/nix_tests/eval-okay-readDir.nix index a7ec9292aae2..a7ec9292aae2 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readDir.nix.disabled +++ b/tvix/eval/src/tests/nix_tests/eval-okay-readDir.nix diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readFileType.exp b/tvix/eval/src/tests/nix_tests/eval-okay-readFileType.exp index 6413f6d4f9ec..6413f6d4f9ec 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readFileType.exp +++ b/tvix/eval/src/tests/nix_tests/eval-okay-readFileType.exp diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readFileType.nix b/tvix/eval/src/tests/nix_tests/eval-okay-readFileType.nix index 174fb6c3a028..174fb6c3a028 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-readFileType.nix +++ b/tvix/eval/src/tests/nix_tests/eval-okay-readFileType.nix diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/bar b/tvix/eval/src/tests/nix_tests/readDir/bar index e69de29bb2d1..e69de29bb2d1 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/bar +++ b/tvix/eval/src/tests/nix_tests/readDir/bar diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/foo/git-hates-directories b/tvix/eval/src/tests/nix_tests/readDir/foo/git-hates-directories index e69de29bb2d1..e69de29bb2d1 100644 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/foo/git-hates-directories +++ b/tvix/eval/src/tests/nix_tests/readDir/foo/git-hates-directories diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/ldir b/tvix/eval/src/tests/nix_tests/readDir/ldir index 19102815663d..19102815663d 120000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/ldir +++ b/tvix/eval/src/tests/nix_tests/readDir/ldir diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/linked b/tvix/eval/src/tests/nix_tests/readDir/linked index c503f86a0cf7..c503f86a0cf7 120000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/readDir/linked +++ b/tvix/eval/src/tests/nix_tests/readDir/linked diff --git a/tvix/eval/src/tests/one_offs.rs b/tvix/eval/src/tests/one_offs.rs index 565d1dd48f77..49ed713fc96c 100644 --- a/tvix/eval/src/tests/one_offs.rs +++ b/tvix/eval/src/tests/one_offs.rs @@ -5,8 +5,9 @@ fn test_source_builtin() { // Test an evaluation with a source-only builtin. The test ensures // that the artificially constructed thunking is correct. - let mut eval = Evaluation::new_impure(); - eval.src_builtins.push(("testSourceBuiltin", "42")); + let eval = Evaluation::builder_pure() + .add_src_builtin("testSourceBuiltin", "42") + .build(); let result = eval.evaluate("builtins.testSourceBuiltin", None); assert!( @@ -25,7 +26,9 @@ fn test_source_builtin() { #[test] fn skip_broken_bytecode() { - let result = Evaluation::new_pure().evaluate(/* code = */ "x", None); + let result = Evaluation::builder_pure() + .build() + .evaluate(/* code = */ "x", None); assert_eq!(result.errors.len(), 1); diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split.exp new file mode 100644 index 000000000000..eb2117a0ce56 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split.exp @@ -0,0 +1 @@ +[ [ "" [ "a" ] "c" ] [ "" [ "a" ] "b" [ "c" ] "" ] [ "" [ "a" null ] "b" [ null "c" ] "" ] [ " " [ "FOO" ] " " ] [ "" [ "abc" ] "" [ "" ] "" ] [ "" [ "abc" ] "" [ "" ] "" ] [ "" [ ] "" ] ] diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split.nix new file mode 100644 index 000000000000..95305040dce2 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split.nix @@ -0,0 +1,10 @@ +[ + (builtins.split "(a)b" "abc") + (builtins.split "([ac])" "abc") + (builtins.split "(a)|(c)" "abc") + (builtins.split "([[:upper:]]+)" " FOO ") + + (builtins.split "(.*)" "abc") + (builtins.split "([abc]*)" "abc") + (builtins.split ".*" "") +] diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp index ca00e3c049d6..5f97039f94d1 100644 --- a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp @@ -1 +1 @@ -[ false false false false ] +[ false false false false false ] diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix index 8ae5e48e9737..56fd1deb7b1b 100644 --- a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix @@ -10,5 +10,8 @@ map (e: (builtins.tryEval (builtins.toJSON e)).success) [ x = 32; y = builtins.throw "second argument"; } + { + __toString = _: builtins.throw "__toString a"; + } # FIXME(raitobezarius): we would like to test coercions, i.e. `toFile` and `derivation` containing throwables. ] diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-path-exists-child-of-file.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-path-exists-child-of-file.exp new file mode 100644 index 000000000000..c508d5366f70 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-path-exists-child-of-file.exp @@ -0,0 +1 @@ +false diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-path-exists-child-of-file.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-path-exists-child-of-file.nix new file mode 100644 index 000000000000..c756bff755ba --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-path-exists-child-of-file.nix @@ -0,0 +1 @@ +builtins.pathExists ("/dev/null/.") diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-readDir.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-readDir.exp deleted file mode 100644 index bf8d2c14ea4f..000000000000 --- a/tvix/eval/src/tests/tvix_tests/eval-okay-readDir.exp +++ /dev/null @@ -1 +0,0 @@ -{ bar = "regular"; foo = "directory"; } diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-readDir.nix.disabled b/tvix/eval/src/tests/tvix_tests/eval-okay-readDir.nix.disabled deleted file mode 100644 index a7ec9292aae2..000000000000 --- a/tvix/eval/src/tests/tvix_tests/eval-okay-readDir.nix.disabled +++ /dev/null @@ -1 +0,0 @@ -builtins.readDir ./readDir diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-toxml-empty.exp.xml b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml-empty.exp.xml new file mode 100644 index 000000000000..468972b2f819 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml-empty.exp.xml @@ -0,0 +1,5 @@ +<?xml version='1.0' encoding='utf-8'?> +<expr> + <attrs> + </attrs> +</expr> diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-toxml-empty.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml-empty.nix new file mode 100644 index 000000000000..ffcd4415b08f --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml-empty.nix @@ -0,0 +1 @@ +{ } diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-toxml.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml.exp new file mode 100644 index 000000000000..9ae16de526f1 --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml.exp @@ -0,0 +1 @@ +"<?xml version='1.0' encoding='utf-8'?>\n<expr>\n <attrs>\n <attr name=\"&-{\">\n <string value=\";&"\" />\n </attr>\n <attr name=\"a\">\n <string value=\"s\" />\n </attr>\n </attrs>\n</expr>\n" diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-toxml.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml.nix new file mode 100644 index 000000000000..7d074048ddec --- /dev/null +++ b/tvix/eval/src/tests/tvix_tests/eval-okay-toxml.nix @@ -0,0 +1,2 @@ +# Check some corner cases regarding escaping. +builtins.toXML { a = "s"; "&-{" = ";&\""; } diff --git a/tvix/eval/src/tests/tvix_tests/readDir/bar b/tvix/eval/src/tests/tvix_tests/readDir/bar deleted file mode 100644 index e69de29bb2d1..000000000000 --- a/tvix/eval/src/tests/tvix_tests/readDir/bar +++ /dev/null diff --git a/tvix/eval/src/tests/tvix_tests/readDir/foo/.keep b/tvix/eval/src/tests/tvix_tests/readDir/foo/.keep deleted file mode 100644 index e69de29bb2d1..000000000000 --- a/tvix/eval/src/tests/tvix_tests/readDir/foo/.keep +++ /dev/null |