diff options
author | Ryan Lahfa <tvl@lahfa.xyz> | 2024-01-14T00·41+0100 |
---|---|---|
committer | clbot <clbot@tvl.fyi> | 2024-01-17T17·31+0000 |
commit | f71bb351d29f58935e6002615cd94d6e6259bf26 (patch) | |
tree | 168311dd206ec4524db0105d8109e0ba4d698ec7 /tvix/glue/src/tests/mod.rs | |
parent | 75cc52ddb136e66b1a79117425fb35f80dcecc07 (diff) |
feat(tvix/glue): introduce test suite for context strings r/7404
This is an additional test suite on the top of the Nix ones for context strings matters. It already smoked out multiple mistakes and potential bugs and non-deterministic result from the evaluator. It uses a similar technology as the one in the tvix-eval albeit we instantiate a fully fledged evaluator with in-memory store. We copy the files instead of symlinking them because crates are built in isolation, so symlinks cannot work. Change-Id: I63ae225ce4f83c6e2c8ccd60d779c2f8eb9d08fb Reviewed-on: https://cl.tvl.fyi/c/depot/+/10619 Autosubmit: raitobezarius <tvl@lahfa.xyz> Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de>
Diffstat (limited to 'tvix/glue/src/tests/mod.rs')
-rw-r--r-- | tvix/glue/src/tests/mod.rs | 140 |
1 files changed, 140 insertions, 0 deletions
diff --git a/tvix/glue/src/tests/mod.rs b/tvix/glue/src/tests/mod.rs new file mode 100644 index 000000000000..9ae17c219ac9 --- /dev/null +++ b/tvix/glue/src/tests/mod.rs @@ -0,0 +1,140 @@ +use std::{cell::RefCell, rc::Rc, sync::Arc}; + +use pretty_assertions::assert_eq; +use std::path::PathBuf; +use tvix_castore::{ + blobservice::{BlobService, MemoryBlobService}, + directoryservice::{DirectoryService, MemoryDirectoryService}, +}; +use tvix_eval::Value; +use tvix_store::pathinfoservice::{MemoryPathInfoService, PathInfoService}; + +use rstest::rstest; + +use crate::{ + builtins::add_derivation_builtins, known_paths::KnownPaths, tvix_store_io::TvixStoreIO, +}; + +fn eval_test(code_path: PathBuf, expect_success: bool) { + assert_eq!( + code_path.extension().unwrap(), + "nix", + "test files always end in .nix" + ); + let exp_path = code_path.with_extension("exp"); + let exp_xml_path = code_path.with_extension("exp.xml"); + + let code = std::fs::read_to_string(&code_path).expect("should be able to read test code"); + + if exp_xml_path.exists() { + // We can't test them at the moment because we don't have XML output yet. + // Checking for success / failure only is a bit disingenious. + return; + } + + let mut eval = tvix_eval::Evaluation::new_impure(); + + let blob_service = Arc::new(MemoryBlobService::default()) as Arc<dyn BlobService>; + let directory_service = + Arc::new(MemoryDirectoryService::default()) as Arc<dyn DirectoryService>; + let path_info_service = Box::new(MemoryPathInfoService::new( + blob_service.clone(), + directory_service.clone(), + )) as Box<dyn PathInfoService>; + let runtime = tokio::runtime::Runtime::new().unwrap(); + + eval.io_handle = Box::new(TvixStoreIO::new( + blob_service, + directory_service, + path_info_service, + runtime.handle().clone(), + )); + + let known_paths: Rc<RefCell<KnownPaths>> = Default::default(); + + eval.strict = true; + add_derivation_builtins(&mut eval, known_paths.clone()); + + let result = eval.evaluate(code, Some(code_path.clone())); + let failed = match result.value { + Some(Value::Catchable(_)) => true, + _ => !result.errors.is_empty(), + }; + if expect_success && failed { + panic!( + "{}: evaluation of eval-okay test should succeed, but failed with {:?}", + code_path.display(), + result.errors, + ); + } + + if !expect_success && failed { + return; + } + + let value = result.value.unwrap(); + let result_str = value.to_string(); + + if let Ok(exp) = std::fs::read_to_string(exp_path) { + if expect_success { + assert_eq!( + result_str, + exp.trim(), + "{}: result value representation (left) must match expectation (right)", + code_path.display() + ); + } else { + assert_ne!( + result_str, + exp.trim(), + "{}: test passed unexpectedly! consider moving it out of notyetpassing", + code_path.display() + ); + } + } else if expect_success { + panic!( + "{}: should be able to read test expectation", + code_path.display() + ); + } else { + panic!( + "{}: test should have failed, but succeeded with output {}", + code_path.display(), + result_str + ); + } +} + +// eval-okay-* tests contain a snippet of Nix code, and an expectation +// of the produced string output of the evaluator. +// +// These evaluations are always supposed to succeed, i.e. all snippets +// are guaranteed to be valid Nix code. +#[rstest] +fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) { + eval_test(code_path, true) +} + +// eval-okay-* tests from the original Nix test suite. +#[cfg(feature = "nix_tests")] +#[rstest] +fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) { + eval_test(code_path, true) +} + +// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix +// +// Eventually there will be none of these left, and this function +// will disappear :) Until then, to run these tests, use `cargo test +// --features expected_failures`. +// +// Please don't submit failing tests unless they're in +// notyetpassing; this makes the test suite much more useful for +// regression testing, since there should always be zero non-ignored +// failing tests. +#[rstest] +fn nix_eval_okay_currently_failing( + #[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf, +) { + eval_test(code_path, false) +} |