From 850e50cd3f9ce83e4e2d73e48a84f496c27b31b3 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Tue, 23 Jan 2024 16:19:01 +0200 Subject: feat(tvix/eval/tests): compare .xml outputs In case a .exp.xml file is provided alongside the test, compare its output with the desired state. Also, add some function .exp.xml that were presumably moved out of the way back to src/tests/nix_tests, as they now produce the correct XML output. Change-Id: Ibd8123f3e6ed7bae3a44407d2284a2b2c8ce9a28 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10687 Reviewed-by: sterni Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: tazjin --- tvix/eval/src/tests/mod.rs | 49 +-- .../src/tests/nix_tests/eval-okay-closure.exp.xml | 343 +++++++++++++++++++++ .../tests/nix_tests/eval-okay-functionargs.exp.xml | 15 + .../src/tests/nix_tests/eval-okay-functionargs.nix | 80 +++++ .../notyetpassing/eval-okay-closure.exp.xml | 343 --------------------- .../notyetpassing/eval-okay-functionargs.exp.xml | 15 - .../notyetpassing/eval-okay-functionargs.nix | 80 ----- 7 files changed, 467 insertions(+), 458 deletions(-) create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml create mode 100644 tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml delete mode 100644 tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix (limited to 'tvix/eval') diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs index 856707119b..5a7708e298 100644 --- a/tvix/eval/src/tests/mod.rs +++ b/tvix/eval/src/tests/mod.rs @@ -48,17 +48,9 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { "nix", "test files always end in .nix" ); - let exp_path = code_path.with_extension("exp"); - let exp_xml_path = code_path.with_extension("exp.xml"); let code = std::fs::read_to_string(&code_path).expect("should be able to read test code"); - if exp_xml_path.exists() { - // We can't test them at the moment because we don't have XML output yet. - // Checking for success / failure only is a bit disingenious. - return; - } - let mut eval = crate::Evaluation::new_impure(); eval.strict = true; eval.builtins.extend(mock_builtins::builtins()); @@ -79,37 +71,54 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { if !expect_success && failed { return; } + // !expect_success can also mean the output differs, so don't panic if the + // evaluation didn't fail. let value = result.value.unwrap(); let result_str = value.to_string(); - if let Ok(exp) = std::fs::read_to_string(exp_path) { + let exp_path = code_path.with_extension("exp"); + if exp_path.exists() { + // If there's an .exp file provided alongside, compare it with the + // output of the NixValue .to_string() method. + let exp_str = std::fs::read_to_string(&exp_path).expect("unable to read .exp file"); + if expect_success { assert_eq!( result_str, - exp.trim(), + exp_str.trim(), "{}: result value representation (left) must match expectation (right)", code_path.display() ); } else { assert_ne!( result_str, - exp.trim(), + exp_str.trim(), "{}: test passed unexpectedly! consider moving it out of notyetpassing", code_path.display() ); + + // Early return here, we don't compare .xml outputs if this is a ! + // expect_success test. + return; } - } else if expect_success { - panic!( - "{}: should be able to read test expectation", + } + + let exp_xml_path = code_path.with_extension("exp.xml"); + if exp_xml_path.exists() { + // If there's an XML file provided alongside, compare it with the + // output produced when serializing the Value as XML. + let exp_xml_str = std::fs::read_to_string(exp_xml_path).expect("unable to read .xml file"); + + let mut xml_actual_buf = Vec::new(); + crate::builtins::value_to_xml(&mut xml_actual_buf, &value).expect("value_to_xml failed"); + + assert_eq!( + String::from_utf8(xml_actual_buf).expect("to_xml produced invalid utf-8"), + exp_xml_str, + "{}: result value representation (left) must match expectation (right)", code_path.display() ); - } else { - panic!( - "{}: test should have failed, but succeeded with output {}", - code_path.display(), - result_str - ); } } diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml b/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml new file mode 100644 index 0000000000..dffc03a998 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-closure.exp.xml @@ -0,0 +1,343 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml new file mode 100644 index 0000000000..651f54c363 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.exp.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix new file mode 100644 index 0000000000..68dca62ee1 --- /dev/null +++ b/tvix/eval/src/tests/nix_tests/eval-okay-functionargs.nix @@ -0,0 +1,80 @@ +let + + stdenvFun = { }: { name = "stdenv"; }; + stdenv2Fun = { }: { name = "stdenv2"; }; + fetchurlFun = { stdenv }: assert stdenv.name == "stdenv"; { name = "fetchurl"; }; + atermFun = { stdenv, fetchurl }: { name = "aterm-${stdenv.name}"; }; + aterm2Fun = { stdenv, fetchurl }: { name = "aterm2-${stdenv.name}"; }; + nixFun = { stdenv, fetchurl, aterm }: { name = "nix-${stdenv.name}-${aterm.name}"; }; + + mplayerFun = + { stdenv, fetchurl, enableX11 ? false, xorg ? null, enableFoo ? true, foo ? null }: + assert stdenv.name == "stdenv2"; + assert enableX11 -> xorg.libXv.name == "libXv"; + assert enableFoo -> foo != null; + { name = "mplayer-${stdenv.name}.${xorg.libXv.name}-${xorg.libX11.name}"; }; + + makeOverridable = f: origArgs: f origArgs // + { override = newArgs: + makeOverridable f (origArgs // (if builtins.isFunction newArgs then newArgs origArgs else newArgs)); + }; + + callPackage_ = pkgs: f: args: + makeOverridable f ((builtins.intersectAttrs (builtins.functionArgs f) pkgs) // args); + + allPackages = + { overrides ? (pkgs: pkgsPrev: { }) }: + let + callPackage = callPackage_ pkgs; + pkgs = pkgsStd // (overrides pkgs pkgsStd); + pkgsStd = { + inherit pkgs; + stdenv = callPackage stdenvFun { }; + stdenv2 = callPackage stdenv2Fun { }; + fetchurl = callPackage fetchurlFun { }; + aterm = callPackage atermFun { }; + xorg = callPackage xorgFun { }; + mplayer = callPackage mplayerFun { stdenv = pkgs.stdenv2; enableFoo = false; }; + nix = callPackage nixFun { }; + }; + in pkgs; + + libX11Fun = { stdenv, fetchurl }: { name = "libX11"; }; + libX11_2Fun = { stdenv, fetchurl }: { name = "libX11_2"; }; + libXvFun = { stdenv, fetchurl, libX11 }: { name = "libXv"; }; + + xorgFun = + { pkgs }: + let callPackage = callPackage_ (pkgs // pkgs.xorg); in + { + libX11 = callPackage libX11Fun { }; + libXv = callPackage libXvFun { }; + }; + +in + +let + + pkgs = allPackages { }; + + pkgs2 = allPackages { + overrides = pkgs: pkgsPrev: { + stdenv = pkgs.stdenv2; + nix = pkgsPrev.nix.override { aterm = aterm2Fun { inherit (pkgs) stdenv fetchurl; }; }; + xorg = pkgsPrev.xorg // { libX11 = libX11_2Fun { inherit (pkgs) stdenv fetchurl; }; }; + }; + }; + +in + + [ pkgs.stdenv.name + pkgs.fetchurl.name + pkgs.aterm.name + pkgs2.aterm.name + pkgs.xorg.libX11.name + pkgs.xorg.libXv.name + pkgs.mplayer.name + pkgs2.mplayer.name + pkgs.nix.name + pkgs2.nix.name + ] diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml deleted file mode 100644 index dffc03a998..0000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-closure.exp.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml deleted file mode 100644 index 651f54c363..0000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.exp.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix b/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix deleted file mode 100644 index 68dca62ee1..0000000000 --- a/tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-functionargs.nix +++ /dev/null @@ -1,80 +0,0 @@ -let - - stdenvFun = { }: { name = "stdenv"; }; - stdenv2Fun = { }: { name = "stdenv2"; }; - fetchurlFun = { stdenv }: assert stdenv.name == "stdenv"; { name = "fetchurl"; }; - atermFun = { stdenv, fetchurl }: { name = "aterm-${stdenv.name}"; }; - aterm2Fun = { stdenv, fetchurl }: { name = "aterm2-${stdenv.name}"; }; - nixFun = { stdenv, fetchurl, aterm }: { name = "nix-${stdenv.name}-${aterm.name}"; }; - - mplayerFun = - { stdenv, fetchurl, enableX11 ? false, xorg ? null, enableFoo ? true, foo ? null }: - assert stdenv.name == "stdenv2"; - assert enableX11 -> xorg.libXv.name == "libXv"; - assert enableFoo -> foo != null; - { name = "mplayer-${stdenv.name}.${xorg.libXv.name}-${xorg.libX11.name}"; }; - - makeOverridable = f: origArgs: f origArgs // - { override = newArgs: - makeOverridable f (origArgs // (if builtins.isFunction newArgs then newArgs origArgs else newArgs)); - }; - - callPackage_ = pkgs: f: args: - makeOverridable f ((builtins.intersectAttrs (builtins.functionArgs f) pkgs) // args); - - allPackages = - { overrides ? (pkgs: pkgsPrev: { }) }: - let - callPackage = callPackage_ pkgs; - pkgs = pkgsStd // (overrides pkgs pkgsStd); - pkgsStd = { - inherit pkgs; - stdenv = callPackage stdenvFun { }; - stdenv2 = callPackage stdenv2Fun { }; - fetchurl = callPackage fetchurlFun { }; - aterm = callPackage atermFun { }; - xorg = callPackage xorgFun { }; - mplayer = callPackage mplayerFun { stdenv = pkgs.stdenv2; enableFoo = false; }; - nix = callPackage nixFun { }; - }; - in pkgs; - - libX11Fun = { stdenv, fetchurl }: { name = "libX11"; }; - libX11_2Fun = { stdenv, fetchurl }: { name = "libX11_2"; }; - libXvFun = { stdenv, fetchurl, libX11 }: { name = "libXv"; }; - - xorgFun = - { pkgs }: - let callPackage = callPackage_ (pkgs // pkgs.xorg); in - { - libX11 = callPackage libX11Fun { }; - libXv = callPackage libXvFun { }; - }; - -in - -let - - pkgs = allPackages { }; - - pkgs2 = allPackages { - overrides = pkgs: pkgsPrev: { - stdenv = pkgs.stdenv2; - nix = pkgsPrev.nix.override { aterm = aterm2Fun { inherit (pkgs) stdenv fetchurl; }; }; - xorg = pkgsPrev.xorg // { libX11 = libX11_2Fun { inherit (pkgs) stdenv fetchurl; }; }; - }; - }; - -in - - [ pkgs.stdenv.name - pkgs.fetchurl.name - pkgs.aterm.name - pkgs2.aterm.name - pkgs.xorg.libX11.name - pkgs.xorg.libXv.name - pkgs.mplayer.name - pkgs2.mplayer.name - pkgs.nix.name - pkgs2.nix.name - ] -- cgit 1.4.1