From 61a7b97c3927f28ef588196bc2efbd8c6cceb8c4 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 19 Jul 2020 15:57:43 +0100 Subject: test(3p/nix): Enable output comparison for evaluator success tests Enables loading of the expected output of evaluator tests from the corresponding .exp files, and checks that the output matches. This again leaves some tests behind in the disabled folder, but we now have almost the entire suite up and running so I can get around to cleaning up the disabled ones. Other note: Some tests had XML output, despite not being related to XML testing at all - I'm not sure why they chose to do this, but have converted those test outputs to normal Nix instead. We have a separate test suite for JSON & XML serialisation already, which was contributed by andi-. Change-Id: Id7c42c836edfec4c22db9d893e35489f3e6dd559 Reviewed-on: https://cl.tvl.fyi/c/depot/+/1285 Tested-by: BuildkiteCI Reviewed-by: isomer Reviewed-by: glittershark --- third_party/nix/default.nix | 2 + third_party/nix/src/tests/lang/disabled/README.txt | 2 +- .../src/tests/lang/disabled/eval-okay-autoargs.exp | 1 + .../src/tests/lang/disabled/eval-okay-autoargs.nix | 15 + .../nix/src/tests/lang/disabled/eval-okay-sort.nix | 8 + .../src/tests/lang/disabled/eval-okay-toxml.nix | 3 + .../nix/src/tests/lang/disabled/eval-okay-xml.nix | 21 ++ .../nix/src/tests/lang/eval-okay-autoargs.exp | 1 - .../nix/src/tests/lang/eval-okay-autoargs.nix | 15 - .../nix/src/tests/lang/eval-okay-closure.exp | 1 + .../nix/src/tests/lang/eval-okay-closure.exp.xml | 343 --------------------- third_party/nix/src/tests/lang/eval-okay-eq.exp | 1 + .../nix/src/tests/lang/eval-okay-eq.exp.disabled | 1 - .../nix/src/tests/lang/eval-okay-functionargs.exp | 1 + .../src/tests/lang/eval-okay-functionargs.exp.xml | 15 - .../nix/src/tests/lang/eval-okay-getenv.nix | 2 +- third_party/nix/src/tests/lang/eval-okay-sort.nix | 8 - third_party/nix/src/tests/lang/eval-okay-toxml.nix | 3 - third_party/nix/src/tests/lang/eval-okay-xml.exp | 52 ++++ .../nix/src/tests/lang/eval-okay-xml.exp.xml | 52 ---- third_party/nix/src/tests/lang/eval-okay-xml.nix | 21 -- third_party/nix/src/tests/language-tests.cc | 33 +- 22 files changed, 137 insertions(+), 464 deletions(-) create mode 100644 third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp create mode 100644 third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix create mode 100644 third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix create mode 100644 third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix create mode 100644 third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix delete mode 100644 third_party/nix/src/tests/lang/eval-okay-autoargs.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-autoargs.nix create mode 100644 third_party/nix/src/tests/lang/eval-okay-closure.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-closure.exp.xml create mode 100644 third_party/nix/src/tests/lang/eval-okay-eq.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled create mode 100644 third_party/nix/src/tests/lang/eval-okay-functionargs.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml delete mode 100644 third_party/nix/src/tests/lang/eval-okay-sort.nix delete mode 100644 third_party/nix/src/tests/lang/eval-okay-toxml.nix create mode 100644 third_party/nix/src/tests/lang/eval-okay-xml.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-xml.exp.xml delete mode 100644 third_party/nix/src/tests/lang/eval-okay-xml.nix diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix index 54e77ec50d11..cceb6b4f1bd5 100644 --- a/third_party/nix/default.nix +++ b/third_party/nix/default.nix @@ -89,6 +89,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation { installCheckPhase = '' export NIX_DATA_DIR=$out/share + export NIX_TEST_VAR=foo # this is required by a language test make test ''; @@ -124,6 +125,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation { shellHook = '' export NIX_DATA_DIR="${toString depotPath}/third_party" + export NIX_TEST_VAR=foo ''; # TODO(tazjin): integration test setup? diff --git a/third_party/nix/src/tests/lang/disabled/README.txt b/third_party/nix/src/tests/lang/disabled/README.txt index 3dbcbf4f0dd9..50225deb63a2 100644 --- a/third_party/nix/src/tests/lang/disabled/README.txt +++ b/third_party/nix/src/tests/lang/disabled/README.txt @@ -1,4 +1,4 @@ -These tests are disabeld primarily because the DummyStore used for +These tests are disabled primarily because the DummyStore used for tests does not interact with real files on disk at the moment, but the tests expect it to. diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp new file mode 100644 index 000000000000..7a8391786a09 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp @@ -0,0 +1 @@ +"xyzzy!xyzzy!foobar" diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix new file mode 100644 index 000000000000..815f51b1d67a --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix @@ -0,0 +1,15 @@ +let + + foobar = "foobar"; + +in + +{ xyzzy2 ? xyzzy # mutually recursive args +, xyzzy ? "blaat" # will be overridden by --argstr +, fb ? foobar +, lib # will be set by --arg +}: + +{ + result = lib.concat [xyzzy xyzzy2 fb]; +} diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix new file mode 100644 index 000000000000..8299c3a4a3aa --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix @@ -0,0 +1,8 @@ +with builtins; + +[ (sort lessThan [ 483 249 526 147 42 77 ]) + (sort (x: y: y < x) [ 483 249 526 147 42 77 ]) + (sort lessThan [ "foo" "bar" "xyzzy" "fnord" ]) + (sort (x: y: x.key < y.key) + [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) +] diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix new file mode 100644 index 000000000000..068c97a6c1b3 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix @@ -0,0 +1,3 @@ +# Make sure the expected XML output is produced; in particular, make sure it +# doesn't contain source location information. +builtins.toXML { a = "s"; } diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix new file mode 100644 index 000000000000..9ee9f8a0b4f5 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix @@ -0,0 +1,21 @@ +rec { + + x = 123; + + y = 567.890; + + a = "foo"; + + b = "bar"; + + c = "foo" + "bar"; + + f = {z, x, y}: if y then x else z; + + id = x: x; + + at = args@{x, y, z}: x; + + ellipsis = {x, y, z, ...}: x; + +} diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp b/third_party/nix/src/tests/lang/eval-okay-autoargs.exp deleted file mode 100644 index 7a8391786a09..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp +++ /dev/null @@ -1 +0,0 @@ -"xyzzy!xyzzy!foobar" diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix b/third_party/nix/src/tests/lang/eval-okay-autoargs.nix deleted file mode 100644 index 815f51b1d67a..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix +++ /dev/null @@ -1,15 +0,0 @@ -let - - foobar = "foobar"; - -in - -{ xyzzy2 ? xyzzy # mutually recursive args -, xyzzy ? "blaat" # will be overridden by --argstr -, fb ? foobar -, lib # will be set by --arg -}: - -{ - result = lib.concat [xyzzy xyzzy2 fb]; -} diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp b/third_party/nix/src/tests/lang/eval-okay-closure.exp new file mode 100644 index 000000000000..e7dbf978160d --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-closure.exp @@ -0,0 +1 @@ +[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ] diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml b/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml deleted file mode 100644 index dffc03a99891..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp b/third_party/nix/src/tests/lang/eval-okay-eq.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled b/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled deleted file mode 100644 index 2015847b65e7..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled +++ /dev/null @@ -1 +0,0 @@ -Bool(True) diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp new file mode 100644 index 000000000000..c1c9f8ffaf69 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp @@ -0,0 +1 @@ +[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml deleted file mode 100644 index 651f54c36341..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.nix b/third_party/nix/src/tests/lang/eval-okay-getenv.nix index 4cfec5f553d9..ea8bb9f0a629 100644 --- a/third_party/nix/src/tests/lang/eval-okay-getenv.nix +++ b/third_party/nix/src/tests/lang/eval-okay-getenv.nix @@ -1 +1 @@ -builtins.getEnv "TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") +builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.nix b/third_party/nix/src/tests/lang/eval-okay-sort.nix deleted file mode 100644 index 8299c3a4a3aa..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-sort.nix +++ /dev/null @@ -1,8 +0,0 @@ -with builtins; - -[ (sort lessThan [ 483 249 526 147 42 77 ]) - (sort (x: y: y < x) [ 483 249 526 147 42 77 ]) - (sort lessThan [ "foo" "bar" "xyzzy" "fnord" ]) - (sort (x: y: x.key < y.key) - [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) -] diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml.nix b/third_party/nix/src/tests/lang/eval-okay-toxml.nix deleted file mode 100644 index 068c97a6c1b3..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-toxml.nix +++ /dev/null @@ -1,3 +0,0 @@ -# Make sure the expected XML output is produced; in particular, make sure it -# doesn't contain source location information. -builtins.toXML { a = "s"; } diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.exp b/third_party/nix/src/tests/lang/eval-okay-xml.exp new file mode 100644 index 000000000000..92b75e0b8b17 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-xml.exp @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml b/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml deleted file mode 100644 index 92b75e0b8b17..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.nix b/third_party/nix/src/tests/lang/eval-okay-xml.nix deleted file mode 100644 index 9ee9f8a0b4f5..000000000000 --- a/third_party/nix/src/tests/lang/eval-okay-xml.nix +++ /dev/null @@ -1,21 +0,0 @@ -rec { - - x = 123; - - y = 567.890; - - a = "foo"; - - b = "bar"; - - c = "foo" + "bar"; - - f = {z, x, y}: if y then x else z; - - id = x: x; - - at = args@{x, y, z}: x; - - ellipsis = {x, y, z, ...}: x; - -} diff --git a/third_party/nix/src/tests/language-tests.cc b/third_party/nix/src/tests/language-tests.cc index daa2fa233e19..cac5aee5841c 100644 --- a/third_party/nix/src/tests/language-tests.cc +++ b/third_party/nix/src/tests/language-tests.cc @@ -23,9 +23,12 @@ #include #include +#include #include +#include #include #include +#include #include #include @@ -86,6 +89,20 @@ std::string TestNameFor( return name; } +// Load the expected output of a given test as a string. +std::string ExpectedOutputFor(absl::string_view stem) { + std::filesystem::path path( + absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp")); + + EXPECT_TRUE(std::filesystem::exists(path)) + << stem << ": expected output file should exist"; + + std::ifstream input(path); + std::stringstream buffer; + buffer << input.rdbuf(); + return std::string(absl::StripTrailingAsciiWhitespace(buffer.str())); +} + } // namespace using nix::tests::DummyStore; @@ -189,7 +206,7 @@ INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest, class EvalSuccessTest : public testing::TestWithParam {}; -// Test pattern for files that should fail to evaluate. +// Test pattern for files that should evaluate successfully. TEST_P(EvalSuccessTest, Fails) { std::shared_ptr store = std::make_shared(); EvalState state({}, ref(store)); @@ -200,8 +217,18 @@ TEST_P(EvalSuccessTest, Fails) { << path.stem().string() << ": should parse successfully"; Value result; - state.eval(expr, result); - state.forceValue(result); + + EXPECT_NO_THROW({ + state.eval(expr, result); + state.forceValueDeep(result); + }) << path.stem().string() + << ": should evaluate successfully"; + + auto expected = ExpectedOutputFor(path.stem().string()); + std::ostringstream value_str; + value_str << result; + + EXPECT_EQ(expected, value_str.str()) << "evaluator output should match"; } INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest, -- cgit 1.4.1