diff options
author | Vincent Ambo <mail@tazj.in> | 2020-07-19T14·57+0100 |
---|---|---|
committer | tazjin <mail@tazj.in> | 2020-07-19T16·33+0000 |
commit | 61a7b97c3927f28ef588196bc2efbd8c6cceb8c4 (patch) | |
tree | e1d5e5c273f1c0fcfce9df4044c346abf78ba59e /third_party/nix/src/tests | |
parent | 97e649922542ef66c9d80cb1b41817ed07c2c5a6 (diff) |
test(3p/nix): Enable output comparison for evaluator success tests r/1398
Enables loading of the expected output of evaluator tests from the corresponding .exp files, and checks that the output matches. This again leaves some tests behind in the disabled folder, but we now have almost the entire suite up and running so I can get around to cleaning up the disabled ones. Other note: Some tests had XML output, despite not being related to XML testing at all - I'm not sure why they chose to do this, but have converted those test outputs to normal Nix instead. We have a separate test suite for JSON & XML serialisation already, which was contributed by andi-. Change-Id: Id7c42c836edfec4c22db9d893e35489f3e6dd559 Reviewed-on: https://cl.tvl.fyi/c/depot/+/1285 Tested-by: BuildkiteCI Reviewed-by: isomer <isomer@tvl.fyi> Reviewed-by: glittershark <grfn@gws.fyi>
Diffstat (limited to 'third_party/nix/src/tests')
15 files changed, 35 insertions, 364 deletions
diff --git a/third_party/nix/src/tests/lang/disabled/README.txt b/third_party/nix/src/tests/lang/disabled/README.txt index 3dbcbf4f0d..50225deb63 100644 --- a/third_party/nix/src/tests/lang/disabled/README.txt +++ b/third_party/nix/src/tests/lang/disabled/README.txt @@ -1,4 +1,4 @@ -These tests are disabeld primarily because the DummyStore used for +These tests are disabled primarily because the DummyStore used for tests does not interact with real files on disk at the moment, but the tests expect it to. diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp index 7a8391786a..7a8391786a 100644 --- a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix index 815f51b1d6..815f51b1d6 100644 --- a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix index 8299c3a4a3..8299c3a4a3 100644 --- a/third_party/nix/src/tests/lang/eval-okay-sort.nix +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix index 068c97a6c1..068c97a6c1 100644 --- a/third_party/nix/src/tests/lang/eval-okay-toxml.nix +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix index 9ee9f8a0b4..9ee9f8a0b4 100644 --- a/third_party/nix/src/tests/lang/eval-okay-xml.nix +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp b/third_party/nix/src/tests/lang/eval-okay-closure.exp new file mode 100644 index 0000000000..e7dbf97816 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-closure.exp @@ -0,0 +1 @@ +[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ] diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml b/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml deleted file mode 100644 index dffc03a998..0000000000 --- a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml +++ /dev/null @@ -1,343 +0,0 @@ -<?xml version='1.0' encoding='utf-8'?> -<expr> - <list> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-13" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-12" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-11" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-9" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-8" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-7" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-5" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-4" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="-3" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="-1" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="0" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="1" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="2" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="4" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="5" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="6" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="8" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="9" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="10" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="13" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="14" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="15" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="17" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="18" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="19" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="22" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="23" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="26" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="27" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="28" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="31" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="32" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="35" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="36" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="40" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="41" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="44" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="45" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="49" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="53" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="54" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="58" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="62" /> - </attr> - </attrs> - <attrs> - <attr name="foo"> - <bool value="true" /> - </attr> - <attr name="key"> - <int value="67" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="71" /> - </attr> - </attrs> - <attrs> - <attr name="key"> - <int value="80" /> - </attr> - </attrs> - </list> -</expr> diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp b/third_party/nix/src/tests/lang/eval-okay-eq.exp new file mode 100644 index 0000000000..27ba77ddaf --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled b/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled deleted file mode 100644 index 2015847b65..0000000000 --- a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled +++ /dev/null @@ -1 +0,0 @@ -Bool(True) diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp new file mode 100644 index 0000000000..c1c9f8ffaf --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp @@ -0,0 +1 @@ +[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml deleted file mode 100644 index 651f54c363..0000000000 --- a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml +++ /dev/null @@ -1,15 +0,0 @@ -<?xml version='1.0' encoding='utf-8'?> -<expr> - <list> - <string value="stdenv" /> - <string value="fetchurl" /> - <string value="aterm-stdenv" /> - <string value="aterm-stdenv2" /> - <string value="libX11" /> - <string value="libXv" /> - <string value="mplayer-stdenv2.libXv-libX11" /> - <string value="mplayer-stdenv2.libXv-libX11_2" /> - <string value="nix-stdenv-aterm-stdenv" /> - <string value="nix-stdenv2-aterm2-stdenv2" /> - </list> -</expr> diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.nix b/third_party/nix/src/tests/lang/eval-okay-getenv.nix index 4cfec5f553..ea8bb9f0a6 100644 --- a/third_party/nix/src/tests/lang/eval-okay-getenv.nix +++ b/third_party/nix/src/tests/lang/eval-okay-getenv.nix @@ -1 +1 @@ -builtins.getEnv "TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") +builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml b/third_party/nix/src/tests/lang/eval-okay-xml.exp index 92b75e0b8b..92b75e0b8b 100644 --- a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml +++ b/third_party/nix/src/tests/lang/eval-okay-xml.exp diff --git a/third_party/nix/src/tests/language-tests.cc b/third_party/nix/src/tests/language-tests.cc index daa2fa233e..cac5aee584 100644 --- a/third_party/nix/src/tests/language-tests.cc +++ b/third_party/nix/src/tests/language-tests.cc @@ -23,9 +23,12 @@ #include <algorithm> #include <filesystem> +#include <fstream> #include <iostream> +#include <iterator> #include <memory> #include <optional> +#include <sstream> #include <string> #include <absl/strings/ascii.h> @@ -86,6 +89,20 @@ std::string TestNameFor( return name; } +// Load the expected output of a given test as a string. +std::string ExpectedOutputFor(absl::string_view stem) { + std::filesystem::path path( + absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp")); + + EXPECT_TRUE(std::filesystem::exists(path)) + << stem << ": expected output file should exist"; + + std::ifstream input(path); + std::stringstream buffer; + buffer << input.rdbuf(); + return std::string(absl::StripTrailingAsciiWhitespace(buffer.str())); +} + } // namespace using nix::tests::DummyStore; @@ -189,7 +206,7 @@ INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest, class EvalSuccessTest : public testing::TestWithParam<std::filesystem::path> {}; -// Test pattern for files that should fail to evaluate. +// Test pattern for files that should evaluate successfully. TEST_P(EvalSuccessTest, Fails) { std::shared_ptr<Store> store = std::make_shared<DummyStore>(); EvalState state({}, ref<Store>(store)); @@ -200,8 +217,18 @@ TEST_P(EvalSuccessTest, Fails) { << path.stem().string() << ": should parse successfully"; Value result; - state.eval(expr, result); - state.forceValue(result); + + EXPECT_NO_THROW({ + state.eval(expr, result); + state.forceValueDeep(result); + }) << path.stem().string() + << ": should evaluate successfully"; + + auto expected = ExpectedOutputFor(path.stem().string()); + std::ostringstream value_str; + value_str << result; + + EXPECT_EQ(expected, value_str.str()) << "evaluator output should match"; } INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest, |