test(3p/nix): Enable output comparison for evaluator success tests r/1398

Enables loading of the expected output of evaluator tests from the
corresponding .exp files, and checks that the output matches.

This again leaves some tests behind in the disabled folder, but we now
have almost the entire suite up and running so I can get around to
cleaning up the disabled ones.

Other note: Some tests had XML output, despite not being related to
XML testing at all - I'm not sure why they chose to do this, but have
converted those test outputs to normal Nix instead.

We have a separate test suite for JSON & XML serialisation already,
which was contributed by andi-.

Change-Id: Id7c42c836edfec4c22db9d893e35489f3e6dd559
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1285
Tested-by: BuildkiteCI
Reviewed-by: isomer <isomer@tvl.fyi>
Reviewed-by: glittershark <grfn@gws.fyi>

16 files changed, 37 insertions, 364 deletions

diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix
index 54e77ec50d..cceb6b4f1b 100644
--- a/third_party/nix/default.nix
+++ b/third_party/nix/default.nix
@@ -89,6 +89,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation {
 
   installCheckPhase = ''
     export NIX_DATA_DIR=$out/share
+    export NIX_TEST_VAR=foo # this is required by a language test
     make test
   '';
 
@@ -124,6 +125,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation {
 
   shellHook = ''
     export NIX_DATA_DIR="${toString depotPath}/third_party"
+    export NIX_TEST_VAR=foo
   '';
 
   # TODO(tazjin): integration test setup?
diff --git a/third_party/nix/src/tests/lang/disabled/README.txt b/third_party/nix/src/tests/lang/disabled/README.txt
index 3dbcbf4f0d..50225deb63 100644
--- a/third_party/nix/src/tests/lang/disabled/README.txt
+++ b/third_party/nix/src/tests/lang/disabled/README.txt
@@ -1,4 +1,4 @@
-These tests are disabeld primarily because the DummyStore used for
+These tests are disabled primarily because the DummyStore used for
 tests does not interact with real files on disk at the moment, but the
 tests expect it to.
 
diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp
index 7a8391786a..7a8391786a 100644
--- a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp
diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix
index 815f51b1d6..815f51b1d6 100644
--- a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix
index 8299c3a4a3..8299c3a4a3 100644
--- a/third_party/nix/src/tests/lang/eval-okay-sort.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix
index 068c97a6c1..068c97a6c1 100644
--- a/third_party/nix/src/tests/lang/eval-okay-toxml.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix
index 9ee9f8a0b4..9ee9f8a0b4 100644
--- a/third_party/nix/src/tests/lang/eval-okay-xml.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp b/third_party/nix/src/tests/lang/eval-okay-closure.exp
new file mode 100644
index 0000000000..e7dbf97816
--- /dev/null
+++ b/third_party/nix/src/tests/lang/eval-okay-closure.exp
@@ -0,0 +1 @@
+[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ]
diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml b/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml
deleted file mode 100644
index dffc03a998..0000000000
--- a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml
+++ /dev/null
@@ -1,343 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<expr>
-  <list>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-13" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-12" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-11" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-9" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-8" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-7" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-5" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-4" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-3" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="-1" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="0" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="1" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="2" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="4" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="5" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="6" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="8" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="9" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="10" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="13" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="14" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="15" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="17" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="18" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="19" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="22" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="23" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="26" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="27" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="28" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="31" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="32" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="35" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="36" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="40" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="41" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="44" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="45" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="49" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="53" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="54" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="58" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="62" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="67" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="71" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="80" />
-      </attr>
-    </attrs>
-  </list>
-</expr>
diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp b/third_party/nix/src/tests/lang/eval-okay-eq.exp
new file mode 100644
index 0000000000..27ba77ddaf
--- /dev/null
+++ b/third_party/nix/src/tests/lang/eval-okay-eq.exp
@@ -0,0 +1 @@
+true
diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled b/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled
deleted file mode 100644
index 2015847b65..0000000000
--- a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled
+++ /dev/null
@@ -1 +0,0 @@
-Bool(True)
diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp
new file mode 100644
index 0000000000..c1c9f8ffaf
--- /dev/null
+++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp
@@ -0,0 +1 @@
+[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ]
diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml
deleted file mode 100644
index 651f54c363..0000000000
--- a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<expr>
-  <list>
-    <string value="stdenv" />
-    <string value="fetchurl" />
-    <string value="aterm-stdenv" />
-    <string value="aterm-stdenv2" />
-    <string value="libX11" />
-    <string value="libXv" />
-    <string value="mplayer-stdenv2.libXv-libX11" />
-    <string value="mplayer-stdenv2.libXv-libX11_2" />
-    <string value="nix-stdenv-aterm-stdenv" />
-    <string value="nix-stdenv2-aterm2-stdenv2" />
-  </list>
-</expr>
diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.nix b/third_party/nix/src/tests/lang/eval-okay-getenv.nix
index 4cfec5f553..ea8bb9f0a6 100644
--- a/third_party/nix/src/tests/lang/eval-okay-getenv.nix
+++ b/third_party/nix/src/tests/lang/eval-okay-getenv.nix
@@ -1 +1 @@
-builtins.getEnv "TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla")
+builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla")
diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml b/third_party/nix/src/tests/lang/eval-okay-xml.exp
index 92b75e0b8b..92b75e0b8b 100644
--- a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml
+++ b/third_party/nix/src/tests/lang/eval-okay-xml.exp
diff --git a/third_party/nix/src/tests/language-tests.cc b/third_party/nix/src/tests/language-tests.cc
index daa2fa233e..cac5aee584 100644
--- a/third_party/nix/src/tests/language-tests.cc
+++ b/third_party/nix/src/tests/language-tests.cc
@@ -23,9 +23,12 @@
 
 #include <algorithm>
 #include <filesystem>
+#include <fstream>
 #include <iostream>
+#include <iterator>
 #include <memory>
 #include <optional>
+#include <sstream>
 #include <string>
 
 #include <absl/strings/ascii.h>
@@ -86,6 +89,20 @@ std::string TestNameFor(
   return name;
 }
 
+// Load the expected output of a given test as a string.
+std::string ExpectedOutputFor(absl::string_view stem) {
+  std::filesystem::path path(
+      absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp"));
+
+  EXPECT_TRUE(std::filesystem::exists(path))
+      << stem << ": expected output file should exist";
+
+  std::ifstream input(path);
+  std::stringstream buffer;
+  buffer << input.rdbuf();
+  return std::string(absl::StripTrailingAsciiWhitespace(buffer.str()));
+}
+
 }  // namespace
 
 using nix::tests::DummyStore;
@@ -189,7 +206,7 @@ INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest,
 
 class EvalSuccessTest : public testing::TestWithParam<std::filesystem::path> {};
 
-// Test pattern for files that should fail to evaluate.
+// Test pattern for files that should evaluate successfully.
 TEST_P(EvalSuccessTest, Fails) {
   std::shared_ptr<Store> store = std::make_shared<DummyStore>();
   EvalState state({}, ref<Store>(store));
@@ -200,8 +217,18 @@ TEST_P(EvalSuccessTest, Fails) {
       << path.stem().string() << ": should parse successfully";
 
   Value result;
-  state.eval(expr, result);
-  state.forceValue(result);
+
+  EXPECT_NO_THROW({
+    state.eval(expr, result);
+    state.forceValueDeep(result);
+  }) << path.stem().string()
+     << ": should evaluate successfully";
+
+  auto expected = ExpectedOutputFor(path.stem().string());
+  std::ostringstream value_str;
+  value_str << result;
+
+  EXPECT_EQ(expected, value_str.str()) << "evaluator output should match";
 }
 
 INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest,