test(3p/nix): Enable output comparison for evaluator success tests r/1398

Enables loading of the expected output of evaluator tests from the
corresponding .exp files, and checks that the output matches.

This again leaves some tests behind in the disabled folder, but we now
have almost the entire suite up and running so I can get around to
cleaning up the disabled ones.

Other note: Some tests had XML output, despite not being related to
XML testing at all - I'm not sure why they chose to do this, but have
converted those test outputs to normal Nix instead.

We have a separate test suite for JSON & XML serialisation already,
which was contributed by andi-.

Change-Id: Id7c42c836edfec4c22db9d893e35489f3e6dd559
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1285
Tested-by: BuildkiteCI
Reviewed-by: isomer <isomer@tvl.fyi>
Reviewed-by: glittershark <grfn@gws.fyi>

16 files changed, 37 insertions, 364 deletions

diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix
index 54e77ec50d11..cceb6b4f1bd5 100644
--- a/third_party/nix/default.nix
+++ b/third_party/nix/default.nix
@@ -89,6 +89,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation {
 
   installCheckPhase = ''
     export NIX_DATA_DIR=$out/share
+    export NIX_TEST_VAR=foo # this is required by a language test
     make test
   '';
 
@@ -124,6 +125,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation {
 
   shellHook = ''
     export NIX_DATA_DIR="${toString depotPath}/third_party"
+    export NIX_TEST_VAR=foo
   '';
 
   # TODO(tazjin): integration test setup?
diff --git a/third_party/nix/src/tests/lang/disabled/README.txt b/third_party/nix/src/tests/lang/disabled/README.txt
index 3dbcbf4f0dd9..50225deb63a2 100644
--- a/third_party/nix/src/tests/lang/disabled/README.txt
+++ b/third_party/nix/src/tests/lang/disabled/README.txt
@@ -1,4 +1,4 @@
-These tests are disabeld primarily because the DummyStore used for
+These tests are disabled primarily because the DummyStore used for
 tests does not interact with real files on disk at the moment, but the
 tests expect it to.
 
diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp
index 7a8391786a09..7a8391786a09 100644
--- a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp
diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix
index 815f51b1d67a..815f51b1d67a 100644
--- a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix
index 8299c3a4a3aa..8299c3a4a3aa 100644
--- a/third_party/nix/src/tests/lang/eval-okay-sort.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix
index 068c97a6c1b3..068c97a6c1b3 100644
--- a/third_party/nix/src/tests/lang/eval-okay-toxml.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix
index 9ee9f8a0b4f5..9ee9f8a0b4f5 100644
--- a/third_party/nix/src/tests/lang/eval-okay-xml.nix
+++ b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix
diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp b/third_party/nix/src/tests/lang/eval-okay-closure.exp
new file mode 100644
index 000000000000..e7dbf978160d
--- /dev/null
+++ b/third_party/nix/src/tests/lang/eval-okay-closure.exp
@@ -0,0 +1 @@
+[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ]
diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml b/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml
deleted file mode 100644
index dffc03a99891..000000000000
--- a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml
+++ /dev/null
@@ -1,343 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<expr>
-  <list>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-13" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-12" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-11" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-9" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-8" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-7" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-5" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-4" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="-3" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="-1" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="0" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="1" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="2" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="4" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="5" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="6" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="8" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="9" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="10" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="13" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="14" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="15" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="17" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="18" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="19" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="22" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="23" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="26" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="27" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="28" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="31" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="32" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="35" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="36" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="40" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="41" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="44" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="45" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="49" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="53" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="54" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="58" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="62" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="foo">
-        <bool value="true" />
-      </attr>
-      <attr name="key">
-        <int value="67" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="71" />
-      </attr>
-    </attrs>
-    <attrs>
-      <attr name="key">
-        <int value="80" />
-      </attr>
-    </attrs>
-  </list>
-</expr>
diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp b/third_party/nix/src/tests/lang/eval-okay-eq.exp
new file mode 100644
index 000000000000..27ba77ddaf61
--- /dev/null
+++ b/third_party/nix/src/tests/lang/eval-okay-eq.exp
@@ -0,0 +1 @@
+true
diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled b/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled
deleted file mode 100644
index 2015847b65e7..000000000000
--- a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled
+++ /dev/null
@@ -1 +0,0 @@
-Bool(True)
diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp
new file mode 100644
index 000000000000..c1c9f8ffaf69
--- /dev/null
+++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp
@@ -0,0 +1 @@
+[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ]
diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml
deleted file mode 100644
index 651f54c36341..000000000000
--- a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<expr>
-  <list>
-    <string value="stdenv" />
-    <string value="fetchurl" />
-    <string value="aterm-stdenv" />
-    <string value="aterm-stdenv2" />
-    <string value="libX11" />
-    <string value="libXv" />
-    <string value="mplayer-stdenv2.libXv-libX11" />
-    <string value="mplayer-stdenv2.libXv-libX11_2" />
-    <string value="nix-stdenv-aterm-stdenv" />
-    <string value="nix-stdenv2-aterm2-stdenv2" />
-  </list>
-</expr>
diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.nix b/third_party/nix/src/tests/lang/eval-okay-getenv.nix
index 4cfec5f553d9..ea8bb9f0a629 100644
--- a/third_party/nix/src/tests/lang/eval-okay-getenv.nix
+++ b/third_party/nix/src/tests/lang/eval-okay-getenv.nix
@@ -1 +1 @@
-builtins.getEnv "TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla")
+builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla")
diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml b/third_party/nix/src/tests/lang/eval-okay-xml.exp
index 92b75e0b8b17..92b75e0b8b17 100644
--- a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml
+++ b/third_party/nix/src/tests/lang/eval-okay-xml.exp
diff --git a/third_party/nix/src/tests/language-tests.cc b/third_party/nix/src/tests/language-tests.cc
index daa2fa233e19..cac5aee5841c 100644
--- a/third_party/nix/src/tests/language-tests.cc
+++ b/third_party/nix/src/tests/language-tests.cc
@@ -23,9 +23,12 @@
 
 #include <algorithm>
 #include <filesystem>
+#include <fstream>
 #include <iostream>
+#include <iterator>
 #include <memory>
 #include <optional>
+#include <sstream>
 #include <string>
 
 #include <absl/strings/ascii.h>
@@ -86,6 +89,20 @@ std::string TestNameFor(
   return name;
 }
 
+// Load the expected output of a given test as a string.
+std::string ExpectedOutputFor(absl::string_view stem) {
+  std::filesystem::path path(
+      absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp"));
+
+  EXPECT_TRUE(std::filesystem::exists(path))
+      << stem << ": expected output file should exist";
+
+  std::ifstream input(path);
+  std::stringstream buffer;
+  buffer << input.rdbuf();
+  return std::string(absl::StripTrailingAsciiWhitespace(buffer.str()));
+}
+
 }  // namespace
 
 using nix::tests::DummyStore;
@@ -189,7 +206,7 @@ INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest,
 
 class EvalSuccessTest : public testing::TestWithParam<std::filesystem::path> {};
 
-// Test pattern for files that should fail to evaluate.
+// Test pattern for files that should evaluate successfully.
 TEST_P(EvalSuccessTest, Fails) {
   std::shared_ptr<Store> store = std::make_shared<DummyStore>();
   EvalState state({}, ref<Store>(store));
@@ -200,8 +217,18 @@ TEST_P(EvalSuccessTest, Fails) {
       << path.stem().string() << ": should parse successfully";
 
   Value result;
-  state.eval(expr, result);
-  state.forceValue(result);
+
+  EXPECT_NO_THROW({
+    state.eval(expr, result);
+    state.forceValueDeep(result);
+  }) << path.stem().string()
+     << ": should evaluate successfully";
+
+  auto expected = ExpectedOutputFor(path.stem().string());
+  std::ostringstream value_str;
+  value_str << result;
+
+  EXPECT_EQ(expected, value_str.str()) << "evaluator output should match";
 }
 
 INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest,