about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--third_party/overlays/tvl.nix2
-rw-r--r--tvix/eval/default.nix120
-rw-r--r--tvix/verify-lang-tests/default.nix183
3 files changed, 185 insertions, 120 deletions
diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix
index 7eb0d22fc88c..afa1c6cdeec4 100644
--- a/third_party/overlays/tvl.nix
+++ b/third_party/overlays/tvl.nix
@@ -26,6 +26,8 @@ in
     systems = [ builtins.currentSystem ];
   }).build."${builtins.currentSystem}";
 
+  nix_latest = super.nix;
+
   clang-tools_11 = self.clang-tools.override {
     llvmPackages = self.llvmPackages_11;
   };
diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix
index 7328ed7ee2c2..d8e180b7cf33 100644
--- a/tvix/eval/default.nix
+++ b/tvix/eval/default.nix
@@ -26,125 +26,5 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: {
 
     passthru = { };
   });
-
-  passthru.cpp-nix-run-lang-tests = pkgs.stdenv.mkDerivation {
-    name = "cpp-nix-run-lang-tests";
-
-    src = ./src/tests;
-    dontConfigure = true;
-
-    nativeBuildInputs = [
-      pkgs.buildPackages.nix
-    ];
-
-    buildPhase = ''
-      chmod +x $scriptPath
-      patchShebangs --build $scriptPath
-
-      mkdir store var
-      export NIX_STORE_DIR="$(realpath store)"
-      export NIX_STATE_DIR="$(realpath var)"
-
-      $scriptPath
-    '';
-
-    installPhase = "touch $out";
-
-    passAsFile = [ "script" ];
-    script = ''
-      #!/usr/bin/env bash
-      # SPDX-License-Identifier: LGPL-2.1-only
-      # SPDX-FileCopyrightText: © 2022 The TVL Contributors
-      # SPDX-FileCopyrightText: © 2004-2022 The Nix Contributors
-      #
-      # Execute language tests found in tvix_tests and nix_tests
-      # using the C++ Nix implementation. Based on NixOS/nix:tests/lang.sh.
-
-      expect() {
-        local expected res
-        expected="$1"
-        shift
-        set +e
-        "$@"
-        res="$?"
-        set -e
-        [[ $res -eq $expected ]]
-      }
-
-      TESTDIR="''${1:-.}"
-
-      fail=0
-
-      for i in "$TESTDIR/"*_tests/parse-fail-*.nix; do
-          echo "parsing $i (should fail)";
-          if ! expect 1 nix-instantiate --parse - < $i 2> /dev/null; then
-              echo "FAIL: $i shouldn't parse"
-              fail=1
-          fi
-      done
-
-      for i in "$TESTDIR/"*_tests/parse-okay-*.nix; do
-          echo "parsing $i (should succeed)";
-          if ! expect 0 nix-instantiate --parse - < $i > /dev/null; then
-              echo "FAIL: $i should parse"
-              fail=1
-          fi
-      done
-
-      for i in "$TESTDIR/"*_tests/eval-fail-*.nix; do
-          echo "evaluating $i (should fail)";
-          if ! expect 1 nix-instantiate --eval $i 2> /dev/null; then
-              echo "FAIL: $i shouldn't evaluate"
-              fail=1
-          fi
-      done
-
-      export TEST_VAR="foo"
-
-      for i in "$TESTDIR/"*_tests/eval-okay-*.nix; do
-          echo "evaluating $i (should succeed)";
-
-          base="$(dirname "$i")/$(basename "$i" ".nix")"
-
-          case "$(basename $i)" in
-            eval-okay-search-path.nix) ;&
-            eval-okay-tail-call-1.nix | \
-            eval-okay-fromjson.nix)
-              # TODO(sterni,grfn): fix these tests
-              echo "SKIPPED: $i"
-              continue
-              ;;
-            *) ;;
-          esac
-
-          if test -e $base.exp; then
-              flags=
-              if test -e $base.flags; then
-                  flags=$(cat $base.flags)
-              fi
-              if ! expect 0 nix-instantiate $flags --eval --strict $base.nix > $base.out; then
-                  echo "FAIL: $i should evaluate"
-                  fail=1
-              elif ! diff $base.out $base.exp; then
-                  echo "FAIL: evaluation result of $i not as expected"
-                  fail=1
-              fi
-          fi
-
-          if test -e $base.exp.xml; then
-              if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
-                      $base.nix > $base.out.xml; then
-                  echo "FAIL: $i should evaluate"
-                  fail=1
-              elif ! cmp -s $base.out.xml $base.exp.xml; then
-                  echo "FAIL: XML evaluation result of $i not as expected"
-                  fail=1
-              fi
-          fi
-      done
-
-      exit $fail
-    '';
-  };
 }))
 )
diff --git a/tvix/verify-lang-tests/default.nix b/tvix/verify-lang-tests/default.nix
new file mode 100644
index 000000000000..41282488d4e3
--- /dev/null
+++ b/tvix/verify-lang-tests/default.nix
@@ -0,0 +1,183 @@
+# SPDX-License-Identifier: LGPL-2.1-only
+# SPDX-FileCopyrightText: © 2022 The TVL Contributors
+# SPDX-FileCopyrightText: © 2004-2022 The Nix Contributors
+#
+# Execute language tests found in tvix_tests and nix_tests
+# using the C++ Nix implementation. Based on NixOS/nix:tests/lang.sh.
+{ depot, pkgs, lib, ... }:
+
+let
+  testRoot = ../eval/src/tests;
+
+  inherit (pkgs.buildPackages) nix nix_latest;
+
+  parseTest = dir: baseName:
+    let
+      tokens = builtins.match "(eval|parse)-(okay|fail).+\\.nix" baseName;
+    in
+    if tokens == null
+    then null
+    else {
+      type = builtins.elemAt tokens 0;
+      expectedSuccess = (builtins.elemAt tokens 1) == "okay";
+      fileName = "${dir}/${baseName}";
+    };
+
+  allLangTests =
+    lib.concatMap
+      (
+        dir:
+        lib.pipe
+          (builtins.readDir (testRoot + "/${dir}"))
+          [
+            builtins.attrNames
+            (builtins.map (parseTest dir))
+            (builtins.filter (t: t != null))
+          ]
+      ) [ "nix_tests" "tvix_tests" ];
+
+  skippedLangTests = {
+    # TODO(sterni): set up NIX_PATH in sandbox
+    "nix_tests/eval-okay-search-path.nix" = true;
+    # Floating point precision differs between tvix and Nix
+    "tvix_tests/eval-okay-fromjson.nix" = true;
+    # C++ Nix can't do TCO
+    "nix_tests/eval-okay-tail-call-1.nix" = true;
+    # Ordering change after 2.3
+    "nix_tests/eval-okay-xml.nix" = [ nix_latest ];
+  };
+
+  runCppNixLangTests = cpp-nix:
+    let
+      testCommand = { fileName, type, expectedSuccess, ... }:
+        let
+          testBase = lib.removeSuffix ".nix" fileName;
+          expFile =
+            let
+              possibleFiles =
+                builtins.filter
+                  (path: builtins.pathExists (testRoot + "/${path}"))
+                  (builtins.map
+                    (ext: "${testBase}.${ext}")
+                    [ "exp" "exp.xml" ]);
+            in
+            if possibleFiles == [ ] then null else builtins.head possibleFiles;
+          outFile = "${testBase}.out";
+
+          # Skip if skippedLangTests prescribes it (possibly just for the current nix)
+          # or if we are missing an exp file for an eval-okay test.
+          skip =
+            let
+              doSkip = skippedLangTests.${fileName} or false;
+            in
+            if type == "eval" && expectedSuccess && (expFile == null) then true
+            else if builtins.isBool doSkip then doSkip
+            else builtins.any (drv: cpp-nix == drv) doSkip;
+
+          flagsFile = "${testBase}.flags";
+
+          instantiateFlags =
+            lib.escapeShellArgs
+              (
+                [ "--${type}" fileName ]
+                ++ lib.optionals (type == "eval") [ "--strict" ]
+                ++ lib.optionals (expFile != null && lib.hasSuffix "xml" expFile)
+                  [
+                    "--no-location"
+                    "--xml"
+                  ]
+              )
+            + lib.optionalString (builtins.pathExists (testRoot + "/${flagsFile}"))
+              " $(cat '${flagsFile}')";
+        in
+
+        if skip
+        then "echo \"SKIP ${type} ${fileName}\"\n"
+        else ''
+          thisTestPassed=true
+
+          echo "RUN  ${type} ${fileName} ${
+            lib.optionalString (!expectedSuccess) "(expecting failure)"
+          }"
+
+          if ! expect ${if expectedSuccess then "0" else "1"} \
+                 nix-instantiate ${instantiateFlags} \
+                 ${if expectedSuccess then "1" else "2"}> \
+                 ${if expFile != null then outFile else "/dev/null"};
+          then
+            echo -n "FAIL"
+            thisTestPassed=false
+          fi
+        '' + lib.optionalString (expFile != null) ''
+          if ! diff --color=always -u '${outFile}' '${expFile}'; then
+            thisTestPassed=false
+          fi
+        '' + ''
+          if $thisTestPassed; then
+            echo -n "PASS"
+          else
+            echo -n "FAIL"
+            passed=false
+          fi
+
+          echo " ${type} ${fileName}"
+
+          unset thisTestPassed
+        '';
+    in
+
+    pkgs.stdenv.mkDerivation {
+      name = "cpp-${cpp-nix.name}-run-lang-tests";
+
+      nativeBuildInputs = [ cpp-nix ];
+
+      # Obtain tests via the unpackPhase
+      src = testRoot;
+      dontConfigure = true;
+
+      # Environment expected by the test suite
+      TEST_VAR = "foo";
+
+      buildPhase = ''
+        # Make nix-instantiate happy in the sandbox
+        export NIX_STORE_DIR="$(realpath "$(mktemp -d store.XXXXXXXXXX)")"
+        export NIX_STATE_DIR="$(realpath "$(mktemp -d state.XXXXXXXXXX)")"
+
+        # Helper function to check expected exit code
+        expect() {
+          local expected res
+          expected="$1"
+          shift
+          set +e
+          "$@"
+          res="$?"
+          set -e
+          [[ $res -eq $expected ]]
+        }
+
+        # Track test results so far
+        passed=true
+
+        source "$testCommandsPath"
+      '';
+
+      # Actually runs into the argv limit
+      passAsFile = [ "testCommands" ];
+      testCommands = lib.concatMapStrings testCommand allLangTests;
+
+      installPhase = ''
+        if $passed; then
+          touch $out
+        else
+          echo "Some test(s) failed!"
+          exit 1
+        fi
+      '';
+    };
+
+in
+
+depot.nix.readTree.drvTargets {
+  "nix-2.3" = runCppNixLangTests nix;
+  "nix-${lib.versions.majorMinor nix_latest.version}" = runCppNixLangTests nix_latest;
+}