diff options
| -rw-r--r-- | third_party/overlays/tvl.nix | 2 | ||||
| -rw-r--r-- | tvix/eval/default.nix | 120 | ||||
| -rw-r--r-- | tvix/verify-lang-tests/default.nix | 183 |
3 files changed, 185 insertions, 120 deletions
diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix index 7eb0d22fc8..afa1c6cdee 100644 --- a/third_party/overlays/tvl.nix +++ b/third_party/overlays/tvl.nix @@ -26,6 +26,8 @@ in systems = [ builtins.currentSystem ]; }).build."${builtins.currentSystem}"; + nix_latest = super.nix; + clang-tools_11 = self.clang-tools.override { llvmPackages = self.llvmPackages_11; }; diff --git a/tvix/eval/default.nix b/tvix/eval/default.nix index 7328ed7ee2..d8e180b7cf 100644 --- a/tvix/eval/default.nix +++ b/tvix/eval/default.nix @@ -26,125 +26,5 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: { passthru = { }; }); - - passthru.cpp-nix-run-lang-tests = pkgs.stdenv.mkDerivation { - name = "cpp-nix-run-lang-tests"; - - src = ./src/tests; - dontConfigure = true; - - nativeBuildInputs = [ - pkgs.buildPackages.nix - ]; - - buildPhase = '' - chmod +x $scriptPath - patchShebangs --build $scriptPath - - mkdir store var - export NIX_STORE_DIR="$(realpath store)" - export NIX_STATE_DIR="$(realpath var)" - - $scriptPath - ''; - - installPhase = "touch $out"; - - passAsFile = [ "script" ]; - script = '' - #!/usr/bin/env bash - # SPDX-License-Identifier: LGPL-2.1-only - # SPDX-FileCopyrightText: © 2022 The TVL Contributors - # SPDX-FileCopyrightText: © 2004-2022 The Nix Contributors - # - # Execute language tests found in tvix_tests and nix_tests - # using the C++ Nix implementation. Based on NixOS/nix:tests/lang.sh. - - expect() { - local expected res - expected="$1" - shift - set +e - "$@" - res="$?" - set -e - [[ $res -eq $expected ]] - } - - TESTDIR="''${1:-.}" - - fail=0 - - for i in "$TESTDIR/"*_tests/parse-fail-*.nix; do - echo "parsing $i (should fail)"; - if ! expect 1 nix-instantiate --parse - < $i 2> /dev/null; then - echo "FAIL: $i shouldn't parse" - fail=1 - fi - done - - for i in "$TESTDIR/"*_tests/parse-okay-*.nix; do - echo "parsing $i (should succeed)"; - if ! expect 0 nix-instantiate --parse - < $i > /dev/null; then - echo "FAIL: $i should parse" - fail=1 - fi - done - - for i in "$TESTDIR/"*_tests/eval-fail-*.nix; do - echo "evaluating $i (should fail)"; - if ! expect 1 nix-instantiate --eval $i 2> /dev/null; then - echo "FAIL: $i shouldn't evaluate" - fail=1 - fi - done - - export TEST_VAR="foo" - - for i in "$TESTDIR/"*_tests/eval-okay-*.nix; do - echo "evaluating $i (should succeed)"; - - base="$(dirname "$i")/$(basename "$i" ".nix")" - - case "$(basename $i)" in - eval-okay-search-path.nix) ;& - eval-okay-tail-call-1.nix | \ - eval-okay-fromjson.nix) - # TODO(sterni,grfn): fix these tests - echo "SKIPPED: $i" - continue - ;; - *) ;; - esac - - if test -e $base.exp; then - flags= - if test -e $base.flags; then - flags=$(cat $base.flags) - fi - if ! expect 0 nix-instantiate $flags --eval --strict $base.nix > $base.out; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! diff $base.out $base.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 - fi - fi - - if test -e $base.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - $base.nix > $base.out.xml; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s $base.out.xml $base.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 - fi - fi - done - - exit $fail - ''; - }; })) ) diff --git a/tvix/verify-lang-tests/default.nix b/tvix/verify-lang-tests/default.nix new file mode 100644 index 0000000000..41282488d4 --- /dev/null +++ b/tvix/verify-lang-tests/default.nix @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: LGPL-2.1-only +# SPDX-FileCopyrightText: © 2022 The TVL Contributors +# SPDX-FileCopyrightText: © 2004-2022 The Nix Contributors +# +# Execute language tests found in tvix_tests and nix_tests +# using the C++ Nix implementation. Based on NixOS/nix:tests/lang.sh. +{ depot, pkgs, lib, ... }: + +let + testRoot = ../eval/src/tests; + + inherit (pkgs.buildPackages) nix nix_latest; + + parseTest = dir: baseName: + let + tokens = builtins.match "(eval|parse)-(okay|fail).+\\.nix" baseName; + in + if tokens == null + then null + else { + type = builtins.elemAt tokens 0; + expectedSuccess = (builtins.elemAt tokens 1) == "okay"; + fileName = "${dir}/${baseName}"; + }; + + allLangTests = + lib.concatMap + ( + dir: + lib.pipe + (builtins.readDir (testRoot + "/${dir}")) + [ + builtins.attrNames + (builtins.map (parseTest dir)) + (builtins.filter (t: t != null)) + ] + ) [ "nix_tests" "tvix_tests" ]; + + skippedLangTests = { + # TODO(sterni): set up NIX_PATH in sandbox + "nix_tests/eval-okay-search-path.nix" = true; + # Floating point precision differs between tvix and Nix + "tvix_tests/eval-okay-fromjson.nix" = true; + # C++ Nix can't do TCO + "nix_tests/eval-okay-tail-call-1.nix" = true; + # Ordering change after 2.3 + "nix_tests/eval-okay-xml.nix" = [ nix_latest ]; + }; + + runCppNixLangTests = cpp-nix: + let + testCommand = { fileName, type, expectedSuccess, ... }: + let + testBase = lib.removeSuffix ".nix" fileName; + expFile = + let + possibleFiles = + builtins.filter + (path: builtins.pathExists (testRoot + "/${path}")) + (builtins.map + (ext: "${testBase}.${ext}") + [ "exp" "exp.xml" ]); + in + if possibleFiles == [ ] then null else builtins.head possibleFiles; + outFile = "${testBase}.out"; + + # Skip if skippedLangTests prescribes it (possibly just for the current nix) + # or if we are missing an exp file for an eval-okay test. + skip = + let + doSkip = skippedLangTests.${fileName} or false; + in + if type == "eval" && expectedSuccess && (expFile == null) then true + else if builtins.isBool doSkip then doSkip + else builtins.any (drv: cpp-nix == drv) doSkip; + + flagsFile = "${testBase}.flags"; + + instantiateFlags = + lib.escapeShellArgs + ( + [ "--${type}" fileName ] + ++ lib.optionals (type == "eval") [ "--strict" ] + ++ lib.optionals (expFile != null && lib.hasSuffix "xml" expFile) + [ + "--no-location" + "--xml" + ] + ) + + lib.optionalString (builtins.pathExists (testRoot + "/${flagsFile}")) + " $(cat '${flagsFile}')"; + in + + if skip + then "echo \"SKIP ${type} ${fileName}\"\n" + else '' + thisTestPassed=true + + echo "RUN ${type} ${fileName} ${ + lib.optionalString (!expectedSuccess) "(expecting failure)" + }" + + if ! expect ${if expectedSuccess then "0" else "1"} \ + nix-instantiate ${instantiateFlags} \ + ${if expectedSuccess then "1" else "2"}> \ + ${if expFile != null then outFile else "/dev/null"}; + then + echo -n "FAIL" + thisTestPassed=false + fi + '' + lib.optionalString (expFile != null) '' + if ! diff --color=always -u '${outFile}' '${expFile}'; then + thisTestPassed=false + fi + '' + '' + if $thisTestPassed; then + echo -n "PASS" + else + echo -n "FAIL" + passed=false + fi + + echo " ${type} ${fileName}" + + unset thisTestPassed + ''; + in + + pkgs.stdenv.mkDerivation { + name = "cpp-${cpp-nix.name}-run-lang-tests"; + + nativeBuildInputs = [ cpp-nix ]; + + # Obtain tests via the unpackPhase + src = testRoot; + dontConfigure = true; + + # Environment expected by the test suite + TEST_VAR = "foo"; + + buildPhase = '' + # Make nix-instantiate happy in the sandbox + export NIX_STORE_DIR="$(realpath "$(mktemp -d store.XXXXXXXXXX)")" + export NIX_STATE_DIR="$(realpath "$(mktemp -d state.XXXXXXXXXX)")" + + # Helper function to check expected exit code + expect() { + local expected res + expected="$1" + shift + set +e + "$@" + res="$?" + set -e + [[ $res -eq $expected ]] + } + + # Track test results so far + passed=true + + source "$testCommandsPath" + ''; + + # Actually runs into the argv limit + passAsFile = [ "testCommands" ]; + testCommands = lib.concatMapStrings testCommand allLangTests; + + installPhase = '' + if $passed; then + touch $out + else + echo "Some test(s) failed!" + exit 1 + fi + ''; + }; + +in + +depot.nix.readTree.drvTargets { + "nix-2.3" = runCppNixLangTests nix; + "nix-${lib.versions.majorMinor nix_latest.version}" = runCppNixLangTests nix_latest; +} |