test(tvix/eval): add test for builtins parity r/5594

This will eventually force us to have a base builtins set in common with
C++ Nix, i.e. all 2.3 builtins except the controversial
builtins.valueSize.

Change-Id: I2c767f07d6a14711911658e87da9f18ede57a143
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7747
Autosubmit: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>

4 files changed, 42 insertions, 1 deletions

diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index b9bb6d8cf3..9692d50bf1 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -123,6 +123,11 @@ fn nix_eval_okay_currently_failing(code_path: &str) {
     eval_test(code_path, false)
 }
 
+#[test_resources("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")]
+fn eval_okay_currently_failing(code_path: &str) {
+    eval_test(code_path, false)
+}
+
 // eval-fail-* tests contain a snippet of Nix code, which is
 // expected to fail evaluation.  The exact type of failure
 // (assertion, parse error, etc) is not currently checked.
diff --git a/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-minimal-2.3-builtins.exp b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-minimal-2.3-builtins.exp
new file mode 100644
index 0000000000..967fc858bc
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-minimal-2.3-builtins.exp
@@ -0,0 +1 @@
+[ "abort" "add" "addErrorContext" "all" "any" "appendContext" "attrNames" "attrValues" "baseNameOf" "bitAnd" "bitOr" "bitXor" "builtins" "catAttrs" "compareVersions" "concatLists" "concatMap" "concatStringsSep" "currentSystem" "currentTime" "deepSeq" "derivation" "derivationStrict" "dirOf" "div" "elem" "elemAt" "false" "fetchGit" "fetchMercurial" "fetchTarball" "fetchurl" "filter" "filterSource" "findFile" "foldl'" "fromJSON" "fromTOML" "functionArgs" "genList" "genericClosure" "getAttr" "getContext" "getEnv" "hasAttr" "hasContext" "hashFile" "hashString" "head" "import" "intersectAttrs" "isAttrs" "isBool" "isFloat" "isFunction" "isInt" "isList" "isNull" "isPath" "isString" "langVersion" "length" "lessThan" "listToAttrs" "map" "mapAttrs" "match" "mul" "nixPath" "nixVersion" "null" "parseDrvName" "partition" "path" "pathExists" "placeholder" "readDir" "readFile" "removeAttrs" "replaceStrings" "scopedImport" "seq" "sort" "split" "splitVersion" "storeDir" "storePath" "stringLength" "sub" "substring" "tail" "throw" "toFile" "toJSON" "toPath" "toString" "toXML" "trace" "true" "tryEval" "typeOf" "unsafeDiscardOutputDependency" "unsafeDiscardStringContext" "unsafeGetAttrPos" ]
diff --git a/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-minimal-2.3-builtins.nix b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-minimal-2.3-builtins.nix
new file mode 100644
index 0000000000..804355f34d
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-minimal-2.3-builtins.nix
@@ -0,0 +1,35 @@
+# This tests verifies that the Nix implementation evaluating this has at least
+# all the builtins given in `minimalBuiltins`. We don't test a precise list of
+# builtins since we accept that there will always be difference between the
+# builtins sets of Tvix, C++ Nix 2.3 and newer C++ Nix versions, as new builtins
+# are added.
+#
+# Tvix also may choose never to implement some builtins if they are only useful
+# for flakes or perform well enough via the shims nixpkgs usually provides.
+
+let
+  # C++ Nix 2.3 builtins except valueSize which is removed in later versions
+  minimalBuiltins = [
+    "abort" "add" "addErrorContext" "all" "any" "appendContext" "attrNames"
+    "attrValues" "baseNameOf" "bitAnd" "bitOr" "bitXor" "builtins" "catAttrs"
+    "compareVersions" "concatLists" "concatMap" "concatStringsSep"
+    "currentSystem" "currentTime" "deepSeq" "derivation" "derivationStrict"
+    "dirOf" "div" "elem" "elemAt" "false" "fetchGit" "fetchMercurial"
+    "fetchTarball" "fetchurl" "filter" "filterSource" "findFile" "foldl'"
+    "fromJSON" "fromTOML" "functionArgs" "genList" "genericClosure" "getAttr"
+    "getContext" "getEnv" "hasAttr" "hasContext" "hashFile" "hashString" "head"
+    "import" "intersectAttrs" "isAttrs" "isBool" "isFloat" "isFunction" "isInt"
+    "isList" "isNull" "isPath" "isString" "langVersion" "length" "lessThan"
+    "listToAttrs" "map" "mapAttrs" "match" "mul" "nixPath" "nixVersion" "null"
+    "parseDrvName" "partition" "path" "pathExists" "placeholder" "readDir"
+    "readFile" "removeAttrs" "replaceStrings" "scopedImport" "seq" "sort"
+    "split" "splitVersion" "storeDir" "storePath" "stringLength" "sub"
+    "substring" "tail" "throw" "toFile" "toJSON" "toPath" "toString" "toXML"
+    "trace" "true" "tryEval" "typeOf" "unsafeDiscardOutputDependency"
+    "unsafeDiscardStringContext" "unsafeGetAttrPos"
+  ];
+
+  intersectLists = as: bs: builtins.filter (a: builtins.elem a bs) as;
+in
+
+intersectLists minimalBuiltins (builtins.attrNames builtins)
diff --git a/tvix/verify-lang-tests/default.nix b/tvix/verify-lang-tests/default.nix
index 4de92ab6c8..2267ecb77d 100644
--- a/tvix/verify-lang-tests/default.nix
+++ b/tvix/verify-lang-tests/default.nix
@@ -34,7 +34,7 @@ let
             (builtins.map (parseTest dir))
             (builtins.filter (t: t != null))
           ]
-      ) [ "nix_tests" "nix_tests/notyetpassing" "tvix_tests" ];
+      ) [ "nix_tests" "nix_tests/notyetpassing" "tvix_tests" "tvix_tests/notyetpassing" ];
 
   skippedLangTests = {
     # TODO(sterni): set up NIX_PATH in sandbox