feat(tvix/verify-lang-tests): Test nixVersions.nix_{2_3,2_23} r/8464

With this change the latest Nix version being checked is 2.23
(previously was 2.18). A warning will now be printed when the
latest verified Nix version is out of date.

I added a test to the skip-list because the behavior of
builtins.dirOf has changed in Nix 2.22

Change-Id: Iaf7fa98dbfa599a2d2723df0e42bb459711e2413
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12105
Tested-by: BuildkiteCI
Autosubmit: Ilan Joselevich <personal@ilanjoselevich.com>
Reviewed-by: flokli <flokli@flokli.de>

1 files changed, 38 insertions, 29 deletions

diff --git a/tvix/verify-lang-tests/default.nix b/tvix/verify-lang-tests/default.nix
index 772c1c532078..b363f32dd9bf 100644
--- a/tvix/verify-lang-tests/default.nix
+++ b/tvix/verify-lang-tests/default.nix
@@ -9,7 +9,9 @@
 let
   testRoot = ../eval/src/tests;
 
-  inherit (pkgs.buildPackages) nix nix_latest;
+  inherit (pkgs.nixVersions) nix_2_3;
+  # The latest Nix version we've verified to work for our testing suite.
+  nix_latest_verified = pkgs.nixVersions.nix_2_23;
 
   parseTest = dir: baseName:
     let
@@ -44,50 +46,54 @@ let
     # C++ Nix can't TCO
     "eval-okay-tail-call-1.nix" = true;
     # Ordering change after 2.3
-    "eval-okay-xml.nix" = [ nix ];
+    "eval-okay-xml.nix" = [ nix_2_3 ];
     # Missing builtins in Nix 2.3
-    "eval-okay-ceil.nix" = [ nix ];
-    "eval-okay-floor-ceil.nix" = [ nix ];
-    "eval-okay-floor.nix" = [ nix ];
-    "eval-okay-groupBy.nix" = [ nix ];
-    "eval-okay-zipAttrsWith.nix" = [ nix ];
-    "eval-okay-builtins-group-by-propagate-catchable.nix" = [ nix ];
+    "eval-okay-ceil.nix" = [ nix_2_3 ];
+    "eval-okay-floor-ceil.nix" = [ nix_2_3 ];
+    "eval-okay-floor.nix" = [ nix_2_3 ];
+    "eval-okay-groupBy.nix" = [ nix_2_3 ];
+    "eval-okay-zipAttrsWith.nix" = [ nix_2_3 ];
+    "eval-okay-builtins-group-by-propagate-catchable.nix" = [ nix_2_3 ];
     # Comparable lists are not in Nix 2.3
-    "eval-okay-sort.nix" = [ nix ];
-    "eval-okay-compare-lists.nix" = [ nix ];
-    "eval-okay-value-pointer-compare.nix" = [ nix ];
-    "eval-okay-builtins-genericClosure-pointer-equality.nix" = [ nix ];
-    "eval-okay-list-comparison.nix" = [ nix ];
+    "eval-okay-sort.nix" = [ nix_2_3 ];
+    "eval-okay-compare-lists.nix" = [ nix_2_3 ];
+    "eval-okay-value-pointer-compare.nix" = [ nix_2_3 ];
+    "eval-okay-builtins-genericClosure-pointer-equality.nix" = [ nix_2_3 ];
+    "eval-okay-list-comparison.nix" = [ nix_2_3 ];
     # getAttrPos gains support for functionArgs-returned sets after 2.3
-    "eval-okay-getattrpos-functionargs.nix" = [ nix ];
+    "eval-okay-getattrpos-functionargs.nix" = [ nix_2_3 ];
     # groupBy appeared (long) after 2.3
-    "eval-okay-builtins-groupby-thunk.nix" = [ nix ];
+    "eval-okay-builtins-groupby-thunk.nix" = [ nix_2_3 ];
     # import is no longer considered a curried primop in Nix > 2.3
-    "eval-okay-import-display.nix" = [ nix ];
+    "eval-okay-import-display.nix" = [ nix_2_3 ];
     # Cycle detection and formatting changed sometime after Nix 2.3
-    "eval-okay-cycle-display-cpp-nix-2.13.nix" = [ nix ];
+    "eval-okay-cycle-display-cpp-nix-2.13.nix" = [ nix_2_3 ];
     # builtins.replaceStrings becomes lazier in Nix 2.16
-    "eval-okay-replacestrings.nix" = [ nix ];
+    "eval-okay-replacestrings.nix" = [ nix_2_3 ];
     # builtins.readFileType is added in Nix 2.15
-    "eval-okay-readFileType.nix" = [ nix ];
+    "eval-okay-readFileType.nix" = [ nix_2_3 ];
     # builtins.fromTOML gains support for timestamps in Nix 2.16
-    "eval-okay-fromTOML-timestamps.nix" = [ nix ];
+    "eval-okay-fromTOML-timestamps.nix" = [ nix_2_3 ];
     # identifier formatting changed in Nix 2.17 due to cppnix commit
     # b72bc4a972fe568744d98b89d63adcd504cb586c
-    "eval-okay-identifier-formatting.nix" = [ nix ];
+    "eval-okay-identifier-formatting.nix" = [ nix_2_3 ];
 
     # Differing strictness in the function argument for some builtins in Nix 2.18
     # https://github.com/NixOS/nix/issues/9779
-    "eval-okay-builtins-map-propagate-catchable.nix" = [ nix_latest ];
-    "eval-okay-builtins-gen-list-propagate-catchable.nix" = [ nix_latest ];
+    "eval-okay-builtins-map-propagate-catchable.nix" = [ nix_latest_verified ];
+    "eval-okay-builtins-gen-list-propagate-catchable.nix" = [ nix_latest_verified ];
     "eval-okay-builtins-replace-strings-propagate-catchable.nix" =
-      [ nix_latest ];
-    "eval-okay-builtins-map-function-strictness.nix" = [ nix_latest ];
-    "eval-okay-builtins-genList-function-strictness.nix" = [ nix_latest ];
+      [ nix_latest_verified ];
+    "eval-okay-builtins-map-function-strictness.nix" = [ nix_latest_verified ];
+    "eval-okay-builtins-genList-function-strictness.nix" = [ nix_latest_verified ];
 
     # TODO(sterni): support diffing working directory and home relative paths
     # like C++ Nix test suite (using string replacement).
     "eval-okay-path-antiquotation.nix" = true;
+
+    # The output of dirOf (and maybe other filesystem builtins) have changed in Nix 2.23
+    # when they switched to using std::filesystem, https://github.com/NixOS/nix/pull/10658.
+    "eval-okay-dirof.nix" = [ nix_latest_verified ];
   };
 
   runCppNixLangTests = cpp-nix:
@@ -219,8 +225,11 @@ let
     };
 
 in
-
 depot.nix.readTree.drvTargets {
-  "nix-2.3" = runCppNixLangTests nix;
-  "nix-${lib.versions.majorMinor nix_latest.version}" = runCppNixLangTests nix_latest;
+  "nix-${lib.versions.majorMinor nix_2_3.version}" = runCppNixLangTests nix_2_3;
+  "nix-${lib.versions.majorMinor nix_latest_verified.version}" = lib.warnIf (lib.versionOlder nix_latest_verified.version pkgs.nixVersions.latest.version)
+    "The latest verified Nix version is out of date, consider updating the value of `nix_latest_verified` and verifying that the tests still pass."
+    runCppNixLangTests
+    nix_latest_verified;
 }
+