feat(tvix/eval): strengthen significantly catchable test suite r/7495

Adds a bunch (notably certain overlapping) tests for catchable situations.

This should cover many scenarios, argument is catchable, element in argument is catchable, function returns
catchable in the middle of the processing, etc.

Co-authored-by: Aspen Smith <root@gws.fyi>
Change-Id: Icd722cf8dbc91a24f45cd540a328711e5826f76c
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10621
Reviewed-by: aspen <root@gws.fyi>
Tested-by: BuildkiteCI

72 files changed, 210 insertions, 0 deletions

diff --git a/tvix/eval/src/tests/tvix_tests/eval-fail-throw-abort-cannot-be-caught.nix b/tvix/eval/src/tests/tvix_tests/eval-fail-throw-abort-cannot-be-caught.nix
new file mode 100644
index 000000000000..10781cb4eada
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-fail-throw-abort-cannot-be-caught.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.throw (builtins.abort "abc"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-abort-throw-can-be-caught.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-abort-throw-can-be-caught.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-abort-throw-can-be-caught.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-abort-throw-can-be-caught.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-abort-throw-can-be-caught.nix
new file mode 100644
index 000000000000..aebeca1dad35
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-abort-throw-can-be-caught.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.abort (builtins.throw "abc"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-all-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-all-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-all-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-all-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-all-propagate-catchable.nix
new file mode 100644
index 000000000000..8902e27c459d
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-all-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.all (builtins.throw "a") [ "" ]) (builtins.all (x: true) (builtins.throw "b")) (builtins.all (_: builtins.throw "x") [ "" ]) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-any-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-any-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-any-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-any-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-any-propagate-catchable.nix
new file mode 100644
index 000000000000..8db5c0c6dce0
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-any-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.any (builtins.throw "a") [ "" ]) (builtins.any (x: true) (builtins.throw "b")) (builtins.any (_: builtins.throw "a") [ "" ]) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-attrvalues-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-attrvalues-propagate-catchable.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-attrvalues-propagate-catchable.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-attrvalues-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-attrvalues-propagate-catchable.nix
new file mode 100644
index 000000000000..b8c15c87488f
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-attrvalues-propagate-catchable.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.attrValues (builtins.throw "a"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-catattrs-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-catattrs-propagate-catchable.exp
new file mode 100644
index 000000000000..c3bb809c9ffb
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-catattrs-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-catattrs-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-catattrs-propagate-catchable.nix
new file mode 100644
index 000000000000..5385591f77aa
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-catattrs-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.catAttrs "a" (builtins.throw "b")) (builtins.catAttrs (builtins.throw "a") { a = 1; }) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-lists-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-lists-propagate-catchable.exp
new file mode 100644
index 000000000000..c82ddd8a801d
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-lists-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false true ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-lists-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-lists-propagate-catchable.nix
new file mode 100644
index 000000000000..383425320c80
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-lists-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.concatLists (builtins.throw "a")) (builtins.concatLists [ [] (builtins.throw "a") ]) (builtins.concatLists [ [] [] [ builtins.throw "a" ] ]) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-map-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-map-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-map-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-map-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-map-propagate-catchable.nix
new file mode 100644
index 000000000000..aa3a4449896e
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-map-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.concatMap (builtins.throw "a") [ "" ]) (builtins.concatMap (_: builtins.throw "x") [ "" ]) (builtins.concatMap (_: []) (builtins.throw "a")) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-strings-sep-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-strings-sep-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-strings-sep-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-strings-sep-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-strings-sep-propagate-catchable.nix
new file mode 100644
index 000000000000..ce5ce4170f7f
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-concat-strings-sep-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.concatStringsSep (builtins.throw "a") [ "" ]) (builtins.concatStringsSep "," (builtins.throw "a")) (builtins.concatStringsSep "," [ "a" (builtins.throw "a") ]) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-filter-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-filter-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-filter-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-filter-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-filter-propagate-catchable.nix
new file mode 100644
index 000000000000..715a0ce34f8b
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-filter-propagate-catchable.nix
@@ -0,0 +1 @@
+map (e: (builtins.tryEval e).success) [ (builtins.filter (builtins.throw "a") [ "" ]) (builtins.filter (x: true) (builtins.throw "b")) (builtins.filter (_: builtins.throw "x") [ "" ]) ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-foldl-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-foldl-propagate-catchable.exp
new file mode 100644
index 000000000000..7bf6c6346688
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-foldl-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false false true true ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-foldl-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-foldl-propagate-catchable.nix
new file mode 100644
index 000000000000..234c95040b67
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-foldl-propagate-catchable.nix
@@ -0,0 +1,8 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.foldl' (builtins.throw "a") {} [ {} {} {} ])
+  (builtins.foldl' (x: y: x // y) {} (builtins.throw "b"))
+  (builtins.foldl' (_: _: builtins.throw "x") {} [ {} ])
+  (builtins.foldl' (x: y: x // y) (builtins.throw "x") [ {} ])
+  (builtins.foldl' (x: y: x // y) {} [ {} { a = builtins.throw "z"; } {} ])
+  (builtins.foldl' (x: y: x // y) {} [ {} { b = 3; a = builtins.throw "u"; } {} ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-from-json-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-from-json-propagate-catchable.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-from-json-propagate-catchable.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-from-json-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-from-json-propagate-catchable.nix
new file mode 100644
index 000000000000..aa973c135278
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-from-json-propagate-catchable.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.fromJSON (builtins.throw "a"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-function-args-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-function-args-propagate-catchable.exp
new file mode 100644
index 000000000000..bd52bff2fa24
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-function-args-propagate-catchable.exp
@@ -0,0 +1 @@
+[ true false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-function-args-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-function-args-propagate-catchable.nix
new file mode 100644
index 000000000000..ca3e6772f28d
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-function-args-propagate-catchable.nix
@@ -0,0 +1,4 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.functionArgs (_: builtins.throw "a"))
+  (builtins.functionArgs (builtins.throw "b"))
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-gen-list-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-gen-list-propagate-catchable.exp
new file mode 100644
index 000000000000..652df3d4da57
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-gen-list-propagate-catchable.exp
@@ -0,0 +1 @@
+[ true false true ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-gen-list-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-gen-list-propagate-catchable.nix
new file mode 100644
index 000000000000..3d4739966ee3
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-gen-list-propagate-catchable.nix
@@ -0,0 +1,5 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.genList (builtins.throw "a") 10)
+  (builtins.genList (i: "") (builtins.throw "b"))
+  (builtins.genList (i: builtins.throw "x") 5)
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-getContext-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-getContext-propagate-catchable.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-getContext-propagate-catchable.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-getContext-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-getContext-propagate-catchable.nix
new file mode 100644
index 000000000000..70521665cafd
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-getContext-propagate-catchable.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.getContext (builtins.throw "a"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hasContext-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hasContext-propagate-catchable.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hasContext-propagate-catchable.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hasContext-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hasContext-propagate-catchable.nix
new file mode 100644
index 000000000000..0c02a82730aa
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hasContext-propagate-catchable.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.hasContext (builtins.throw "a"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-head-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-head-propagate-catchable.exp
new file mode 100644
index 000000000000..c3bb809c9ffb
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-head-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-head-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-head-propagate-catchable.nix
new file mode 100644
index 000000000000..0e69f2f6fc1e
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-head-propagate-catchable.nix
@@ -0,0 +1,4 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.head (builtins.throw "a"))
+  (builtins.head [ (builtins.throw "a") ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-isType-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-isType-propagate-catchable.exp
new file mode 100644
index 000000000000..cefd8652b41f
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-isType-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false false false false false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-isType-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-isType-propagate-catchable.nix
new file mode 100644
index 000000000000..28cf2351f658
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-isType-propagate-catchable.nix
@@ -0,0 +1,14 @@
+let
+  isTypeFns = [
+    builtins.isAttrs
+    builtins.isBool
+    builtins.isFloat
+    builtins.isFunction
+    builtins.isInt
+    builtins.isList
+    builtins.isNull
+    builtins.isPath
+    builtins.isString
+  ];
+in
+map (fn: (builtins.tryEval (fn (builtins.throw "is type"))).success) isTypeFns
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-list-to-attrs-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-list-to-attrs-propagate-catchable.exp
new file mode 100644
index 000000000000..5ee59ced8394
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-list-to-attrs-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false true true false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-list-to-attrs-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-list-to-attrs-propagate-catchable.nix
new file mode 100644
index 000000000000..232758980fb5
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-list-to-attrs-propagate-catchable.nix
@@ -0,0 +1,7 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.listToAttrs [ { name = builtins.throw "a"; value = "b"; } ])
+  (builtins.listToAttrs [ { name = "a"; value = builtins.throw "b"; } ])
+  (builtins.listToAttrs [ { name = "a"; value = "b"; } { name = "c"; value = builtins.throw "d"; } ])
+  (builtins.listToAttrs [ { name = "a"; value = "b"; } (builtins.throw "e") ])
+  (builtins.listToAttrs (builtins.throw "f"))
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-map-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-map-propagate-catchable.exp
new file mode 100644
index 000000000000..652df3d4da57
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-map-propagate-catchable.exp
@@ -0,0 +1 @@
+[ true false true ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-map-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-map-propagate-catchable.nix
new file mode 100644
index 000000000000..3ebb006c3f0d
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-map-propagate-catchable.nix
@@ -0,0 +1,5 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.map (builtins.throw "a") [ "" ])
+  (builtins.map (x: true) (builtins.throw "b"))
+  (builtins.map (_: builtins.throw "x") [ "" ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-parse-drv-name-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-parse-drv-name-propagate-catchable.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-parse-drv-name-propagate-catchable.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-parse-drv-name-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-parse-drv-name-propagate-catchable.nix
new file mode 100644
index 000000000000..c718727e7426
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-parse-drv-name-propagate-catchable.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.parseDrvName (builtins.throw "a"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-partition-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-partition-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-partition-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-partition-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-partition-propagate-catchable.nix
new file mode 100644
index 000000000000..1960b1790fdb
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-partition-propagate-catchable.nix
@@ -0,0 +1,5 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.partition (builtins.throw "a") [ "" ])
+  (builtins.partition (x: true) (builtins.throw "b"))
+  (builtins.partition (_: builtins.throw "x") [ "" ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-remove-attrs-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-remove-attrs-propagate-catchable.exp
new file mode 100644
index 000000000000..1e950b5aa2a4
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-remove-attrs-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false true false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-remove-attrs-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-remove-attrs-propagate-catchable.nix
new file mode 100644
index 000000000000..2eb011aedf07
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-remove-attrs-propagate-catchable.nix
@@ -0,0 +1,6 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.removeAttrs (builtins.throw "a") [ "a" ])
+  (builtins.removeAttrs { a = {}; } (builtins.throw "b"))
+  (builtins.removeAttrs { a = builtins.throw "b"; } [ "a" ])
+  (builtins.removeAttrs { "${builtins.throw "c"}" = "b"; } [ "c" ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-replace-strings-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-replace-strings-propagate-catchable.exp
new file mode 100644
index 000000000000..cefd8652b41f
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-replace-strings-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false false false false false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-replace-strings-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-replace-strings-propagate-catchable.nix
new file mode 100644
index 000000000000..ad9734ba9aee
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-replace-strings-propagate-catchable.nix
@@ -0,0 +1,16 @@
+map (e: (builtins.tryEval e).success) [
+  # This one may be hard to read for non-experts.
+  # Replace strings is a special built-in compared to others in the sense
+  # it might attempt to lazily evaluate things upon successful replacements,
+  # so it would not be surprising that some of the non-replacements which could throw
+  # could be ignored by laziness. It is not the case though.
+  (builtins.replaceStrings [ "a" (builtins.throw "b") ] [ "c" "d" ] "ab")
+  (builtins.replaceStrings [ "a" (builtins.throw "b") ] [ "c" "d" ] "a")
+  (builtins.replaceStrings [ "a" "b" ] [ "c" (builtins.throw "d") ] "a")
+  (builtins.replaceStrings [ "a" "b" ] [ "c" (builtins.throw "d") ] "ab")
+  (builtins.replaceStrings [ "" ] [ (builtins.throw "d") ] "ab")
+  (builtins.replaceStrings [ "a" "" ] [ "b" (builtins.throw "d") ] "ab")
+  (builtins.replaceStrings (builtins.throw "z") [ ] "ab")
+  (builtins.replaceStrings [ ] (builtins.throw "z") "ab")
+  (builtins.replaceStrings [ ] [ ] (builtins.throw "z"))
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-sort-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-sort-propagate-catchable.exp
new file mode 100644
index 000000000000..1e950b5aa2a4
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-sort-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false true false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-sort-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-sort-propagate-catchable.nix
new file mode 100644
index 000000000000..66ca4b98ed60
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-sort-propagate-catchable.nix
@@ -0,0 +1,6 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.sort (builtins.throw "a") [ "" ])
+  (builtins.sort (x: y: true) (builtins.throw "b"))
+  (builtins.sort (_: _: builtins.throw "x") [ "" ])
+  (builtins.sort (_: _: builtins.throw "x") [ "" "" ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split-propagate-catchable.exp
new file mode 100644
index 000000000000..c3bb809c9ffb
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split-propagate-catchable.nix
new file mode 100644
index 000000000000..1c86e9d6f8b1
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-split-propagate-catchable.nix
@@ -0,0 +1,4 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.split (builtins.throw "regex") "abc")
+  (builtins.split "[^/]" (builtins.throw "string"))
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-string-length-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-string-length-propagate-catchable.exp
new file mode 100644
index 000000000000..d181ebcff6c9
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-string-length-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-string-length-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-string-length-propagate-catchable.nix
new file mode 100644
index 000000000000..0904acd11446
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-string-length-propagate-catchable.nix
@@ -0,0 +1,4 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.stringLength (builtins.throw "a"))
+  # FIXME(raitobezarius): test coercions too.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-tail-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-tail-propagate-catchable.exp
new file mode 100644
index 000000000000..d4cd584d2232
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-tail-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false true true true ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-tail-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-tail-propagate-catchable.nix
new file mode 100644
index 000000000000..b4e461e12bf0
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-tail-propagate-catchable.nix
@@ -0,0 +1,6 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.tail (builtins.throw "a"))
+  (builtins.tail [ (builtins.throw "a") ])
+  (builtins.tail [ (builtins.throw "a") "a" ])
+  (builtins.tail [ (builtins.throw "a") (builtins.throw "a") ])
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp
new file mode 100644
index 000000000000..ca00e3c049d6
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix
new file mode 100644
index 000000000000..8ae5e48e9737
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-json-propagate-catchable.nix
@@ -0,0 +1,14 @@
+map (e: (builtins.tryEval (builtins.toJSON e)).success) [
+  (builtins.throw "a")
+  {
+    a = builtins.throw "attribute a";
+  }
+  {
+    a.b.c.d.e.f.g.h.i = builtins.throw "deep i";
+  }
+  {
+    x = 32;
+    y = builtins.throw "second argument";
+  }
+  # FIXME(raitobezarius): we would like to test coercions, i.e. `toFile` and `derivation` containing throwables.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-path-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-path-propagate-catchable.exp
new file mode 100644
index 000000000000..c3bb809c9ffb
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-path-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-path-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-path-propagate-catchable.nix
new file mode 100644
index 000000000000..808fb8c46ef3
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-path-propagate-catchable.nix
@@ -0,0 +1,5 @@
+map (e: (builtins.tryEval (builtins.toPath e)).success) [
+  (builtins.throw "a")
+  (./xyz + (builtins.throw "p"))
+  # FIXME: test derivations and files.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-string-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-string-propagate-catchable.exp
new file mode 100644
index 000000000000..ca00e3c049d6
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-string-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-string-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-string-propagate-catchable.nix
new file mode 100644
index 000000000000..7b19ff53c360
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-string-propagate-catchable.nix
@@ -0,0 +1,7 @@
+map (e: (builtins.tryEval (builtins.toString e)).success) [
+  (builtins.throw "a")
+  [ (builtins.throw "a") ]
+  [ "abc" (builtins.throw "a") ]
+  "abc${builtins.throw "c"}"
+  # FIXME: test derivations and files.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-xml-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-xml-propagate-catchable.exp
new file mode 100644
index 000000000000..366f7adf0d17
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-xml-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false false true false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-xml-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-xml-propagate-catchable.nix
new file mode 100644
index 000000000000..80d9e688be3f
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-to-xml-propagate-catchable.nix
@@ -0,0 +1,15 @@
+map (e: (builtins.tryEval (builtins.toXML e)).success) [
+  (builtins.throw "a")
+  [ (builtins.throw "a") ]
+  [ "abc" (builtins.throw "a") ]
+  "abc${builtins.throw "c"}"
+  (_: builtins.throw "d")
+  {
+    u = builtins.throw "x";
+    v = "a";
+  }
+  {
+    u.i.w.x.z = builtins.throw "n";
+  }
+  # FIXME: test derivations and files.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-type-of-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-type-of-propagate-catchable.exp
new file mode 100644
index 000000000000..41f22d3ee4f2
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-type-of-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false true true false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-type-of-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-type-of-propagate-catchable.nix
new file mode 100644
index 000000000000..4b70609bbf3c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-type-of-propagate-catchable.nix
@@ -0,0 +1,9 @@
+map (e: (builtins.tryEval (builtins.typeOf e)).success) [
+  (builtins.throw "a")
+  {
+    a = builtins.throw "b";
+  }
+  [ (builtins.throw "c") ]
+  (./xyz + (builtins.throw "p"))
+  # FIXME: test derivations and files.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-unsafe-discard-string-context-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-unsafe-discard-string-context-propagate-catchable.exp
new file mode 100644
index 000000000000..d181ebcff6c9
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-unsafe-discard-string-context-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false ]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-unsafe-discard-string-context-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-unsafe-discard-string-context-propagate-catchable.nix
new file mode 100644
index 000000000000..8ef5f35a1739
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-builtins-unsafe-discard-string-context-propagate-catchable.nix
@@ -0,0 +1,4 @@
+map (e: (builtins.tryEval (builtins.unsafeDiscardStringContext e)).success) [
+  (builtins.throw "a")
+  # FIXME: test derivations with throwables.
+]
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-catchable-double-throw.exp b/tvix/eval/src/tests/tvix_tests/eval-okay-catchable-double-throw.exp
new file mode 100644
index 000000000000..c508d5366f70
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-catchable-double-throw.exp
@@ -0,0 +1 @@
+false
diff --git a/tvix/eval/src/tests/tvix_tests/eval-okay-catchable-double-throw.nix b/tvix/eval/src/tests/tvix_tests/eval-okay-catchable-double-throw.nix
new file mode 100644
index 000000000000..7c2132b5028c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/eval-okay-catchable-double-throw.nix
@@ -0,0 +1 @@
+(builtins.tryEval (builtins.throw (builtins.throw "a"))).success
diff --git a/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-builtins-group-by-propagate-catchable.exp b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-builtins-group-by-propagate-catchable.exp
new file mode 100644
index 000000000000..48e341f4d85c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-builtins-group-by-propagate-catchable.exp
@@ -0,0 +1 @@
+[ false false false ]
diff --git a/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-builtins-group-by-propagate-catchable.nix b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-builtins-group-by-propagate-catchable.nix
new file mode 100644
index 000000000000..182601abb18c
--- /dev/null
+++ b/tvix/eval/src/tests/tvix_tests/notyetpassing/eval-okay-builtins-group-by-propagate-catchable.nix
@@ -0,0 +1,5 @@
+map (e: (builtins.tryEval e).success) [
+  (builtins.groupBy (builtins.throw "a") [ "" ])
+  (builtins.groupBy (x: true) (builtins.throw "b"))
+  (builtins.groupBy (_: builtins.throw "x") [ "" ])
+]
diff --git a/tvix/verify-lang-tests/default.nix b/tvix/verify-lang-tests/default.nix
index a6c494569ff0..276a57f12935 100644
--- a/tvix/verify-lang-tests/default.nix
+++ b/tvix/verify-lang-tests/default.nix
@@ -51,6 +51,7 @@ let
     "eval-okay-floor.nix" = [ nix ];
     "eval-okay-groupBy.nix" = [ nix ];
     "eval-okay-zipAttrsWith.nix" = [ nix ];
+    "eval-okay-builtins-group-by-propagate-catchable.nix" = [ nix ];
     # Comparable lists are not in Nix 2.3
     "eval-okay-sort.nix" = [ nix ];
     "eval-okay-compare-lists.nix" = [ nix ];
@@ -75,6 +76,12 @@ let
     # b72bc4a972fe568744d98b89d63adcd504cb586c
     "eval-okay-identifier-formatting.nix" = [ nix ];
 
+    # Different catchable behavior between nix 2.3 and 2.18
+    "eval-okay-builtins-map-propagate-catchable.nix" = [ nix_latest ];
+    "eval-okay-builtins-gen-list-propagate-catchable.nix" = [ nix_latest ];
+    "eval-okay-builtins-replace-strings-propagate-catchable.nix" =
+      [ nix_latest ];
+
     # TODO(sterni): support diffing working directory and home relative paths
     # like C++ Nix test suite (using string replacement).
     "eval-okay-path-antiquotation.nix" = true;