feat(nix/readTree): implement .skip-tree marker for subtree ignoring r/5862

With this change, readTree gains the ability to notice a `.skip-tree`
marker in addition to the `.skip-subtree` marker.

The behaviour of the new marker will completely ignore the folder that
the marker is located in (i.e. no node will be present for it in the
parent at all).

To make this work, the recursive function in readTree had to be
modified to return a sentinel value (noting that a tree has requested
to be skipped) which is then filtered out when constructing the list
of children. The actual `readTree` function is now a wrapper around
this inner, sentinel-yielding implementation which unwraps the result
set.

For obvious reasons, `.skip-tree` is not allowed at the top-level and
readTree will throw an error if it encounters it there.

Fixes: b/244
Change-Id: Ica731bc1af356e881fd3d31c7109f62ffd2762ea
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8185
Autosubmit: tazjin <tazjin@tvl.su>
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI

6 files changed, 76 insertions, 18 deletions

diff --git a/nix/readTree/README.md b/nix/readTree/README.md
index f8bbe2255e..5d430d1cfc 100644
--- a/nix/readTree/README.md
+++ b/nix/readTree/README.md
@@ -52,14 +52,17 @@ true;` attribute merged into it.
 `readTree` will follow any subdirectories of a tree and import all Nix files,
 with some exceptions:
 
+* If a folder contains a `default.nix` file, no *sibling* Nix files will be
+  imported - however children are traversed as normal.
+* If a folder contains a `default.nix` it is loaded and, if it
+  evaluates to a set, *merged* with the children. If it evaluates to
+  anything other than a set, else the children are *not traversed*.
+* A folder can opt out from readTree completely by containing a
+  `.skip-tree` file. The content of the file is not read. These
+  folders will be missing completely from the readTree structure.
 * A folder can declare that its children are off-limit by containing a
   `.skip-subtree` file. Since the content of the file is not checked, it can be
   useful to leave a note for a human in the file.
-* If a folder contains a `default.nix` file, no *sibling* Nix files will be
-  imported - however children are traversed as normal.
-* If a folder contains a `default.nix` it is loaded and, if it evaluates to a
-  set, *merged* with the children. If it evaluates to anything else the children
-  are *not traversed*.
 * The `default.nix` of the top-level folder on which readTree is
   called is **not** read to avoid infinite recursion (as, presumably,
   this file is where readTree itself is called).
diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix
index e243e85517..15680082cd 100644
--- a/nix/readTree/default.nix
+++ b/nix/readTree/default.nix
@@ -41,7 +41,8 @@ let
   readDirVisible = path:
     let
       children = readDir path;
-      isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
+      # skip hidden files, except for those that contain special instructions to readTree
+      isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != ".";
       names = filter isVisible (attrNames children);
     in
     listToAttrs (map
@@ -86,16 +87,39 @@ let
       pathType = builtins.typeOf importedFile;
     in
     if pathType != "lambda"
-    then builtins.throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }"
+    then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }"
     else importedFile (filter parts (argsWithPath args parts));
 
   nixFileName = file:
     let res = match "(.*)\\.nix" file;
     in if res == null then null else head res;
 
-  readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }:
+  # Internal implementation of readTree, which handles things like the
+  # skipping of trees and subtrees.
+  #
+  # This method returns an attribute sets with either of two shapes:
+  #
+  # { ok = ...; }    # a tree was read successfully
+  # { skip = true; } # a tree was skipped
+  #
+  # The higher-level `readTree` method assembles the final attribute
+  # set out of these results at the top-level, and the internal
+  # `children` implementation unwraps and processes nested trees.
+  readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }:
     let
       dir = readDirVisible initPath;
+
+      # Determine whether any part of this tree should be skipped.
+      #
+      # Adding a `.skip-subtree` file will still allow the import of
+      # the current node's "default.nix" file, but stop recursion
+      # there.
+      #
+      # Adding a `.skip-tree` file will completely ignore the folder
+      # in which this file is located.
+      skipTree = hasAttr ".skip-tree" dir;
+      skipSubtree = skipTree || hasAttr ".skip-subtree" dir;
+
       joinChild = c: initPath + ("/" + c);
 
       self =
@@ -103,19 +127,17 @@ let
         then { __readTree = [ ]; }
         else importFile args scopedArgs initPath parts argsFilter;
 
-      # Import subdirectories of the current one, unless the special
-      # `.skip-subtree` file exists which makes readTree ignore the
-      # children.
+      # Import subdirectories of the current one, unless any skip
+      # instructions exist.
       #
       # This file can optionally contain information on why the tree
       # should be ignored, but its content is not inspected by
       # readTree
       filterDir = f: dir."${f}" == "directory";
-      children = if hasAttr ".skip-subtree" dir then [ ] else
-      map
+      filteredChildren = map
         (c: {
           name = c;
-          value = readTree {
+          value = readTreeImpl {
             inherit argsFilter scopedArgs;
             args = args;
             initPath = (joinChild c);
@@ -125,9 +147,15 @@ let
         })
         (filter filterDir (attrNames dir));
 
+      # Remove skipped children from the final set, and unwrap the
+      # result set.
+      children =
+        if skipSubtree then [ ]
+        else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren);
+
       # Import Nix files
       nixFiles =
-        if hasAttr ".skip-subtree" dir then [ ]
+        if skipSubtree then [ ]
         else filter (f: f != null) (map nixFileName (attrNames dir));
       nixChildren = map
         (c:
@@ -154,9 +182,23 @@ let
       );
 
     in
-    if isAttrs nodeValue
-    then merge nodeValue (allChildren // (marker parts allChildren))
-    else nodeValue;
+    if skipTree
+    then { skip = true; }
+    else {
+      ok =
+        if isAttrs nodeValue
+        then merge nodeValue (allChildren // (marker parts allChildren))
+        else nodeValue;
+    };
+
+  # Top-level implementation of readTree itself.
+  readTree = args:
+    let
+      tree = readTreeImpl args;
+    in
+    if tree ? skip
+    then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!"
+    else tree.ok;
 
   # Helper function to fetch subtargets from a target. This is a
   # temporary helper to warn on the use of the `meta.targets`
diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix
index fcca141714..6f9eb02eff 100644
--- a/nix/readTree/tests/default.nix
+++ b/nix/readTree/tests/default.nix
@@ -41,6 +41,16 @@ let
   };
 
   traversal-logic = it "corresponds to the traversal logic in the README" [
+    (assertEq "skip-tree/a is read"
+      tree-tl.skip-tree.a
+      "a is read normally")
+    (assertEq "skip-tree does not contain b"
+      (builtins.attrNames tree-tl.skip-tree)
+      [ "__readTree" "__readTreeChildren" "a" ])
+    (assertEq "skip-tree children list does not contain b"
+      tree-tl.skip-tree.__readTreeChildren
+      [ "a" ])
+
     (assertEq "skip subtree default.nix is read"
       tree-tl.skip-subtree.but
       "the default.nix is still read")
diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix
new file mode 100644
index 0000000000..186488be3c
--- /dev/null
+++ b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix
@@ -0,0 +1 @@
+_: "a is read normally"
diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree
new file mode 100644
index 0000000000..34936b45d1
--- /dev/null
+++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree
@@ -0,0 +1 @@
+b subfolder should be skipped completely
diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix
new file mode 100644
index 0000000000..7903f8e95a
--- /dev/null
+++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix
@@ -0,0 +1 @@
+throw "b is skipped completely"