From 44a5e142009932f89f3cd7033a62a85b6ed0c49b Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Tue, 28 Feb 2023 13:41:17 +0300 Subject: feat(nix/readTree): implement .skip-tree marker for subtree ignoring With this change, readTree gains the ability to notice a `.skip-tree` marker in addition to the `.skip-subtree` marker. The behaviour of the new marker will completely ignore the folder that the marker is located in (i.e. no node will be present for it in the parent at all). To make this work, the recursive function in readTree had to be modified to return a sentinel value (noting that a tree has requested to be skipped) which is then filtered out when constructing the list of children. The actual `readTree` function is now a wrapper around this inner, sentinel-yielding implementation which unwraps the result set. For obvious reasons, `.skip-tree` is not allowed at the top-level and readTree will throw an error if it encounters it there. Fixes: b/244 Change-Id: Ica731bc1af356e881fd3d31c7109f62ffd2762ea Reviewed-on: https://cl.tvl.fyi/c/depot/+/8185 Autosubmit: tazjin Reviewed-by: flokli Tested-by: BuildkiteCI --- nix/readTree/README.md | 13 +++-- nix/readTree/default.nix | 68 +++++++++++++++++----- nix/readTree/tests/default.nix | 10 ++++ .../test-tree-traversal/skip-tree/a/default.nix | 1 + .../test-tree-traversal/skip-tree/b/.skip-tree | 1 + .../test-tree-traversal/skip-tree/b/default.nix | 1 + 6 files changed, 76 insertions(+), 18 deletions(-) create mode 100644 nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix create mode 100644 nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree create mode 100644 nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix diff --git a/nix/readTree/README.md b/nix/readTree/README.md index f8bbe2255e..5d430d1cfc 100644 --- a/nix/readTree/README.md +++ b/nix/readTree/README.md @@ -52,14 +52,17 @@ true;` attribute merged into it. `readTree` will follow any subdirectories of a tree and import all Nix files, with some exceptions: +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it + evaluates to a set, *merged* with the children. If it evaluates to + anything other than a set, else the children are *not traversed*. +* A folder can opt out from readTree completely by containing a + `.skip-tree` file. The content of the file is not read. These + folders will be missing completely from the readTree structure. * A folder can declare that its children are off-limit by containing a `.skip-subtree` file. Since the content of the file is not checked, it can be useful to leave a note for a human in the file. -* If a folder contains a `default.nix` file, no *sibling* Nix files will be - imported - however children are traversed as normal. -* If a folder contains a `default.nix` it is loaded and, if it evaluates to a - set, *merged* with the children. If it evaluates to anything else the children - are *not traversed*. * The `default.nix` of the top-level folder on which readTree is called is **not** read to avoid infinite recursion (as, presumably, this file is where readTree itself is called). diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix index e243e85517..15680082cd 100644 --- a/nix/readTree/default.nix +++ b/nix/readTree/default.nix @@ -41,7 +41,8 @@ let readDirVisible = path: let children = readDir path; - isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + # skip hidden files, except for those that contain special instructions to readTree + isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != "."; names = filter isVisible (attrNames children); in listToAttrs (map @@ -86,16 +87,39 @@ let pathType = builtins.typeOf importedFile; in if pathType != "lambda" - then builtins.throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" else importedFile (filter parts (argsWithPath args parts)); nixFileName = file: let res = match "(.*)\\.nix" file; in if res == null then null else head res; - readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + # Internal implementation of readTree, which handles things like the + # skipping of trees and subtrees. + # + # This method returns an attribute sets with either of two shapes: + # + # { ok = ...; } # a tree was read successfully + # { skip = true; } # a tree was skipped + # + # The higher-level `readTree` method assembles the final attribute + # set out of these results at the top-level, and the internal + # `children` implementation unwraps and processes nested trees. + readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: let dir = readDirVisible initPath; + + # Determine whether any part of this tree should be skipped. + # + # Adding a `.skip-subtree` file will still allow the import of + # the current node's "default.nix" file, but stop recursion + # there. + # + # Adding a `.skip-tree` file will completely ignore the folder + # in which this file is located. + skipTree = hasAttr ".skip-tree" dir; + skipSubtree = skipTree || hasAttr ".skip-subtree" dir; + joinChild = c: initPath + ("/" + c); self = @@ -103,19 +127,17 @@ let then { __readTree = [ ]; } else importFile args scopedArgs initPath parts argsFilter; - # Import subdirectories of the current one, unless the special - # `.skip-subtree` file exists which makes readTree ignore the - # children. + # Import subdirectories of the current one, unless any skip + # instructions exist. # # This file can optionally contain information on why the tree # should be ignored, but its content is not inspected by # readTree filterDir = f: dir."${f}" == "directory"; - children = if hasAttr ".skip-subtree" dir then [ ] else - map + filteredChildren = map (c: { name = c; - value = readTree { + value = readTreeImpl { inherit argsFilter scopedArgs; args = args; initPath = (joinChild c); @@ -125,9 +147,15 @@ let }) (filter filterDir (attrNames dir)); + # Remove skipped children from the final set, and unwrap the + # result set. + children = + if skipSubtree then [ ] + else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren); + # Import Nix files nixFiles = - if hasAttr ".skip-subtree" dir then [ ] + if skipSubtree then [ ] else filter (f: f != null) (map nixFileName (attrNames dir)); nixChildren = map (c: @@ -154,9 +182,23 @@ let ); in - if isAttrs nodeValue - then merge nodeValue (allChildren // (marker parts allChildren)) - else nodeValue; + if skipTree + then { skip = true; } + else { + ok = + if isAttrs nodeValue + then merge nodeValue (allChildren // (marker parts allChildren)) + else nodeValue; + }; + + # Top-level implementation of readTree itself. + readTree = args: + let + tree = readTreeImpl args; + in + if tree ? skip + then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!" + else tree.ok; # Helper function to fetch subtargets from a target. This is a # temporary helper to warn on the use of the `meta.targets` diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix index fcca141714..6f9eb02eff 100644 --- a/nix/readTree/tests/default.nix +++ b/nix/readTree/tests/default.nix @@ -41,6 +41,16 @@ let }; traversal-logic = it "corresponds to the traversal logic in the README" [ + (assertEq "skip-tree/a is read" + tree-tl.skip-tree.a + "a is read normally") + (assertEq "skip-tree does not contain b" + (builtins.attrNames tree-tl.skip-tree) + [ "__readTree" "__readTreeChildren" "a" ]) + (assertEq "skip-tree children list does not contain b" + tree-tl.skip-tree.__readTreeChildren + [ "a" ]) + (assertEq "skip subtree default.nix is read" tree-tl.skip-subtree.but "the default.nix is still read") diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix new file mode 100644 index 0000000000..186488be3c --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix @@ -0,0 +1 @@ +_: "a is read normally" diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree new file mode 100644 index 0000000000..34936b45d1 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree @@ -0,0 +1 @@ +b subfolder should be skipped completely diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix new file mode 100644 index 0000000000..7903f8e95a --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix @@ -0,0 +1 @@ +throw "b is skipped completely" -- cgit 1.4.1