diff options
Diffstat (limited to 'nix/readTree')
-rw-r--r-- | nix/readTree/README.md | 25 | ||||
-rw-r--r-- | nix/readTree/default.nix | 331 | ||||
-rw-r--r-- | nix/readTree/tests/default.nix | 56 | ||||
-rw-r--r-- | nix/readTree/tests/test-marker/directory-marked/default.nix | 3 | ||||
-rw-r--r-- | nix/readTree/tests/test-marker/directory-marked/nested/default.nix | 3 | ||||
-rw-r--r-- | nix/readTree/tests/test-marker/file-children/one.nix | 3 | ||||
-rw-r--r-- | nix/readTree/tests/test-marker/file-children/two.nix | 3 | ||||
-rw-r--r-- | nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix | 1 | ||||
-rw-r--r-- | nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree | 1 | ||||
-rw-r--r-- | nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix | 1 |
10 files changed, 356 insertions, 71 deletions
diff --git a/nix/readTree/README.md b/nix/readTree/README.md index 138abbe30583..5d430d1cfced 100644 --- a/nix/readTree/README.md +++ b/nix/readTree/README.md @@ -52,14 +52,17 @@ true;` attribute merged into it. `readTree` will follow any subdirectories of a tree and import all Nix files, with some exceptions: +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it + evaluates to a set, *merged* with the children. If it evaluates to + anything other than a set, else the children are *not traversed*. +* A folder can opt out from readTree completely by containing a + `.skip-tree` file. The content of the file is not read. These + folders will be missing completely from the readTree structure. * A folder can declare that its children are off-limit by containing a `.skip-subtree` file. Since the content of the file is not checked, it can be useful to leave a note for a human in the file. -* If a folder contains a `default.nix` file, no *sibling* Nix files will be - imported - however children are traversed as normal. -* If a folder contains a `default.nix` it is loaded and, if it evaluates to a - set, *merged* with the children. If it evaluates to anything else the children - are *not traversed*. * The `default.nix` of the top-level folder on which readTree is called is **not** read to avoid infinite recursion (as, presumably, this file is where readTree itself is called). @@ -70,8 +73,16 @@ the tree as empty nodes (`{}`). ## Import structure -`readTree` is called with two parameters: The arguments to pass to all imports, -and the initial path at which to start the traversal. +`readTree` is called with an argument set containing a few parameters: + +* `path`: Initial path at which to start the traversal. +* `args`: Arguments to pass to all imports. +* `filter`: (optional) A function to filter the argument set on each + import based on the location in the tree. This can be used to, for + example, implement a "visibility" system inside of a tree. +* `scopedArgs`: (optional) An argument set that is passed to all + imported files via `builtins.scopedImport`. This will forcefully + override the given values in the import scope, use with care! The package headers in this repository follow the form `{ pkgs, ... }:` where `pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix index 4d5385921ee2..4a745ce33c13 100644 --- a/nix/readTree/default.nix +++ b/nix/readTree/default.nix @@ -2,31 +2,38 @@ # Copyright (c) 2020-2021 The TVL Authors # SPDX-License-Identifier: MIT # -# Provides a function to automatically read a a filesystem structure +# Provides a function to automatically read a filesystem structure # into a Nix attribute set. +# +# Called with an attribute set taking the following arguments: +# +# path: Path to a directory from which to start reading the tree. +# +# args: Argument set to pass to each imported file. +# +# filter: Function to filter `args` based on the tree location. This should +# be a function of the form `args -> location -> args`, where the +# location is a list of strings representing the path components of +# the current readTree target. Optional. { ... }: let inherit (builtins) attrNames - baseNameOf + concatMap concatStringsSep + elem + elemAt filter hasAttr head isAttrs - length listToAttrs map match readDir substring; - assertMsg = pred: msg: - if pred - then true - else builtins.trace msg false; - argsWithPath = args: parts: let meta.locatedAt = parts; in meta // (if isAttrs args then args else args meta); @@ -34,78 +41,286 @@ let readDirVisible = path: let children = readDir path; - isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + # skip hidden files, except for those that contain special instructions to readTree + isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != "."; names = filter isVisible (attrNames children); - in listToAttrs (map (name: { - inherit name; - value = children.${name}; - }) names); + in + listToAttrs (map + (name: { + inherit name; + value = children.${name}; + }) + names); - # Create a mark containing the location of this attribute. - marker = parts: { + # Create a mark containing the location of this attribute and + # a list of all child attribute names added by readTree. + marker = parts: children: { __readTree = parts; + __readTreeChildren = builtins.attrNames children; }; - # The marker is added to every set that was imported directly by - # readTree. - importWithMark = args: path: parts: + # Create a label from a target's tree location. + mkLabel = target: + let label = concatStringsSep "/" target.__readTree; + in if target ? __subtarget + then "${label}:${target.__subtarget}" + else label; + + # Merge two attribute sets, but place attributes in `passthru` via + # `overrideAttrs` for derivation targets that support it. + merge = a: b: + if a ? overrideAttrs + then + a.overrideAttrs + (prev: { + passthru = (prev.passthru or { }) // b; + }) + else a // b; + + # Import a file and enforce our calling convention + importFile = args: scopedArgs: path: parts: filter: let - importedFile = import path; + importedFile = + if scopedArgs != { } && builtins ? scopedImport # For tvix + then builtins.scopedImport scopedArgs path + else import path; pathType = builtins.typeOf importedFile; - imported = - assert assertMsg - (pathType == "lambda") - "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }"; - importedFile (argsWithPath args parts); - in if (isAttrs imported) - then imported // (marker parts) - else imported; + in + if pathType != "lambda" + then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + else importedFile (filter parts (argsWithPath args parts)); nixFileName = file: let res = match "(.*)\\.nix" file; in if res == null then null else head res; - readTree = { args, initPath, rootDir, parts }: + # Internal implementation of readTree, which handles things like the + # skipping of trees and subtrees. + # + # This method returns an attribute sets with either of two shapes: + # + # { ok = ...; } # a tree was read successfully + # { skip = true; } # a tree was skipped + # + # The higher-level `readTree` method assembles the final attribute + # set out of these results at the top-level, and the internal + # `children` implementation unwraps and processes nested trees. + readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: let dir = readDirVisible initPath; + + # Determine whether any part of this tree should be skipped. + # + # Adding a `.skip-subtree` file will still allow the import of + # the current node's "default.nix" file, but stop recursion + # there. + # + # Adding a `.skip-tree` file will completely ignore the folder + # in which this file is located. + skipTree = hasAttr ".skip-tree" dir; + skipSubtree = skipTree || hasAttr ".skip-subtree" dir; + joinChild = c: initPath + ("/" + c); - self = if rootDir - then { __readTree = []; } - else importWithMark args initPath parts; + self = + if rootDir + then { __readTree = [ ]; } + else importFile args scopedArgs initPath parts argsFilter; - # Import subdirectories of the current one, unless the special - # `.skip-subtree` file exists which makes readTree ignore the - # children. + # Import subdirectories of the current one, unless any skip + # instructions exist. # # This file can optionally contain information on why the tree # should be ignored, but its content is not inspected by # readTree filterDir = f: dir."${f}" == "directory"; - children = if hasAttr ".skip-subtree" dir then [] else map (c: { - name = c; - value = readTree { - args = args; - initPath = (joinChild c); - rootDir = false; - parts = (parts ++ [ c ]); - }; - }) (filter filterDir (attrNames dir)); + filteredChildren = map + (c: { + name = c; + value = readTreeImpl { + inherit argsFilter scopedArgs; + args = args; + initPath = (joinChild c); + rootDir = false; + parts = (parts ++ [ c ]); + }; + }) + (filter filterDir (attrNames dir)); + + # Remove skipped children from the final set, and unwrap the + # result set. + children = + if skipSubtree then [ ] + else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren); # Import Nix files - nixFiles = filter (f: f != null) (map nixFileName (attrNames dir)); - nixChildren = map (c: let p = joinChild (c + ".nix"); in { - name = c; - value = importWithMark args p (parts ++ [ c ]); - }) nixFiles; - in if dir ? "default.nix" - then (if isAttrs self then self // (listToAttrs children) else self) - else (listToAttrs (nixChildren ++ children) // (marker parts)); - -in { - __functor = _: args: initPath: readTree { - inherit args initPath; - rootDir = true; - parts = []; - }; + nixFiles = + if skipSubtree then [ ] + else filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map + (c: + let + p = joinChild (c + ".nix"); + childParts = parts ++ [ c ]; + imported = importFile args scopedArgs p childParts argsFilter; + in + { + name = c; + value = + if isAttrs imported + then merge imported (marker childParts { }) + else imported; + }) + nixFiles; + + nodeValue = if dir ? "default.nix" then self else { }; + + allChildren = listToAttrs ( + if dir ? "default.nix" + then children + else nixChildren ++ children + ); + + in + if skipTree + then { skip = true; } + else { + ok = + if isAttrs nodeValue + then merge nodeValue (allChildren // (marker parts allChildren)) + else nodeValue; + }; + + # Top-level implementation of readTree itself. + readTree = args: + let + tree = readTreeImpl args; + in + if tree ? skip + then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!" + else tree.ok; + + # Helper function to fetch subtargets from a target. This is a + # temporary helper to warn on the use of the `meta.targets` + # attribute, which is deprecated in favour of `meta.ci.targets`. + subtargets = node: + let targets = (node.meta.targets or [ ]) ++ (node.meta.ci.targets or [ ]); + in if node ? meta.targets then + builtins.trace '' + [1;31mWarning: The meta.targets attribute is deprecated. + + Please move the subtargets of //${mkLabel node} to the + meta.ci.targets attribute. + [0m + '' + targets else targets; + + # Function which can be used to find all readTree targets within an + # attribute set. + # + # This function will gather physical targets, that is targets which + # correspond directly to a location in the repository, as well as + # subtargets (specified in the meta.ci.targets attribute of a node). + # + # This can be used to discover targets for inclusion in CI + # pipelines. + # + # Called with the arguments: + # + # eligible: Function to determine whether the given derivation + # should be included in the build. + gather = eligible: node: + if node ? __readTree then + # Include the node itself if it is eligible. + (if eligible node then [ node ] else [ ]) + # Include eligible children of the node + ++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren) + # Include specified sub-targets of the node + ++ filter eligible (map + (k: (node."${k}" or { }) // { + # Keep the same tree location, but explicitly mark this + # node as a subtarget. + __readTree = node.__readTree; + __readTreeChildren = [ ]; + __subtarget = k; + }) + (subtargets node)) + else [ ]; + + # Determine whether a given value is a derivation. + # Copied from nixpkgs/lib for cases where lib is not available yet. + isDerivation = x: isAttrs x && x ? type && x.type == "derivation"; +in +{ + inherit gather mkLabel; + + __functor = _: + { path + , args + , filter ? (_parts: x: x) + , scopedArgs ? { } + }: + readTree { + inherit args scopedArgs; + argsFilter = filter; + initPath = path; + rootDir = true; + parts = [ ]; + }; + + # In addition to readTree itself, some functionality is exposed that + # is useful for users of readTree. + + # Create a readTree filter disallowing access to the specified + # top-level folder in the repository, except for specific exceptions + # specified by their (full) paths. + # + # Called with the arguments: + # + # folder: Name of the restricted top-level folder (e.g. 'experimental') + # + # exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]), + # which should be able to access the restricted folder. + # + # reason: Textual explanation for the restriction (included in errors) + restrictFolder = { folder, exceptions ? [ ], reason }: parts: args: + if (elemAt parts 0) == folder || elem parts exceptions + then args + else args // { + depot = args.depot // { + "${folder}" = throw '' + Access to targets under //${folder} is not permitted from + other repository paths. Specific exceptions are configured + at the top-level. + + ${reason} + At location: ${builtins.concatStringsSep "." parts} + ''; + }; + }; + + # This definition of fix is identical to <nixpkgs>.lib.fix, but is + # provided here for cases where readTree is used before nixpkgs can + # be imported. + # + # It is often required to create the args attribute set. + fix = f: let x = f x; in x; + + # Takes an attribute set and adds a meta.ci.targets attribute to it + # which contains all direct children of the attribute set which are + # derivations. + # + # Type: attrs -> attrs + drvTargets = attrs: + attrs // { + # preserve .meta from original attrs + meta = (attrs.meta or { }) // { + # preserve .meta.ci (except .targets) from original attrs + ci = (attrs.meta.ci or { }) // { + targets = builtins.filter + (x: isDerivation attrs."${x}") + (builtins.attrNames attrs); + }; + }; + }; } diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix index f3cab2844785..6f9eb02effb9 100644 --- a/nix/readTree/tests/default.nix +++ b/nix/readTree/tests/default.nix @@ -8,12 +8,15 @@ let assertThrows ; - tree-ex = depot.nix.readTree {} ./test-example; + tree-ex = depot.nix.readTree { + path = ./test-example; + args = { }; + }; example = it "corresponds to the README example" [ (assertEq "third_party attrset" (lib.isAttrs tree-ex.third_party - && (! lib.isDerivation tree-ex.third_party)) + && (! lib.isDerivation tree-ex.third_party)) true) (assertEq "third_party attrset other attribute" tree-ex.third_party.favouriteColour @@ -32,9 +35,22 @@ let "roquefort") ]; - tree-tl = depot.nix.readTree {} ./test-tree-traversal; + tree-tl = depot.nix.readTree { + path = ./test-tree-traversal; + args = { }; + }; traversal-logic = it "corresponds to the traversal logic in the README" [ + (assertEq "skip-tree/a is read" + tree-tl.skip-tree.a + "a is read normally") + (assertEq "skip-tree does not contain b" + (builtins.attrNames tree-tl.skip-tree) + [ "__readTree" "__readTreeChildren" "a" ]) + (assertEq "skip-tree children list does not contain b" + tree-tl.skip-tree.__readTreeChildren + [ "a" ]) + (assertEq "skip subtree default.nix is read" tree-tl.skip-subtree.but "the default.nix is still read") @@ -76,20 +92,48 @@ let "Picked up through the drv") (assertEq "default.nix drv is not changed by readTree" tree-tl.default-nix.can-be-drv - (import ./test-tree-traversal/default-nix/can-be-drv/default.nix {})) + (import ./test-tree-traversal/default-nix/can-be-drv/default.nix { })) ]; # these each call readTree themselves because the throws have to happen inside assertThrows wrong = it "cannot read these files and will complain" [ (assertThrows "this file is not a function" - (depot.nix.readTree {} ./test-wrong-not-a-function).not-a-function) + (depot.nix.readTree { + path = ./test-wrong-not-a-function; + args = { }; + }).not-a-function) # can’t test for that, assertThrows can’t catch this error # (assertThrows "this file is a function but doesn’t have dots" # (depot.nix.readTree {} ./test-wrong-no-dots).no-dots-in-function) ]; -in runTestsuite "readTree" [ + read-markers = depot.nix.readTree { + path = ./test-marker; + args = { }; + }; + + assertMarkerByPath = path: + assertEq "${lib.concatStringsSep "." path} is marked correctly" + (lib.getAttrFromPath path read-markers).__readTree + path; + + markers = it "marks nodes correctly" [ + (assertMarkerByPath [ "directory-marked" ]) + (assertMarkerByPath [ "directory-marked" "nested" ]) + (assertMarkerByPath [ "file-children" "one" ]) + (assertMarkerByPath [ "file-children" "two" ]) + (assertEq "nix file children are marked correctly" + read-markers.file-children.__readTreeChildren [ "one" "two" ]) + (assertEq "directory children are marked correctly" + read-markers.directory-marked.__readTreeChildren [ "nested" ]) + (assertEq "absence of children is marked" + read-markers.directory-marked.nested.__readTreeChildren [ ]) + ]; + +in +runTestsuite "readTree" [ example traversal-logic wrong + markers ] diff --git a/nix/readTree/tests/test-marker/directory-marked/default.nix b/nix/readTree/tests/test-marker/directory-marked/default.nix new file mode 100644 index 000000000000..5bd3e36b5397 --- /dev/null +++ b/nix/readTree/tests/test-marker/directory-marked/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +{ } diff --git a/nix/readTree/tests/test-marker/directory-marked/nested/default.nix b/nix/readTree/tests/test-marker/directory-marked/nested/default.nix new file mode 100644 index 000000000000..5bd3e36b5397 --- /dev/null +++ b/nix/readTree/tests/test-marker/directory-marked/nested/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +{ } diff --git a/nix/readTree/tests/test-marker/file-children/one.nix b/nix/readTree/tests/test-marker/file-children/one.nix new file mode 100644 index 000000000000..5bd3e36b5397 --- /dev/null +++ b/nix/readTree/tests/test-marker/file-children/one.nix @@ -0,0 +1,3 @@ +{ ... }: + +{ } diff --git a/nix/readTree/tests/test-marker/file-children/two.nix b/nix/readTree/tests/test-marker/file-children/two.nix new file mode 100644 index 000000000000..5bd3e36b5397 --- /dev/null +++ b/nix/readTree/tests/test-marker/file-children/two.nix @@ -0,0 +1,3 @@ +{ ... }: + +{ } diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix new file mode 100644 index 000000000000..186488be3c9b --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix @@ -0,0 +1 @@ +_: "a is read normally" diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree new file mode 100644 index 000000000000..34936b45d141 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree @@ -0,0 +1 @@ +b subfolder should be skipped completely diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix new file mode 100644 index 000000000000..7903f8e95a38 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix @@ -0,0 +1 @@ +throw "b is skipped completely" |