diff options
Diffstat (limited to 'nix/readTree')
29 files changed, 522 insertions, 0 deletions
diff --git a/nix/readTree/README.md b/nix/readTree/README.md new file mode 100644 index 000000000000..f8bbe2255e5e --- /dev/null +++ b/nix/readTree/README.md @@ -0,0 +1,92 @@ +readTree +======== + +This is a Nix program that builds up an attribute set tree for a large +repository based on the filesystem layout. + +It is in fact the tool that lays out the attribute set of this repository. + +As an example, consider a root (`.`) of a repository and a layout such as: + +``` +. +├── third_party +│ ├── default.nix +│ └── rustpkgs +│ ├── aho-corasick.nix +│ └── serde.nix +└── tools + ├── cheddar + │ └── default.nix + └── roquefort.nix +``` + +When `readTree` is called on that tree, it will construct an attribute set with +this shape: + +```nix +{ + tools = { + cheddar = ...; + roquefort = ...; + }; + + third_party = { + # the `default.nix` of this folder might have had arbitrary other + # attributes here, such as this: + favouriteColour = "orange"; + + rustpkgs = { + aho-corasick = ...; + serde = ...; + }; + }; +} +``` + +Every imported Nix file that yields an attribute set will have a `__readTree = +true;` attribute merged into it. + +## Traversal logic + +`readTree` will follow any subdirectories of a tree and import all Nix files, +with some exceptions: + +* A folder can declare that its children are off-limit by containing a + `.skip-subtree` file. Since the content of the file is not checked, it can be + useful to leave a note for a human in the file. +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it evaluates to a + set, *merged* with the children. If it evaluates to anything else the children + are *not traversed*. +* The `default.nix` of the top-level folder on which readTree is + called is **not** read to avoid infinite recursion (as, presumably, + this file is where readTree itself is called). + +Traversal is lazy, `readTree` will only build up the tree as requested. This +currently has the downside that directories with no importable files end up in +the tree as empty nodes (`{}`). + +## Import structure + +`readTree` is called with an argument set containing a few parameters: + +* `path`: Initial path at which to start the traversal. +* `args`: Arguments to pass to all imports. +* `filter`: (optional) A function to filter the argument set on each + import based on the location in the tree. This can be used to, for + example, implement a "visibility" system inside of a tree. +* `scopedArgs`: (optional) An argument set that is passed to all + imported files via `builtins.scopedImport`. This will forcefully + override the given values in the import scope, use with care! + +The package headers in this repository follow the form `{ pkgs, ... }:` where +`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the +root of the depot). + +In theory `readTree` can pass arguments of different shapes, but I have found +this to be a good solution for the most part. + +Note that `readTree` does not currently make functions overridable, though it is +feasible that it could do that in the future. diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix new file mode 100644 index 000000000000..5468d41fd2c7 --- /dev/null +++ b/nix/readTree/default.nix @@ -0,0 +1,231 @@ +# Copyright (c) 2019 Vincent Ambo +# Copyright (c) 2020-2021 The TVL Authors +# SPDX-License-Identifier: MIT +# +# Provides a function to automatically read a a filesystem structure +# into a Nix attribute set. +# +# Called with an attribute set taking the following arguments: +# +# path: Path to a directory from which to start reading the tree. +# +# args: Argument set to pass to each imported file. +# +# filter: Function to filter `args` based on the tree location. This should +# be a function of the form `args -> location -> args`, where the +# location is a list of strings representing the path components of +# the current readTree target. Optional. +{ ... }: + +let + inherit (builtins) + attrNames + concatMap + concatStringsSep + elem + elemAt + filter + hasAttr + head + isAttrs + listToAttrs + map + match + readDir + substring; + + argsWithPath = args: parts: + let meta.locatedAt = parts; + in meta // (if isAttrs args then args else args meta); + + readDirVisible = path: + let + children = readDir path; + isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + names = filter isVisible (attrNames children); + in listToAttrs (map (name: { + inherit name; + value = children.${name}; + }) names); + + # Create a mark containing the location of this attribute and + # a list of all child attribute names added by readTree. + marker = parts: children: { + __readTree = parts; + __readTreeChildren = builtins.attrNames children; + }; + + # Import a file and enforce our calling convention + importFile = args: scopedArgs: path: parts: filter: + let + importedFile = if scopedArgs != {} + then builtins.scopedImport scopedArgs path + else import path; + pathType = builtins.typeOf importedFile; + in + if pathType != "lambda" + then builtins.throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + else importedFile (filter parts (argsWithPath args parts)); + + nixFileName = file: + let res = match "(.*)\\.nix" file; + in if res == null then null else head res; + + readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + let + dir = readDirVisible initPath; + joinChild = c: initPath + ("/" + c); + + self = if rootDir + then { __readTree = []; } + else importFile args scopedArgs initPath parts argsFilter; + + # Import subdirectories of the current one, unless the special + # `.skip-subtree` file exists which makes readTree ignore the + # children. + # + # This file can optionally contain information on why the tree + # should be ignored, but its content is not inspected by + # readTree + filterDir = f: dir."${f}" == "directory"; + children = if hasAttr ".skip-subtree" dir then [] else map (c: { + name = c; + value = readTree { + inherit argsFilter scopedArgs; + args = args; + initPath = (joinChild c); + rootDir = false; + parts = (parts ++ [ c ]); + }; + }) (filter filterDir (attrNames dir)); + + # Import Nix files + nixFiles = if hasAttr ".skip-subtree" dir then [] + else filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map (c: let + p = joinChild (c + ".nix"); + childParts = parts ++ [ c ]; + imported = importFile args scopedArgs p childParts argsFilter; + in { + name = c; + value = + if isAttrs imported + then imported // marker childParts {} + else imported; + }) nixFiles; + + nodeValue = if dir ? "default.nix" then self else {}; + + allChildren = listToAttrs ( + if dir ? "default.nix" + then children + else nixChildren ++ children + ); + + in + if isAttrs nodeValue + then nodeValue // allChildren // (marker parts allChildren) + else nodeValue; + + # Function can be used to find all readTree targets within an + # attribute set. + # + # This function will gather physical targets, that is targets which + # correspond directly to a location in the repository, as well as + # subtargets (specified in the meta.targets attribute of a node). + # + # This can be used to discover targets for inclusion in CI + # pipelines. + # + # Called with the arguments: + # + # eligible: Function to determine whether the given derivation + # should be included in the build. + gather = eligible: node: + if node ? __readTree then + # Include the node itself if it is eligible. + (if eligible node then [ node ] else []) + # Include eligible children of the node + ++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren) + # Include specified sub-targets of the node + ++ filter eligible (map + (k: (node."${k}" or {}) // { + # Keep the same tree location, but explicitly mark this + # node as a subtarget. + __readTree = node.__readTree; + __readTreeChildren = []; + __subtarget = k; + }) + (node.meta.targets or [])) + else []; + + # Determine whether a given value is a derivation. + # Copied from nixpkgs/lib for cases where lib is not available yet. + isDerivation = x: isAttrs x && x ? type && x.type == "derivation"; +in { + inherit gather; + + __functor = _: + { path + , args + , filter ? (_parts: x: x) + , scopedArgs ? {} }: + readTree { + inherit args scopedArgs; + argsFilter = filter; + initPath = path; + rootDir = true; + parts = []; + }; + + # In addition to readTree itself, some functionality is exposed that + # is useful for users of readTree. + + # Create a readTree filter disallowing access to the specified + # top-level folder in the repository, except for specific exceptions + # specified by their (full) paths. + # + # Called with the arguments: + # + # folder: Name of the restricted top-level folder (e.g. 'experimental') + # + # exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]), + # which should be able to access the restricted folder. + # + # reason: Textual explanation for the restriction (included in errors) + restrictFolder = { folder, exceptions ? [], reason }: parts: args: + if (elemAt parts 0) == folder || elem parts exceptions + then args + else args // { + depot = args.depot // { + "${folder}" = throw '' + Access to targets under //${folder} is not permitted from + other repository paths. Specific exceptions are configured + at the top-level. + + ${reason} + At location: ${builtins.concatStringsSep "." parts} + ''; + }; + }; + + # This definition of fix is identical to <nixpkgs>.lib.fix, but is + # provided here for cases where readTree is used before nixpkgs can + # be imported. + # + # It is often required to create the args attribute set. + fix = f: let x = f x; in x; + + # Takes an attribute set and adds a meta.targets attribute to it + # which contains all direct children of the attribute set which are + # derivations. + # + # Type: attrs -> attrs + drvTargets = attrs: attrs // { + meta = { + targets = builtins.filter + (x: isDerivation attrs."${x}") + (builtins.attrNames attrs); + } // (attrs.meta or {}); + }; +} diff --git a/nix/readTree/tests/.skip-subtree b/nix/readTree/tests/.skip-subtree new file mode 100644 index 000000000000..46952a4ca635 --- /dev/null +++ b/nix/readTree/tests/.skip-subtree @@ -0,0 +1 @@ +These tests call their own readTree, so the toplevel one shouldn’t bother diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix new file mode 100644 index 000000000000..3354a4fe5e75 --- /dev/null +++ b/nix/readTree/tests/default.nix @@ -0,0 +1,127 @@ +{ depot, lib, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + ; + + tree-ex = depot.nix.readTree { + path = ./test-example; + args = {}; + }; + + example = it "corresponds to the README example" [ + (assertEq "third_party attrset" + (lib.isAttrs tree-ex.third_party + && (! lib.isDerivation tree-ex.third_party)) + true) + (assertEq "third_party attrset other attribute" + tree-ex.third_party.favouriteColour + "orange") + (assertEq "rustpkgs attrset aho-corasick" + tree-ex.third_party.rustpkgs.aho-corasick + "aho-corasick") + (assertEq "rustpkgs attrset serde" + tree-ex.third_party.rustpkgs.serde + "serde") + (assertEq "tools cheddear" + "cheddar" + tree-ex.tools.cheddar) + (assertEq "tools roquefort" + tree-ex.tools.roquefort + "roquefort") + ]; + + tree-tl = depot.nix.readTree { + path = ./test-tree-traversal; + args = {}; + }; + + traversal-logic = it "corresponds to the traversal logic in the README" [ + (assertEq "skip subtree default.nix is read" + tree-tl.skip-subtree.but + "the default.nix is still read") + (assertEq "skip subtree a/default.nix is skipped" + (tree-tl.skip-subtree ? a) + false) + (assertEq "skip subtree b/c.nix is skipped" + (tree-tl.skip-subtree ? b) + false) + (assertEq "skip subtree a/default.nix would be read without .skip-subtree" + (tree-tl.no-skip-subtree.a) + "am I subtree yet?") + (assertEq "skip subtree b/c.nix would be read without .skip-subtree" + (tree-tl.no-skip-subtree.b.c) + "cool") + + (assertEq "default.nix attrset is merged with siblings" + tree-tl.default-nix.no + "siblings should be read") + (assertEq "default.nix means sibling isn’t read" + (tree-tl.default-nix ? sibling) + false) + (assertEq "default.nix means subdirs are still read and merged into default.nix" + (tree-tl.default-nix.subdir.a) + "but I’m picked up") + + (assertEq "default.nix can be not an attrset" + tree-tl.default-nix.no-merge + "I’m not merged with any children") + (assertEq "default.nix is not an attrset -> children are not merged" + (tree-tl.default-nix.no-merge ? subdir) + false) + + (assertEq "default.nix can contain a derivation" + (lib.isDerivation tree-tl.default-nix.can-be-drv) + true) + (assertEq "Even if default.nix is a derivation, children are traversed and merged" + tree-tl.default-nix.can-be-drv.subdir.a + "Picked up through the drv") + (assertEq "default.nix drv is not changed by readTree" + tree-tl.default-nix.can-be-drv + (import ./test-tree-traversal/default-nix/can-be-drv/default.nix {})) + ]; + + # these each call readTree themselves because the throws have to happen inside assertThrows + wrong = it "cannot read these files and will complain" [ + (assertThrows "this file is not a function" + (depot.nix.readTree { + path = ./test-wrong-not-a-function; + args = {}; + }).not-a-function) + # can’t test for that, assertThrows can’t catch this error + # (assertThrows "this file is a function but doesn’t have dots" + # (depot.nix.readTree {} ./test-wrong-no-dots).no-dots-in-function) + ]; + + read-markers = depot.nix.readTree { + path = ./test-marker; + args = {}; + }; + + assertMarkerByPath = path: + assertEq "${lib.concatStringsSep "." path} is marked correctly" + (lib.getAttrFromPath path read-markers).__readTree path; + + markers = it "marks nodes correctly" [ + (assertMarkerByPath [ "directory-marked" ]) + (assertMarkerByPath [ "directory-marked" "nested" ]) + (assertMarkerByPath [ "file-children" "one" ]) + (assertMarkerByPath [ "file-children" "two" ]) + (assertEq "nix file children are marked correctly" + read-markers.file-children.__readTreeChildren [ "one" "two" ]) + (assertEq "directory children are marked correctly" + read-markers.directory-marked.__readTreeChildren [ "nested" ]) + (assertEq "absence of children is marked" + read-markers.directory-marked.nested.__readTreeChildren [ ]) + ]; + +in runTestsuite "readTree" [ + example + traversal-logic + wrong + markers +] diff --git a/nix/readTree/tests/test-example/third_party/default.nix b/nix/readTree/tests/test-example/third_party/default.nix new file mode 100644 index 000000000000..27d87242189d --- /dev/null +++ b/nix/readTree/tests/test-example/third_party/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + favouriteColour = "orange"; +} diff --git a/nix/readTree/tests/test-example/third_party/rustpkgs/aho-corasick.nix b/nix/readTree/tests/test-example/third_party/rustpkgs/aho-corasick.nix new file mode 100644 index 000000000000..f964fa583c43 --- /dev/null +++ b/nix/readTree/tests/test-example/third_party/rustpkgs/aho-corasick.nix @@ -0,0 +1 @@ +{ ... }: "aho-corasick" diff --git a/nix/readTree/tests/test-example/third_party/rustpkgs/serde.nix b/nix/readTree/tests/test-example/third_party/rustpkgs/serde.nix new file mode 100644 index 000000000000..54b3c0503f12 --- /dev/null +++ b/nix/readTree/tests/test-example/third_party/rustpkgs/serde.nix @@ -0,0 +1 @@ +{ ... }: "serde" diff --git a/nix/readTree/tests/test-example/tools/cheddar/default.nix b/nix/readTree/tests/test-example/tools/cheddar/default.nix new file mode 100644 index 000000000000..a1919442d86d --- /dev/null +++ b/nix/readTree/tests/test-example/tools/cheddar/default.nix @@ -0,0 +1 @@ +{ ... }: "cheddar" diff --git a/nix/readTree/tests/test-example/tools/roquefort.nix b/nix/readTree/tests/test-example/tools/roquefort.nix new file mode 100644 index 000000000000..03c7492bb6b1 --- /dev/null +++ b/nix/readTree/tests/test-example/tools/roquefort.nix @@ -0,0 +1 @@ +{ ... }: "roquefort" diff --git a/nix/readTree/tests/test-marker/directory-marked/default.nix b/nix/readTree/tests/test-marker/directory-marked/default.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/directory-marked/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-marker/directory-marked/nested/default.nix b/nix/readTree/tests/test-marker/directory-marked/nested/default.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/directory-marked/nested/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-marker/file-children/one.nix b/nix/readTree/tests/test-marker/file-children/one.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/file-children/one.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-marker/file-children/two.nix b/nix/readTree/tests/test-marker/file-children/two.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/file-children/two.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/default.nix b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/default.nix new file mode 100644 index 000000000000..95d13d3c2750 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/default.nix @@ -0,0 +1,7 @@ +{ ... }: +derivation { + name = "im-a-drv"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" ''echo "" > $out'' ]; +} diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/subdir/a.nix b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/subdir/a.nix new file mode 100644 index 000000000000..2ee2d648f042 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/subdir/a.nix @@ -0,0 +1,3 @@ +{ ... }: + +"Picked up through the drv" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/default.nix b/nix/readTree/tests/test-tree-traversal/default-nix/default.nix new file mode 100644 index 000000000000..6003b5398305 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + no = "siblings should be read"; +} diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/default.nix b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/default.nix new file mode 100644 index 000000000000..c469533fbe5b --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +"I’m not merged with any children" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/subdir/a.nix b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/subdir/a.nix new file mode 100644 index 000000000000..2008c2d2411f --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/subdir/a.nix @@ -0,0 +1 @@ +"not accessible since parent default.nix is not an attrset" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/sibling.nix b/nix/readTree/tests/test-tree-traversal/default-nix/sibling.nix new file mode 100644 index 000000000000..8c57f2c16137 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/sibling.nix @@ -0,0 +1 @@ +"I’m left alone" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/subdir/a.nix b/nix/readTree/tests/test-tree-traversal/default-nix/subdir/a.nix new file mode 100644 index 000000000000..cf0ac2c8f3cf --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/subdir/a.nix @@ -0,0 +1,3 @@ +{ ... }: + +"but I’m picked up" diff --git a/nix/readTree/tests/test-tree-traversal/no-skip-subtree/a/default.nix b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/a/default.nix new file mode 100644 index 000000000000..a586ce534ce2 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/a/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +"am I subtree yet?" diff --git a/nix/readTree/tests/test-tree-traversal/no-skip-subtree/b/c.nix b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/b/c.nix new file mode 100644 index 000000000000..06216c417be9 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/b/c.nix @@ -0,0 +1,3 @@ +{ ... }: + +"cool" diff --git a/nix/readTree/tests/test-tree-traversal/no-skip-subtree/default.nix b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/default.nix new file mode 100644 index 000000000000..3d9f241cddb5 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + but = "the default.nix is still read"; +} diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/.skip-subtree b/nix/readTree/tests/test-tree-traversal/skip-subtree/.skip-subtree new file mode 100644 index 000000000000..87271ba5e101 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/.skip-subtree @@ -0,0 +1 @@ +this file makes subdirs be skipped, I hope diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-subtree/a/default.nix new file mode 100644 index 000000000000..a586ce534ce2 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/a/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +"am I subtree yet?" diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/b/c.nix b/nix/readTree/tests/test-tree-traversal/skip-subtree/b/c.nix new file mode 100644 index 000000000000..06216c417be9 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/b/c.nix @@ -0,0 +1,3 @@ +{ ... }: + +"cool" diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/default.nix b/nix/readTree/tests/test-tree-traversal/skip-subtree/default.nix new file mode 100644 index 000000000000..3d9f241cddb5 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + but = "the default.nix is still read"; +} diff --git a/nix/readTree/tests/test-wrong-no-dots/no-dots-in-function.nix b/nix/readTree/tests/test-wrong-no-dots/no-dots-in-function.nix new file mode 100644 index 000000000000..4681253af831 --- /dev/null +++ b/nix/readTree/tests/test-wrong-no-dots/no-dots-in-function.nix @@ -0,0 +1,3 @@ +{}: + +"This is a function, but readTree wants to pass a bunch of arguments, and not having dots means we depend on exactly which arguments." diff --git a/nix/readTree/tests/test-wrong-not-a-function/not-a-function.nix b/nix/readTree/tests/test-wrong-not-a-function/not-a-function.nix new file mode 100644 index 000000000000..f46ee2a35565 --- /dev/null +++ b/nix/readTree/tests/test-wrong-not-a-function/not-a-function.nix @@ -0,0 +1 @@ +"This file needs to be a function, otherwise readTree doesn’t like it!" |