diff options
Diffstat (limited to 'nix')
44 files changed, 1915 insertions, 480 deletions
diff --git a/nix/OWNERS b/nix/OWNERS index a742d0d22b..a640227914 100644 --- a/nix/OWNERS +++ b/nix/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - Profpatsch +Profpatsch diff --git a/nix/bufCheck/default.nix b/nix/bufCheck/default.nix index 039303ba68..ec98cfc376 100644 --- a/nix/bufCheck/default.nix +++ b/nix/bufCheck/default.nix @@ -1,9 +1,26 @@ -# Check protobuf syntax and breaking. +# Check protobuf breaking. Lints already happen in individual targets. # -{ depot, pkgs, ... }: +{ depot, pkgs, lib, ... }: -pkgs.writeShellScriptBin "ci-buf-check" '' - ${depot.third_party.bufbuild}/bin/buf check lint --input . - # Report-only - ${depot.third_party.bufbuild}/bin/buf check breaking --input "." --against-input "./.git#branch=canon" || true -'' +let + inherit (depot.nix) bufCheck;# self reference + + script = pkgs.writeShellScriptBin "ci-buf-check" '' + export PATH="$PATH:${pkgs.lib.makeBinPath [ pkgs.buf ]}" + # Report-only + (cd $(git rev-parse --show-toplevel) && (buf breaking . --against "./.git#ref=HEAD~1" || true)) + ''; +in + +script.overrideAttrs (old: { + meta = lib.recursiveUpdate old.meta { + # Protobuf check step executed in the buildkite pipeline which + # validates that changes to .proto files between revisions + # don't cause backwards-incompatible or otherwise flawed changes. + ci.extraSteps.protoCheck = { + alwaysRun = true; + label = ":water_buffalo: protoCheck"; + command = pkgs.writeShellScript "ci-buf-check-step" "exec ${depot.nix.bufCheck}/bin/ci-buf-check"; + }; + }; +}) diff --git a/nix/buildGo/README.md b/nix/buildGo/README.md index 37e0c06933..e9667c039a 100644 --- a/nix/buildGo/README.md +++ b/nix/buildGo/README.md @@ -2,8 +2,7 @@ buildGo.nix =========== This is an alternative [Nix][] build system for [Go][]. It supports building Go -libraries and programs, and even automatically generating Protobuf & gRPC -libraries. +libraries and programs. *Note:* This will probably end up being folded into [Nixery][]. @@ -33,7 +32,6 @@ Given a program layout like this: ├── lib <-- some library component │ ├── bar.go │ └── foo.go -├── api.proto <-- gRPC API definition ├── main.go <-- program implementation └── default.nix <-- build instructions ``` @@ -44,11 +42,6 @@ The contents of `default.nix` could look like this: { buildGo }: let - api = buildGo.grpc { - name = "someapi"; - proto = ./api.proto; - }; - lib = buildGo.package { name = "somelib"; srcs = [ @@ -58,7 +51,7 @@ let }; in buildGo.program { name = "my-program"; - deps = [ api lib ]; + deps = [ lib ]; srcs = [ ./main.go @@ -105,22 +98,6 @@ in buildGo.program { | `src` | `path` | Path to the source **directory** | yes | | `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no | - For some examples of how `buildGo.external` is used, check out - [`proto.nix`](./proto.nix). - -* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. - - | parameter | type | use | required? | - |-------------|-------------|--------------------------------------------------|-----------| - | `name` | `string` | Name for the resulting library | yes | - | `proto` | `path` | Path to the Protobuf definition file | yes | - | `path` | `string` | Import path for the resulting Go library | no | - | `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no | - -* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. - - The parameters are identical to `buildGo.proto`. - ## Current status This project is work-in-progress. Crucially it is lacking the following features: diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix index 92951b3cb2..c93642a127 100644 --- a/nix/buildGo/default.nix +++ b/nix/buildGo/default.nix @@ -22,7 +22,8 @@ let replaceStrings toString; - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; + inherit (pkgs) lib runCommand fetchFromGitHub protobuf symlinkJoin go; + goStdlib = buildStdlib go; # Helpers for low-level Go compiler invocations spaceOut = lib.concatStringsSep " "; @@ -41,8 +42,6 @@ let xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p); - # Add an `overrideGo` attribute to a function result that works # similar to `overrideAttrs`, but is used specifically for the # arguments passed to Go builders. @@ -50,16 +49,52 @@ let overrideGo = new: makeOverridable f (orig // (new orig)); }; + buildStdlib = go: runCommand "go-stdlib-${go.version}" + { + nativeBuildInputs = [ go ]; + } '' + HOME=$NIX_BUILD_TOP/home + mkdir $HOME + + goroot="$(go env GOROOT)" + cp -R "$goroot/src" "$goroot/pkg" . + + chmod -R +w . + GODEBUG=installgoroot=all GOROOT=$NIX_BUILD_TOP go install -v --trimpath std + + mkdir $out + cp -r pkg/*_*/* $out + + find $out -name '*.a' | while read -r ARCHIVE_FULL; do + ARCHIVE="''${ARCHIVE_FULL#"$out/"}" + PACKAGE="''${ARCHIVE%.a}" + echo "packagefile $PACKAGE=$ARCHIVE_FULL" + done > $out/importcfg + ''; + + importcfgCmd = { name, deps, out ? "importcfg" }: '' + echo "# nix buildGo ${name}" > "${out}" + cat "${goStdlib}/importcfg" >> "${out}" + ${lib.concatStringsSep "\n" (map (dep: '' + find "${dep}" -name '*.a' | while read -r pkgp; do + relpath="''${pkgp#"${dep}/"}" + pkgname="''${relpath%.a}" + echo "packagefile $pkgname=$pkgp" + done >> "${out}" + '') deps)} + ''; + # High-level build functions # Build a Go program out of the specified files and dependencies. program = { name, srcs, deps ? [ ], x_defs ? { } }: let uniqueDeps = allDeps (map (d: d.gopkg) deps); in runCommand name { } '' - ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} + ${importcfgCmd { inherit name; deps = uniqueDeps; }} + ${go}/bin/go tool compile -o ${name}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p main ${includeSources uniqueDeps} ${spaceOut srcs} mkdir -p $out/bin export GOROOT_FINAL=go - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a + ${go}/bin/go tool link -o $out/bin/${name} -importcfg=importcfg -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a ''; # Build a Go library assembled out of the specified files. @@ -76,8 +111,8 @@ let # This is required for several popular packages (e.g. x/sys). ifAsm = do: lib.optionalString (sfiles != [ ]) do; asmBuild = ifAsm '' - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} + ${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} ''; asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; asmPack = ifAsm '' @@ -88,7 +123,8 @@ let mkdir -p $out/${path} ${srcList path (map (s: "${s}") srcs)} ${asmBuild} - ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${importcfgCmd { inherit name; deps = uniqueDeps; }} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} ${asmPack} '').overrideAttrs (_: { passthru = { @@ -108,33 +144,14 @@ let # named "gopkg", and an attribute named "gobin" for binaries. external = import ./external { inherit pkgs program package; }; - # Import support libraries needed for protobuf & gRPC support - protoLibs = import ./proto.nix { - inherit external; - }; - - # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, goPackage ? name, extraDeps ? [ ] }: (makeOverridable package) { - inherit name path; - deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; - srcs = lib.singleton (runCommand "goproto-${name}.pb.go" { } '' - cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ - --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} - mv ./${goPackage}/*.pb.go $out - ''); - }; - - # Build a Go library out of the specified gRPC definition. - grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); - in { # Only the high-level builder functions are exposed, but made # overrideable. program = makeOverridable program; package = makeOverridable package; - proto = makeOverridable proto; - grpc = makeOverridable grpc; external = makeOverridable external; + + # re-expose the Go version used + inherit go; } diff --git a/nix/buildGo/example/default.nix b/nix/buildGo/example/default.nix index 08da075e18..6756bf39e2 100644 --- a/nix/buildGo/example/default.nix +++ b/nix/buildGo/example/default.nix @@ -19,13 +19,6 @@ let ]; }; - # Example use of buildGo.proto, which generates a Go library from a - # Protobuf definition file. - exampleProto = buildGo.proto { - name = "exampleproto"; - proto = ./thing.proto; - }; - # Example use of buildGo.program, which builds an executable using # the specified name and dependencies (which in turn must have been # created via buildGo.package etc.) @@ -39,7 +32,6 @@ buildGo.program { deps = [ examplePackage - exampleProto ]; x_defs = { diff --git a/nix/buildGo/example/thing.proto b/nix/buildGo/example/thing.proto deleted file mode 100644 index 0f6d6575e0..0000000000 --- a/nix/buildGo/example/thing.proto +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; -package example; - -message Thing { - string id = 1; - string kind_of_thing = 2; -} diff --git a/nix/buildGo/external/default.nix b/nix/buildGo/external/default.nix index f713783a58..42592c67e4 100644 --- a/nix/buildGo/external/default.nix +++ b/nix/buildGo/external/default.nix @@ -13,6 +13,7 @@ let readFile replaceStrings tail + unsafeDiscardStringContext throw; inherit (pkgs) lib runCommand go jq ripgrep; @@ -102,7 +103,9 @@ let analysisOutput = runCommand "${name}-structure.json" { } '' ${analyser}/bin/analyser -path ${path} -source ${src} > $out ''; - analysis = fromJSON (readFile analysisOutput); + # readFile adds the references of the read in file to the string context for + # Nix >= 2.6 which would break the attribute set construction in fromJSON + analysis = fromJSON (unsafeDiscardStringContext (readFile analysisOutput)); in lib.fix (self: foldl' lib.recursiveUpdate { } ( map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go index a77c43b371..4402a8eb86 100644 --- a/nix/buildGo/external/main.go +++ b/nix/buildGo/external/main.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "go/build" - "io/ioutil" "log" "os" "path" @@ -74,8 +73,8 @@ func findGoDirs(at string) ([]string, error) { } goDirs := []string{} - for k, _ := range dirSet { - goDirs = append(goDirs, k) + for goDir := range dirSet { + goDirs = append(goDirs, goDir) } return goDirs, nil @@ -148,7 +147,7 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk } func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { - f, err := ioutil.ReadFile(from) + f, err := os.ReadFile(from) if err != nil { return } diff --git a/nix/buildGo/proto.nix b/nix/buildGo/proto.nix deleted file mode 100644 index 6c37f758ce..0000000000 --- a/nix/buildGo/proto.nix +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# This file provides derivations for the dependencies of a gRPC -# service in Go. - -{ external }: - -let - inherit (builtins) fetchGit map; -in -rec { - goProto = external { - path = "github.com/golang/protobuf"; - src = fetchGit { - url = "https://github.com/golang/protobuf"; - rev = "ed6926b37a637426117ccab59282c3839528a700"; - }; - }; - - xnet = external { - path = "golang.org/x/net"; - - src = fetchGit { - url = "https://go.googlesource.com/net"; - rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; - }; - - deps = [ - xtext.secure.bidirule - xtext.unicode.bidi - xtext.unicode.norm - ]; - }; - - xsys = external { - path = "golang.org/x/sys"; - src = fetchGit { - url = "https://go.googlesource.com/sys"; - rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; - }; - }; - - xtext = external { - path = "golang.org/x/text"; - src = fetchGit { - url = "https://go.googlesource.com/text"; - rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; - }; - }; - - genproto = external { - path = "google.golang.org/genproto"; - src = fetchGit { - url = "https://github.com/google/go-genproto"; - # necessary because https://github.com/NixOS/nix/issues/1923 - ref = "main"; - rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; - }; - - deps = with goProto; [ - proto - ptypes.any - ]; - }; - - goGrpc = external { - path = "google.golang.org/grpc"; - deps = ([ - xnet.trace - xnet.http2 - xsys.unix - xnet.http2.hpack - genproto.googleapis.rpc.status - ] ++ (with goProto; [ - proto - ptypes - ptypes.duration - ptypes.timestamp - ])); - - src = fetchGit { - url = "https://github.com/grpc/grpc-go"; - rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; - }; - }; -} diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix index a8168334a7..0d68a2818b 100644 --- a/nix/buildLisp/default.nix +++ b/nix/buildLisp/default.nix @@ -8,7 +8,7 @@ let inherit (builtins) map elemAt match filter; - inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; + inherit (pkgs) lib runCommand makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; inherit (pkgs.stdenv) targetPlatform; # @@ -187,7 +187,7 @@ let lispNativeDeps = allNative native lispDeps; filteredSrcs = implFilter implementation srcs; in - runCommandNoCC name + runCommand name { LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; LANG = "C.UTF-8"; @@ -475,7 +475,7 @@ let } $@ ''; - bundled = name: runCommandNoCC "${name}-cllib" + bundled = name: runCommand "${name}-cllib" { passthru = { lispName = name; @@ -513,9 +513,8 @@ let # See https://ccl.clozure.com/docs/ccl.html#building-definitions faslExt = - /**/ - if targetPlatform.isPowerPC && targetPlatform.is32bit then "pfsl" - else if targetPlatform.isPowerPC && targetPlatform.is64bit then "p64fsl" + if targetPlatform.isPower && targetPlatform.is32bit then "pfsl" + else if targetPlatform.isPower && targetPlatform.is64bit then "p64fsl" else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl" else if targetPlatform.isx86_32 && targetPlatform.isLinux then "lx32fsl" else if targetPlatform.isAarch32 && targetPlatform.isLinux then "lafsl" @@ -640,7 +639,7 @@ let } else null; in - lib.fix (self: runCommandNoCC "${name}-cllib" + lib.fix (self: runCommand "${name}-cllib" { LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; LANG = "C.UTF-8"; @@ -707,7 +706,7 @@ let } else null; in - lib.fix (self: runCommandNoCC "${name}" + lib.fix (self: runCommand "${name}" { nativeBuildInputs = [ makeWrapper ]; LD_LIBRARY_PATH = libPath; diff --git a/nix/buildLisp/tests/argv0.nix b/nix/buildLisp/tests/argv0.nix index bc29337d06..ca5f2b9741 100644 --- a/nix/buildLisp/tests/argv0.nix +++ b/nix/buildLisp/tests/argv0.nix @@ -1,36 +1,58 @@ -{ depot, pkgs, ... }: - -depot.nix.buildLisp.program { - name = "argv0-test"; - - srcs = [ - (pkgs.writeText "argv0-test.lisp" '' - (defpackage :argv0-test (:use :common-lisp :uiop) (:export :main)) - (in-package :argv0-test) - - (defun main () - (format t "~A~%" (uiop:argv0))) - '') - ]; - - deps = [ - { - sbcl = depot.nix.buildLisp.bundled "uiop"; - default = depot.nix.buildLisp.bundled "asdf"; - } - ]; - - passthru.meta.ci = { - extraSteps.verify = { - label = "verify argv[0] output"; - needsOutput = true; - command = pkgs.writeShellScript "check-argv0" '' - set -eux - - for invocation in "$(pwd)/result/bin/argv0-test" "./result/bin/argv0-test"; do - test "$invocation" = "$("$invocation")" - done - ''; +{ depot, pkgs, lib, ... }: + +let + # Trivial test program that outputs argv[0] and exits + prog = + depot.nix.buildLisp.program { + name = "argv0-test"; + + srcs = [ + (pkgs.writeText "argv0-test.lisp" '' + (defpackage :argv0-test (:use :common-lisp :uiop) (:export :main)) + (in-package :argv0-test) + + (defun main () + (format t "~A~%" (uiop:argv0))) + '') + ]; + + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + ]; }; - }; -} + + # Extract verify argv[0] output for given buildLisp program + checkImplementation = prog: + pkgs.runCommand "check-argv0" { } '' + set -eux + + checkInvocation() { + invocation="$1" + test "$invocation" = "$("$invocation")" + } + + checkInvocation "${prog}/bin/argv0-test" + + cd ${prog} + checkInvocation "./bin/argv0-test" + + cd bin + checkInvocation ./argv0-test + + set +x + + touch "$out" + ''; + + inherit (prog.meta.ci) targets; +in + +(checkImplementation prog).overrideAttrs (_: { + # Wire up a subtarget all (active) non-default implementations + passthru = lib.genAttrs targets (name: checkImplementation prog.${name}); + + meta.ci = { inherit targets; }; +}) diff --git a/nix/buildManPages/OWNERS b/nix/buildManPages/OWNERS index f16dd105d7..2e95807063 100644 --- a/nix/buildManPages/OWNERS +++ b/nix/buildManPages/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni +sterni diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 6a0e9d246d..9abba9408a 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -11,11 +11,10 @@ let inherit (builtins) attrValues - concatMap + concatLists concatStringsSep - filter + elem foldl' - getEnv hasAttr hashString isNull @@ -23,70 +22,117 @@ let length listToAttrs mapAttrs - partition - pathExists toJSON unsafeDiscardStringContext; - inherit (pkgs) lib runCommandNoCC writeText; + inherit (pkgs) lib runCommand writeText; inherit (depot.nix.readTree) mkLabel; + + inherit (depot.nix) dependency-analyzer; in rec { - # Creates a Nix expression that yields the target at the specified - # location in the repository. - # - # This makes a distinction between normal targets (which physically - # exist in the repository) and subtargets (which are "virtual" - # targets exposed by a physical one) to make it clear in the build - # output which is which. - mkBuildExpr = target: + # Create a unique key for the buildkite pipeline based on the given derivation + # or drvPath. A consequence of using such keys is that every derivation may + # only be exposed as a single, unique step in the pipeline. + keyForDrv = drvOrPath: + let + drvPath = + if lib.isDerivation drvOrPath then drvOrPath.drvPath + else if lib.isString drvOrPath then drvOrPath + else builtins.throw "keyForDrv: expected string or derivation"; + + # Only use the drv hash to prevent escaping problems. Buildkite also has a + # limit of 100 characters on keys. + in + "drv-" + (builtins.substring 0 32 + (builtins.baseNameOf (unsafeDiscardStringContext drvPath)) + ); + + # Given an arbitrary attribute path generate a Nix expression which obtains + # this from the root of depot (assumed to be ./.). Attributes may be any + # Nix strings suitable as attribute names, not just Nix literal-safe strings. + mkBuildExpr = attrPath: let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; - targetExpr = foldl' descend "import ./. {}" target.__readTree; - subtargetExpr = descend targetExpr target.__subtarget; in - if target ? __subtarget then subtargetExpr else targetExpr; + foldl' descend "import ./. {}" attrPath; # Determine whether to skip a target if it has not diverged from the # HEAD branch. - shouldSkip = parentTargetMap: label: drvPath: + shouldSkip = { parentTargetMap ? { }, label, drvPath }: if (hasAttr label parentTargetMap) && parentTargetMap."${label}".drvPath == drvPath then "Target has not changed." else false; - # Create build command for a derivation target. - mkBuildCommand = target: drvPath: concatStringsSep " " [ + # Create build command for an attribute path pointing to a derivation. + mkBuildCommand = { attrPath, drvPath, outLink ? "result" }: concatStringsSep " " [ + # If the nix build fails, the Nix command's exit status should be used. + "set -o pipefail;" + # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on # at least one of its `outPath`s, so we need to discard the string context # if we don't want to build everything during pipeline construction. - "(nix-store --realise '${drvPath}' --add-root result --indirect && readlink result)" + # + # To make this more uniform with how nix-build(1) works, we call realpath(1) + # on nix-store(1)'s output since it has the habit of printing the path of the + # out link, not the store path. + "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs -r realpath)" # Since we don't gcroot the derivation files, they may be deleted by the # garbage collector. In that case we can reevaluate and build the attribute # using nix-build. - "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)" + "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr attrPath}' --show-trace --out-link '${outLink}')" ]; + # Attribute path of a target relative to the depot root. Needs to take into + # account whether the target is a physical target (which corresponds to a path + # in the filesystem) or the subtarget of a physical target. + targetAttrPath = target: + target.__readTree + ++ lib.optionals (target ? __subtarget) [ target.__subtarget ]; + + # Given a derivation (identified by drvPath) that is part of the list of + # targets passed to mkPipeline, determine all derivations that it depends on + # and are also part of the pipeline. Finally, return the keys of the steps + # that build them. This is used to populate `depends_on` in `mkStep`. + # + # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`. + getTargetPipelineDeps = targetDepMap: drvPath: + # Sanity check: We should only call this function on targets explicitly + # passed to mkPipeline. Thus it should have been passed as a “known” drv to + # dependency-analyzer. + assert targetDepMap.${drvPath}.known; + builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps; + # Create a pipeline step from a single target. - mkStep = headBranch: parentTargetMap: target: + mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }: let label = mkLabel target; drvPath = unsafeDiscardStringContext target.drvPath; - shouldSkip' = shouldSkip parentTargetMap; in { label = ":nix: " + label; - key = hashString "sha1" label; - skip = shouldSkip' label drvPath; - command = mkBuildCommand target drvPath; + key = keyForDrv target; + skip = shouldSkip { inherit label drvPath parentTargetMap; }; + command = mkBuildCommand { + attrPath = targetAttrPath target; + inherit drvPath; + }; env.READTREE_TARGET = label; + cancel_on_build_failing = cancelOnBuildFailing; # Add a dependency on the initial static pipeline step which # always runs. This allows build steps uploaded in batches to # start running before all batches have been uploaded. - depends_on = ":init:"; - }; + depends_on = [ ":init:" ] + ++ getTargetPipelineDeps targetDepMap drvPath + ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + } // lib.optionalAttrs (target ? meta.timeout) { + timeout_in_minutes = target.meta.timeout / 60; + # Additional arguments to set on the step. + # Keep in mind these *overwrite* existing step args, not extend. Use with caution. + } // lib.optionalAttrs (target ? meta.ci.buildkiteExtraStepArgs) target.meta.ci.buildkiteExtraStepArgs; # Helper function to inelegantly divide a list into chunks of at # most n elements. @@ -111,7 +157,7 @@ rec { }); }; - # Split the pipeline into chunks of at most 256 steps at once, which + # Split the pipeline into chunks of at most 192 steps at once, which # are uploaded sequentially. This is because of a limitation in the # Buildkite backend which struggles to process more than a specific # number of chunks at once. @@ -145,68 +191,102 @@ rec { # # Can be used for status reporting steps and the like. postBuildSteps ? [ ] + # The list of phases known by the current Buildkite + # pipeline. Dynamic pipeline chunks for each phase are uploaded + # to Buildkite on execution of static part of the + # pipeline. Phases selection is hard-coded in the static + # pipeline. + # + # Pipeline generation will fail when an extra step with + # unregistered phase is added. + # + # Common scenarios for different phase: + # - "build" - main phase for building all Nix targets + # - "release" - pushing artifacts to external repositories + # - "deploy" - updating external deployment configurations + , phases ? [ "build" "release" ] + # Build phases that are active for this invocation (i.e. their + # steps should be generated). + # + # This can be used to disable outputting parts of a pipeline if, + # for example, build and release phases are created in separate + # eval contexts. + # + # TODO(tazjin): Fail/warn if unknown phase is requested. + , activePhases ? phases + # Setting this attribute to true cancels dynamic pipeline steps + # as soon as the build is marked as failing. + # + # To enable this feature one should enable "Fail Fast" setting + # at Buildkite pipeline or on organization level. + , cancelOnBuildFailing ? false }: let - # Convert a target into all of its build and post-build steps, - # treated separately as they need to be in different chunks. + # List of phases to include. + enabledPhases = lib.intersectLists activePhases phases; + + # Is the 'build' phase included? This phase is treated specially + # because it always contains the plain Nix builds, and some + # logic/optimisation depends on knowing whether is executing. + buildEnabled = elem "build" enabledPhases; + + # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer. + targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets); + + # Convert a target into all of its steps, separated by build + # phase (as phases end up in different chunks). targetToSteps = target: let - step = mkStep headBranch parentTargetMap target; + mkStepArgs = { + inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing; + }; + step = mkStep mkStepArgs; # Same step, but with an override function applied. This is # used in mkExtraStep if the extra step needs to modify the # parent derivation somehow. # # Note that this will never affect the label. - overridable = f: mkStep headBranch parentTargetMap (f target); - - # Split build/post-build steps - splitExtraSteps = partition ({ postStep, ... }: postStep) - (attrValues (mapAttrs - (name: value: { - inherit name value; - postStep = (value ? prompt) || (value.postBuild or false); - }) + overridable = f: mkStep (mkStepArgs // { target = (f target); }); + + # Split extra steps by phase. + splitExtraSteps = lib.groupBy ({ phase, ... }: phase) + (attrValues (mapAttrs (normaliseExtraStep phases overridable) (target.meta.ci.extraSteps or { }))); - mkExtraStep' = { name, value, ... }: mkExtraStep overridable name value; - extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt - extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt + extraSteps = mapAttrs + (_: steps: + map (mkExtraStep (targetAttrPath target) buildEnabled) steps) + splitExtraSteps; in - { - buildSteps = [ step ] ++ extraBuildSteps; - postSteps = extraPostSteps; + if !buildEnabled then extraSteps + else extraSteps // { + build = [ step ] ++ (extraSteps.build or [ ]); }; - # Combine all target steps into separate build and post-build step lists. - steps = foldl' - (acc: t: { - buildSteps = acc.buildSteps ++ t.buildSteps; - postSteps = acc.postSteps ++ t.postSteps; - }) - { buildSteps = [ ]; postSteps = [ ]; } - (map targetToSteps drvTargets); - - buildSteps = - # Add build steps for each derivation target and their extra - # steps. - steps.buildSteps - - # Add additional steps (if set). - ++ additionalSteps; - - postSteps = - # Add post-build steps for each derivation target. - steps.postSteps - - # Add any globally defined post-build steps. - ++ postBuildSteps; - - buildChunks = pipelineChunks "build" buildSteps; - postBuildChunks = pipelineChunks "post" postSteps; - chunks = buildChunks ++ postBuildChunks; + # Combine all target steps into step lists per phase. + # + # TODO(tazjin): Refactor when configurable phases show up. + globalSteps = { + build = additionalSteps; + release = postBuildSteps; + }; + + phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists) + ((map targetToSteps drvTargets) ++ [ globalSteps ]); + + # Generate pipeline chunks for each phase. + chunks = foldl' + (acc: phase: + let phaseSteps = phasesWithSteps.${phase} or [ ]; in + if phaseSteps == [ ] + then acc + else acc ++ (pipelineChunks phase phaseSteps)) + [ ] + enabledPhases; + in - runCommandNoCC "buildkite-pipeline" { } '' + runCommand "buildkite-pipeline" { } '' mkdir $out echo "Generated ${toString (length chunks)} pipeline chunks" ${ @@ -226,9 +306,7 @@ rec { # Include the attrPath in the output to reconstruct the drv # without parsing the human-readable label. - attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ - target.__subtarget - ]; + attrPath = targetAttrPath target; }; }) drvTargets))); @@ -252,10 +330,6 @@ rec { # confirmation. These steps always run after the main build is # done and have no influence on CI status. # - # postBuild (optional): If set to true, this step will run after - # all primary build steps (that is, after status has been reported - # back to CI). - # # needsOutput (optional): If set to true, the parent derivation # will be built in the working directory before running the # command. Output will be available as 'result'. @@ -282,8 +356,8 @@ rec { steps = [ { - inherit (step) branches; inherit prompt; + branches = step.branches or [ ]; block = ":radio_button: Run ${label}? (from ${parent.env.READTREE_TARGET})"; } @@ -294,42 +368,124 @@ rec { ]; }; - # Create the Buildkite configuration for an extra step, optionally - # wrapping it in a gate group. - mkExtraStep = overridableParent: key: + # Validate and normalise extra step configuration before actually + # generating build steps, in order to use user-provided metadata + # during the pipeline generation. + normaliseExtraStep = phases: overridableParent: key: { command , label ? key - , prompt ? false , needsOutput ? false , parentOverride ? (x: x) , branches ? null , alwaysRun ? false - , postBuild ? false - }@cfg: + , prompt ? false + , softFail ? false + , phase ? "build" + , skip ? false + , agents ? null + }: let parent = overridableParent parentOverride; parentLabel = parent.env.READTREE_TARGET; + validPhase = lib.throwIfNot (elem phase phases) '' + In step '${label}' (from ${parentLabel}): + + Phase '${phase}' is not valid. + + Known phases: ${concatStringsSep ", " phases} + '' + phase; + in + { + inherit + alwaysRun + branches + command + key + label + needsOutput + parent + parentLabel + softFail + skip + agents; + + phase = validPhase; + + prompt = lib.throwIf (prompt != false && phase == "build") '' + In step '${label}' (from ${parentLabel}): + + The 'prompt' feature can not be used by steps in the "build" + phase, because CI builds should not be gated on manual human + approvals. + '' + prompt; + }; + + # Create the Buildkite configuration for an extra step, optionally + # wrapping it in a gate group. + mkExtraStep = parentAttrPath: buildEnabled: cfg: + let + # ATTN: needs to match an entry in .gitignore so that the tree won't get dirty + commandScriptLink = "nix-buildkite-extra-step-command-script"; + step = { - label = ":gear: ${label} (from ${parentLabel})"; - skip = if alwaysRun then false else parent.skip or false; - depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key; - branches = if branches != null then lib.concatStringsSep " " branches else null; + key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; + label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; + skip = + let + # When parent doesn't have skip attribute set, default to false + parentSkip = cfg.parent.skip or false; + # Extra step skip parameter can be string explaining the + # skip reason. + extraStepSkip = if builtins.isString cfg.skip then true else cfg.skip; + # Don't run if extra step is explicitly set to skip. If + # parameter is not set or equal to false, follow parent behavior. + skip' = if extraStepSkip then cfg.skip else parentSkip; + in + if cfg.alwaysRun then false else skip'; - command = pkgs.writeShellScript "${key}-script" '' + depends_on = lib.optional + (buildEnabled && !cfg.alwaysRun && !cfg.needsOutput) + cfg.parent.key; + + command = pkgs.writeShellScript "${cfg.key}-script" '' set -ueo pipefail - ${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"} - ${lib.optionalString needsOutput parent.command} - echo '+++ Running extra step command' - exec ${command} + ${lib.optionalString cfg.needsOutput + "echo '~~~ Preparing build output of ${cfg.parentLabel}'" + } + ${lib.optionalString cfg.needsOutput cfg.parent.command} + echo '--- Building extra step script' + command_script="$(${ + # Using command substitution in this way assumes the script drv only has one output + assert builtins.length cfg.command.outputs == 1; + mkBuildCommand { + # script is exposed at <parent>.meta.ci.extraSteps.<key>.command + attrPath = + parentAttrPath + ++ [ "meta" "ci" "extraSteps" cfg.key "command" ]; + drvPath = unsafeDiscardStringContext cfg.command.drvPath; + # make sure it doesn't conflict with result (from needsOutput) + outLink = commandScriptLink; + } + })" + echo '+++ Running extra step script' + exec "$command_script" ''; - }; + + soft_fail = cfg.softFail; + } // (lib.optionalAttrs (cfg.agents != null) { inherit (cfg) agents; }) + // (lib.optionalAttrs (cfg.branches != null) { + branches = lib.concatStringsSep " " cfg.branches; + }); in - if (isString prompt) + if (isString cfg.prompt) then mkGatedStep { - inherit step label parent prompt; + inherit step; + inherit (cfg) label parent prompt; } else step; } diff --git a/nix/buildkite/fetch-parent-targets.sh b/nix/buildkite/fetch-parent-targets.sh index 8afac1e5ec..08c2d1f3ab 100755 --- a/nix/buildkite/fetch-parent-targets.sh +++ b/nix/buildkite/fetch-parent-targets.sh @@ -2,43 +2,54 @@ set -ueo pipefail # Each Buildkite build stores the derivation target map as a pipeline -# artifact. This script determines the most appropriate commit (the -# fork point of the current chain from HEAD) and fetches the artifact. +# artifact. To reduce the amount of work done by CI, each CI build is +# diffed against the latest such derivation map found for the +# repository. # -# New builds can be based on HEAD before the pipeline for the last -# commit has finished, in which case it is possible that the fork -# point has no derivation map. To account for this, up to 3 commits -# prior to HEAD are also queried to find a map. +# Note that this does not take into account when the currently +# processing CL was forked off from the canonical branch, meaning that +# things like nixpkgs updates in between will cause mass rebuilds in +# any case. # # If no map is found, the failure mode is not critical: We simply # build all targets. +readonly REPO_ROOT=$(git rev-parse --show-toplevel) + : ${DRVMAP_PATH:=pipeline/drvmap.json} : ${BUILDKITE_TOKEN_PATH:=~/buildkite-token} -git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" - -FIRST=$(git merge-base FETCH_HEAD "${BUILDKITE_COMMIT}") -SECOND=$(git rev-parse "$FIRST~1") -THIRD=$(git rev-parse "$FIRST~2") - -function most_relevant_builds { +# Runs a fairly complex Buildkite GraphQL query that attempts to fetch all +# pipeline-gen steps from the default branch, as long as one appears within the +# last 50 builds or so. The query restricts build states to running or passed +# builds, which means that it *should* be unlikely that nothing is found. +# +# There is no way to filter this more loosely (e.g. by saying "any recent build +# matching these conditions"). +# +# The returned data structure is complex, and disassembled by a JQ script that +# first filters out all builds with no matching jobs (e.g. builds that are still +# in progress), and then filters those down to builds with artifacts, and then +# to drvmap artifacts specifically. +# +# If a recent drvmap was found, this returns its download URL. Otherwise, it +# returns the string "null". +function latest_drvmap_url { set -u curl 'https://graphql.buildkite.com/v1' \ --silent \ -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \ - -d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \ - jq -r '.data.pipeline.builds.edges[] | .node.uuid' + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(first: 50, branch: [\\\"%default\\\"], state: [RUNNING, PASSED]) { edges { node { jobs(passed: true, first: 1, type: [COMMAND], step: {key: [\\\"pipeline-gen\\\"]}) { edges { node { ... on JobTypeCommand { url artifacts { edges { node { downloadURL path }}}}}}}}}}}}\"}" | tee out.json | \ + jq -r '[.data.pipeline.builds.edges[] | select((.node.jobs.edges | length) > 0) | .node.jobs.edges[] | .node.artifacts[][] | select(.node.path == "pipeline/drvmap.json")][0].node.downloadURL' } -mkdir -p tmp -for build in $(most_relevant_builds); do - echo "Checking artifacts for build $build" - buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true +readonly DOWNLOAD_URL=$(latest_drvmap_url) - if [[ -f "tmp/${DRVMAP_PATH}" ]]; then - echo "Fetched target map from build ${build}" - mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json - break - fi -done +if [[ ${DOWNLOAD_URL} != "null" ]]; then + mkdir -p tmp + curl -o tmp/parent-target-map.json ${DOWNLOAD_URL} && echo "downloaded parent derivation map" \ + || echo "failed to download derivation map!" +else + echo "no derivation map found!" +fi diff --git a/nix/dependency-analyzer/default.nix b/nix/dependency-analyzer/default.nix new file mode 100644 index 0000000000..2ec8d7b5b9 --- /dev/null +++ b/nix/dependency-analyzer/default.nix @@ -0,0 +1,263 @@ +{ lib, depot, pkgs, ... }: + +let + inherit (builtins) unsafeDiscardStringContext appendContext; + + # + # Utilities + # + + # Determine all paths a derivation depends on, i.e. input derivations and + # files imported into the Nix store. + # + # Implementation for Nix < 2.6 is quite hacky at the moment. + # + # Type: str -> [str] + # + # TODO(sterni): clean this up and expose it + directDrvDeps = + let + getDeps = + if lib.versionAtLeast builtins.nixVersion "2.6" + then + # Since https://github.com/NixOS/nix/pull/1643, Nix apparently »preserves + # string context« through a readFile invocation. This has the side effect + # that it becomes possible to query the actual references a store path has. + # Not a 100% sure this is intended, but _very_ convenient for us here. + drvPath: + builtins.attrNames (builtins.getContext (builtins.readFile drvPath)) + else + # For Nix < 2.6 we have to rely on HACK, namely grepping for quoted + # store path references in the file. In the future this should be + # replaced by a proper derivation parser. + drvPath: builtins.concatLists ( + builtins.filter builtins.isList ( + builtins.split + "\"(${lib.escapeRegex builtins.storeDir}/[[:alnum:]+._?=-]+.drv)\"" + (builtins.readFile drvPath) + ) + ); + in + drvPath: + # if the passed path is not a derivation we can't necessarily get its + # dependencies, since it may not be representable as a Nix string due to + # NUL bytes, e.g. compressed patch files imported into the Nix store. + if builtins.match "^.+\\.drv$" drvPath == null + then [ ] + else getDeps drvPath; + + # Maps a list of derivation to the list of corresponding `drvPath`s. + # + # Type: [drv] -> [str] + drvsToPaths = drvs: + builtins.map (drv: builtins.unsafeDiscardOutputDependency drv.drvPath) drvs; + + # + # Calculate map of direct derivation dependencies + # + + # Create the dependency map entry for a given `drvPath` which mainly includes + # a list of other `drvPath`s it depends on. Additionally we store whether the + # derivation is `known`, i.e. part of the initial list of derivations we start + # generating the map from + # + # Type: bool -> string -> set + drvEntry = known: drvPath: + let + # key may not refer to a store path, … + key = unsafeDiscardStringContext drvPath; + # but we must read from the .drv file. + path = builtins.unsafeDiscardOutputDependency drvPath; + in + { + inherit key; + # trick so we can call listToAttrs directly on the result of genericClosure + name = key; + value = { + deps = directDrvDeps path; + inherit known; + }; + }; + + # Create an attribute set that maps every derivation in the combined + # dependency closure of the list of input derivation paths to every of their + # direct dependencies. Additionally every entry will have set their `known` + # attribute to `true` if it is in the list of input derivation paths. + # + # Type: [str] -> set + plainDrvDepMap = drvPaths: + builtins.listToAttrs ( + builtins.genericClosure { + startSet = builtins.map (drvEntry true) drvPaths; + operator = { value, ... }: builtins.map (drvEntry false) value.deps; + } + ); + + # + # Calculate closest known dependencies in the dependency map + # + + inherit (depot.nix.stateMonad) + after + bind + for_ + get + getAttr + run + setAttr + pure + ; + + # This is an action in stateMonad which expects the (initial) state to have + # been produced by `plainDrvDepMap`. Given a `drvPath`, it calculates a + # `knownDeps` list which holds the `drvPath`s of the closest derivation marked + # as `known` along every edge. This list is inserted into the dependency map + # for `drvPath` and every other derivation in its dependecy closure (unless + # the information was already present). This means that the known dependency + # information for a derivation never has to be recalculated, as long as they + # are part of the same stateful computation. + # + # The upshot is that after calling `insertKnownDeps drvPath`, + # `fmap (builtins.getAttr "knownDeps") (getAttr drvPath)` will always succeed. + # + # Type: str -> stateMonad drvDepMap null + insertKnownDeps = drvPathWithContext: + let + # We no longer need to read from the store, so context is irrelevant, but + # we need to check for attr names which requires the absence of context. + drvPath = unsafeDiscardStringContext drvPathWithContext; + in + bind get (initDepMap: + # Get the dependency map's state before we've done anything to obtain the + # entry we'll be manipulating later as well as its dependencies. + let + entryPoint = initDepMap.${drvPath}; + + # We don't need to recurse if our direct dependencies either have their + # knownDeps list already populated or are known dependencies themselves. + depsPrecalculated = + builtins.partition + (dep: + initDepMap.${dep}.known + || initDepMap.${dep} ? knownDeps + ) + entryPoint.deps; + + # If a direct dependency is known, it goes right to our known dependency + # list. If it is unknown, we can copy its knownDeps list into our own. + initiallyKnownDeps = + builtins.concatLists ( + builtins.map + (dep: + if initDepMap.${dep}.known + then [ dep ] + else initDepMap.${dep}.knownDeps + ) + depsPrecalculated.right + ); + in + + # If the information was already calculated before, we can exit right away + if entryPoint ? knownDeps + then pure null + else + after + # For all unknown direct dependencies which don't have a `knownDeps` + # list, we call ourselves recursively to populate it. Since this is + # done sequentially in the state monad, we avoid recalculating the + # list for the same derivation multiple times. + (for_ + depsPrecalculated.wrong + insertKnownDeps) + # After this we can obtain the updated dependency map which will have + # a `knownDeps` list for all our direct dependencies and update the + # entry for the input `drvPath`. + (bind + get + (populatedDepMap: + (setAttr drvPath (entryPoint // { + knownDeps = + lib.unique ( + initiallyKnownDeps + ++ builtins.concatLists ( + builtins.map + (dep: populatedDepMap.${dep}.knownDeps) + depsPrecalculated.wrong + ) + ); + })))) + ); + + # This function puts it all together and is exposed via `__functor`. + # + # For a list of `drvPath`s, calculate an attribute set which maps every + # `drvPath` to a set of the following form: + # + # { + # known = true /* if it is in the list of input derivation paths */; + # deps = [ + # /* list of derivation paths it depends on directly */ + # ]; + # knownDeps = [ + # /* list of the closest derivation paths marked as known this + # derivation depends on. + # */ + # ]; + # } + knownDrvDepMap = knownDrvPaths: + run + (plainDrvDepMap knownDrvPaths) + (after + (for_ + knownDrvPaths + insertKnownDeps) + get); + + # + # Other things based on knownDrvDepMap + # + + # Create a SVG visualizing `knownDrvDepMap`. Nodes are identified by derivation + # name, so multiple entries can be collapsed if they have the same name. + # + # Type: [drv] -> drv + knownDependencyGraph = name: drvs: + let + justName = drvPath: + builtins.substring + (builtins.stringLength builtins.storeDir + 1 + 32 + 1) + (builtins.stringLength drvPath) + (unsafeDiscardStringContext drvPath); + + gv = pkgs.writeText "${name}-dependency-analysis.gv" '' + digraph depot { + ${ + (lib.concatStringsSep "\n" + (lib.mapAttrsToList (name: value: + if !value.known then "" + else lib.concatMapStringsSep "\n" + (knownDep: " \"${justName name}\" -> \"${justName knownDep}\"") + value.knownDeps + ) + (depot.nix.dependency-analyzer ( + drvsToPaths drvs + )))) + } + } + ''; + in + + pkgs.runCommand "${name}-dependency-analysis.svg" + { + nativeBuildInputs = [ + pkgs.buildPackages.graphviz + ]; + } + "dot -Tsvg < ${gv} > $out"; +in + +{ + __functor = _: knownDrvDepMap; + + inherit knownDependencyGraph plainDrvDepMap drvsToPaths; +} diff --git a/nix/dependency-analyzer/examples/ci-targets.nix b/nix/dependency-analyzer/examples/ci-targets.nix new file mode 100644 index 0000000000..597abd4109 --- /dev/null +++ b/nix/dependency-analyzer/examples/ci-targets.nix @@ -0,0 +1,12 @@ +{ depot, lib, ... }: + +( + depot.nix.dependency-analyzer.knownDependencyGraph + "depot" + depot.ci.targets +).overrideAttrs (old: { + # Causes an infinite recursion via ci.targets otherwise + meta = lib.recursiveUpdate (old.meta or { }) { + ci.skip = true; + }; +}) diff --git a/nix/dependency-analyzer/examples/lisp.nix b/nix/dependency-analyzer/examples/lisp.nix new file mode 100644 index 0000000000..775eb9ab57 --- /dev/null +++ b/nix/dependency-analyzer/examples/lisp.nix @@ -0,0 +1,5 @@ +{ depot, lib, ... }: + +depot.nix.dependency-analyzer.knownDependencyGraph "3p-lisp" ( + builtins.filter lib.isDerivation (builtins.attrValues depot.third_party.lisp) +) diff --git a/nix/dependency-analyzer/tests/default.nix b/nix/dependency-analyzer/tests/default.nix new file mode 100644 index 0000000000..79ac127e92 --- /dev/null +++ b/nix/dependency-analyzer/tests/default.nix @@ -0,0 +1,36 @@ +{ depot, lib, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + assertEq + it + ; + + inherit (depot.nix.dependency-analyzer) + plainDrvDepMap + drvsToPaths + ; + + knownDrvs = drvsToPaths ( + builtins.filter lib.isDerivation (builtins.attrValues depot.third_party.lisp) + ); + exampleMap = plainDrvDepMap knownDrvs; + + # These will be needed to index into the attribute set which can't have context + # in the attribute names. + knownDrvsNoContext = builtins.map builtins.unsafeDiscardStringContext knownDrvs; +in + +runTestsuite "dependency-analyzer" [ + (it "checks plainDrvDepMap properties" [ + (assertEq "all known drvs are marked known" + (builtins.all (drv: exampleMap.${drv}.known) knownDrvsNoContext) + true) + (assertEq "no unknown drv is marked known" + (builtins.all (entry: !entry.known) ( + builtins.attrValues (builtins.removeAttrs exampleMap knownDrvsNoContext) + )) + true) + ]) +] diff --git a/nix/emptyDerivation/OWNERS b/nix/emptyDerivation/OWNERS index a742d0d22b..a640227914 100644 --- a/nix/emptyDerivation/OWNERS +++ b/nix/emptyDerivation/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - Profpatsch +Profpatsch diff --git a/nix/emptyDerivation/default.nix b/nix/emptyDerivation/default.nix index 8433984012..f808aa228d 100644 --- a/nix/emptyDerivation/default.nix +++ b/nix/emptyDerivation/default.nix @@ -1,10 +1,11 @@ -{ depot, pkgs, ... }: +{ depot, pkgs, localSystem, ... }: let emptyDerivation = import ./emptyDerivation.nix { inherit pkgs; inherit (pkgs) stdenv; inherit (depot.nix) getBins; + system = localSystem; }; tests = import ./tests.nix { diff --git a/nix/emptyDerivation/emptyDerivation.nix b/nix/emptyDerivation/emptyDerivation.nix index 772df96352..d7de7ccfbc 100644 --- a/nix/emptyDerivation/emptyDerivation.nix +++ b/nix/emptyDerivation/emptyDerivation.nix @@ -1,4 +1,4 @@ -{ stdenv, pkgs, getBins }: +{ stdenv, system, pkgs, getBins }: # The empty derivation. All it does is touch $out. # Basically the unit value for derivations. @@ -15,9 +15,7 @@ let emptiness = { name = "empty-derivation"; - - # TODO(Profpatsch): can we get system from tvl? - inherit (stdenv) system; + inherit system; builder = bins.exec; args = [ diff --git a/nix/lazy-deps/default.nix b/nix/lazy-deps/default.nix index 3cce48d8a5..fbdb30b38e 100644 --- a/nix/lazy-deps/default.nix +++ b/nix/lazy-deps/default.nix @@ -9,11 +9,11 @@ # evaluation, and expects both `git` and `nix-build` to exist in the # user's $PATH. If required, this can be done in the shell # configuration invoking this function. -{ pkgs, ... }: +{ pkgs, lib, ... }: let inherit (builtins) attrNames attrValues mapAttrs; - inherit (pkgs.lib) concatStringsSep; + inherit (lib) fix concatStringsSep; # Create the case statement for a command invocations, optionally # overriding the `TARGET_TOOL` variable. @@ -28,20 +28,21 @@ let invocations = tools: concatStringsSep "\n" (attrValues (mapAttrs invoke tools)); in +fix (self: # Attribute set of tools that should be lazily-added to the $PATH. - - # The name of each attribute is used as the command name (on $PATH). - # It must contain the keys 'attr' (containing the Nix attribute path - # to the tool's derivation from the top-level), and may optionally - # contain the key 'cmd' to override the name of the binary inside the - # derivation. +# +# The name of each attribute is used as the command name (on $PATH). +# It must contain the keys 'attr' (containing the Nix attribute path +# to the tool's derivation from the top-level), and may optionally +# contain the key 'cmd' to override the name of the binary inside the +# derivation. tools: -pkgs.writeTextFile { - name = "lazy-dispatch"; - executable = true; - destination = "/bin/__dispatch"; +pkgs.runCommandNoCC "lazy-dispatch" +{ + passthru.overrideDeps = newTools: self (tools // newTools); + passthru.tools = tools; text = '' #!${pkgs.runtimeShell} @@ -68,8 +69,23 @@ pkgs.writeTextFile { exec "''${TARGET_TOOL}" "''${@}" ''; - checkPhase = '' - ${pkgs.stdenv.shellDryRun} "$target" - ${concatStringsSep "\n" (map link (attrNames tools))} - ''; + # Access this to get a compatible nix-shell + passthru.devShell = pkgs.mkShellNoCC { + name = "${self.name}-shell"; + packages = [ self ]; + }; } + '' + # Write the dispatch code + target=$out/bin/__dispatch + mkdir -p "$(dirname "$target")" + echo "$text" > $target + chmod +x $target + + # Add symlinks from all the tools to the dispatch + ${concatStringsSep "\n" (map link (attrNames tools))} + + # Check that it's working-ish + ${pkgs.stdenv.shellDryRun} $target + '' +) diff --git a/nix/nint/OWNERS b/nix/nint/OWNERS index f16dd105d7..2e95807063 100644 --- a/nix/nint/OWNERS +++ b/nix/nint/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni +sterni diff --git a/nix/nix-1p/README.md b/nix/nix-1p/README.md new file mode 100644 index 0000000000..309eddb51e --- /dev/null +++ b/nix/nix-1p/README.md @@ -0,0 +1,648 @@ +> [!TIP] +> Are you interested in hacking on Nix projects for a week, together +> with other Nix users? Do you have time at the end of August? Great, +> come join us at [Volga Sprint](https://volgasprint.org/)! + +Nix - A One Pager +================= + +[Nix](https://nixos.org/nix/), the package manager, is built on and with Nix, +the language. This page serves as a fast intro to most of the (small) language. + +Unless otherwise specified, the word "Nix" refers only to the language below. + +Please file an issue if something in here confuses you or you think something +important is missing. + +If you have Nix installed, you can try the examples below by running `nix repl` +and entering code snippets there. + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Overview](#overview) +- [Language constructs](#language-constructs) + - [Primitives / literals](#primitives--literals) + - [Operators](#operators) + - [`//` (merge) operator](#-merge-operator) + - [Variable bindings](#variable-bindings) + - [Functions](#functions) + - [Multiple arguments (currying)](#multiple-arguments-currying) + - [Multiple arguments (attribute sets)](#multiple-arguments-attribute-sets) + - [`if ... then ... else ...`](#if--then--else-) + - [`inherit` keyword](#inherit-keyword) + - [`with` statements](#with-statements) + - [`import` / `NIX_PATH` / `<entry>`](#import--nix_path--entry) + - [`or` expressions](#or-expressions) +- [Standard libraries](#standard-libraries) + - [`builtins`](#builtins) + - [`pkgs.lib`](#pkgslib) + - [`pkgs` itself](#pkgs-itself) +- [Derivations](#derivations) +- [Nix Idioms](#nix-idioms) + - [File lambdas](#file-lambdas) + - [`callPackage`](#callpackage) + - [Overrides / Overlays](#overrides--overlays) + +<!-- markdown-toc end --> + + +# Overview + +Nix is: + +* **purely functional**. It has no concept of sequential steps being executed, + any dependency between operations is established by depending on *data* from + previous operations. + + Any valid piece of Nix code is an *expression* that returns a value. + + Evaluating a Nix expression *yields a single data structure*, it does not + execute a sequence of operations. + + Every Nix file evaluates to a *single expression*. +* **lazy**. It will only evaluate expressions when their result is actually + requested. + + For example, the builtin function `throw` causes evaluation to stop. + Entering the following expression works fine however, because we never + actually ask for the part of the structure that causes the `throw`. + + ```nix + let attrs = { a = 15; b = builtins.throw "Oh no!"; }; + in "The value of 'a' is ${toString attrs.a}" + ``` +* **purpose-built**. Nix only exists to be the language for Nix, the package + manager. While people have occasionally used it for other use-cases, it is + explicitly not a general-purpose language. + +# Language constructs + +This section describes the language constructs in Nix. It is a small language +and most of these should be self-explanatory. + +## Primitives / literals + +Nix has a handful of data types which can be represented literally in source +code, similar to many other languages. + +```nix +# numbers +42 +1.72394 + +# strings & paths +"hello" +./some-file.json + +# strings support interpolation +"Hello ${name}" + +# multi-line strings (common prefix whitespace is dropped) +'' +first line +second line +'' + +# lists (note: no commas!) +[ 1 2 3 ] + +# attribute sets (field access with dot syntax) +{ a = 15; b = "something else"; } + +# recursive attribute sets (fields can reference each other) +rec { a = 15; b = a * 2; } +``` + +## Operators + +Nix has several operators, most of which are unsurprising: + +| Syntax | Description | +|---------------------------|-----------------------------------------------------------------------------| +| `+`, `-`, `*`, `/` | Numerical operations | +| `+` | String concatenation | +| `++` | List concatenation | +| `==` | Equality | +| `>`, `>=`, `<`, `<=` | Ordering comparators | +| `&&` | Logical `AND` | +| <code>||</code> | Logical `OR` | +| `e1 -> e2` | Logical implication (i.e. <code>!e1 || e2</code>) | +| `!` | Boolean negation | +| `set.attr` | Access attribute `attr` in attribute set `set` | +| `set ? attribute` | Test whether attribute set contains an attribute | +| `left // right` | Merge `left` & `right` attribute sets, with the right set taking precedence | + + +### `//` (merge) operator + +The `//`-operator is used pervasively in Nix code. You should familiarise +yourself with it, as it is likely also the least familiar one. + +It merges the left and right attribute sets given to it: + +```nix +{ a = 1; } // { b = 2; } + +# yields { a = 1; b = 2; } +``` + +Values from the right side take precedence: + +```nix +{ a = "left"; } // { a = "right"; } + +# yields { a = "right"; } +``` + +The merge operator does *not* recursively merge attribute sets; + +```nix +{ a = { b = 1; }; } // { a = { c = 2; }; } + +# yields { a = { c = 2; }; } +``` + +Helper functions for recursive merging exist in the [`lib` library](#pkgslib). + +## Variable bindings + +Bindings in Nix are introduced locally via `let` expressions, which make some +variables available within a given scope. + +For example: + +```nix +let + a = 15; + b = 2; +in a * b + +# yields 30 +``` + +Variables are immutable. This means that after defining what `a` or `b` are, you +can not *modify* their value in the scope in which they are available. + +You can nest `let`-expressions to shadow variables. + +Variables are *not* available outside of the scope of the `let` expression. +There are no global variables. + +## Functions + +All functions in Nix are anonymous lambdas. This means that they are treated +just like data. Giving them names is accomplished by assigning them to +variables, or setting them as values in an attribute set (more on that below). + +``` +# simple function +# declaration is simply the argument followed by a colon +name: "Hello ${name}" +``` + +### Multiple arguments (currying) + +Technically any Nix function can only accept **one argument**. Sometimes +however, a function needs multiple arguments. This is achieved in Nix via +[currying][], which means to create a function with one argument, that returns a +function with another argument, that returns ... and so on. + +For example: + +```nix +name: age: "${name} is ${toString age} years old" +``` + +An additional benefit of this approach is that you can pass one parameter to a +curried function, and receive back a function that you can re-use (similar to +partial application): + +```nix +let + multiply = a: b: a * b; + doubleIt = multiply 2; # at this point we have passed in the value for 'a' and + # receive back another function that still expects 'b' +in + doubleIt 15 + +# yields 30 +``` + +### Multiple arguments (attribute sets) + +Another way of specifying multiple arguments to a function in Nix is to make it +accept an attribute set, which enables multiple other features: + +```nix +{ name, age }: "${name} is ${toString age} years old" +``` + +Using this method, we gain the ability to specify default arguments (so that +callers can omit them): + +```nix +{ name, age ? 42 }: "${name} is ${toString age} years old" + +``` + +Or in practice: + +```nix +let greeter = { name, age ? 42 }: "${name} is ${toString age} years old"; +in greeter { name = "Slartibartfast"; } + +# yields "Slartibartfast is 42 years old" +# (note: Slartibartfast is actually /significantly/ older) +``` + +Additionally we can introduce an ellipsis using `...`, meaning that we can +accept an attribute set as our input that contains more variables than are +needed for the function. + +```nix +let greeter = { name, age, ... }: "${name} is ${toString age} years old"; + person = { + name = "Slartibartfast"; + age = 42; + # the 'email' attribute is not expected by the 'greeter' function ... + email = "slartibartfast@magrath.ea"; + }; +in greeter person # ... but the call works due to the ellipsis. +``` + +Nix also supports binding the whole set of passed in attributes to a +parameter using the `@` syntax: + +```nix +let func = { name, age, ... }@args: builtins.attrNames args; +in func { + name = "Slartibartfast"; + age = 42; + email = "slartibartfast@magrath.ea"; +} + +# yields: [ "age" "email" "name" ] +``` + +**Warning:** Combining the `@` syntax with default arguments can lead +to surprising behaviour, as the passed attributes are bound verbatim. +This means that defaulted arguments are not included in the bound +attribute set: + +```nix +({ a ? 1, b }@args: args.a) { b = 1; } +# throws: error: attribute 'a' missing + +({ a ? 1, b }@args: args.a) { b = 1; a = 2; } +# => 2 +``` + +## `if ... then ... else ...` + +Nix has simple conditional support. Note that `if` is an **expression** in Nix, +which means that both branches must be specified. + +```nix +if someCondition +then "it was true" +else "it was false" +``` + +## `inherit` keyword + +The `inherit` keyword is used in attribute sets or `let` bindings to "inherit" +variables from the parent scope. + +In short, a statement like `inherit foo;` expands to `foo = foo;`. + +Consider this example: + +```nix +let + name = "Slartibartfast"; + # ... other variables +in { + name = name; # set the attribute set key 'name' to the value of the 'name' var + # ... other attributes +} +``` + +The `name = name;` line can be replaced with `inherit name;`: + +```nix +let + name = "Slartibartfast"; + # ... other variables +in { + inherit name; + # ... other attributes +} +``` + +This is often convenient, especially because inherit supports multiple variables +at the same time as well as "inheritance" from other attribute sets: + +```nix +{ + inherit name age; # equivalent to `name = name; age = age;` + inherit (otherAttrs) email; # equivalent to `email = otherAttrs.email`; +} +``` + +## `with` statements + +The `with` statement "imports" all attributes from an attribute set into +variables of the same name: + +```nix +let attrs = { a = 15; b = 2; }; +in with attrs; a + b # 'a' and 'b' become variables in the scope following 'with' +``` + +The scope of a `with`-"block" is the expression immediately following the +semicolon, i.e.: + +```nix +let attrs = { /* some attributes */ }; +in with attrs; (/* this is the scope of the `with` */) +``` + +## `import` / `NIX_PATH` / `<entry>` + +Nix files can import each other by using the builtin `import` function and a +literal path: + +```nix +# assuming there is a file lib.nix with some useful functions +let myLib = import ./lib.nix; +in myLib.usefulFunction 42 +``` + +The `import` function will read and evaluate the file, and return its Nix value. + +Nix files often begin with a function header to pass parameters into the rest of +the file, so you will often see imports of the form `import ./some-file { ... }`. + +Nix has a concept of a `NIX_PATH` (similar to the standard `PATH` environment +variable) which contains named aliases for file paths containing Nix +expressions. + +In a standard Nix installation, several [channels][] will be present (for +example `nixpkgs` or `nixos-unstable`) on the `NIX_PATH`. + +`NIX_PATH` entries can be accessed using the `<entry>` syntax, which simply +evaluates to their file path: + +```nix +<nixpkgs> +# might yield something like `/home/tazjin/.nix-defexpr/channels/nixpkgs` +``` + +This is commonly used to import from channels: + +```nix +let pkgs = import <nixpkgs> {}; +in pkgs.something +``` + +## `or` expressions + +Nix has a keyword called `or` which can be used to access a value from an +attribute set while providing a fallback to a default value. + +The syntax is simple: + +```nix +# Access an existing attribute +let set = { a = 42; }; +in set.a or 23 +``` + +Since the attribute `a` exists, this will return `42`. + + +```nix +# ... or fall back to a default if there is no such key +let set = { }; +in set.a or 23 +``` + +Since the attribute `a` does not exist, this will fall back to returning the +default value `23`. + +Note that `or` expressions also work for nested attribute set access. + +# Standard libraries + +Yes, libraries, plural. + +Nix has three major things that could be considered its standard library and +while there's a lot of debate to be had about this point, you still need to know +all three. + +## `builtins` + +Nix comes with several functions that are baked into the language. These work +regardless of which other Nix code you may or may not have imported. + +Most of these functions are implemented in the Nix interpreter itself, which +means that they are rather fast when compared to some of the equivalents which +are implemented in Nix itself. + +The Nix manual has [a section listing all `builtins`][builtins] and their usage. + +Examples of builtins that you will commonly encounter include, but are not +limited to: + +* `derivation` (see [Derivations](#derivations)) +* `toJSON` / `fromJSON` +* `toString` +* `toPath` / `fromPath` + +The builtins also include several functions that have the (spooky) ability to +break Nix' evaluation purity. No functions written in Nix itself can do this. + +Examples of those include: + +* `fetchGit` which can fetch a git-repository using the environment's default + git/ssh configuration +* `fetchTarball` which can fetch & extract archives without having to specify + hashes + +Read through the manual linked above to get the full overview. + +## `pkgs.lib` + +The Nix package set, commonly referred to by Nixers simply as [nixpkgs][], +contains a child attribute set called `lib` which provides a large number of +useful functions. + +The canonical definition of these functions is [their source code][lib-src]. I +wrote a tool ([nixdoc][]) in 2018 which generates manual entries for these +functions, however not all of the files are included as of July 2019. + +See the [Nixpkgs manual entry on `lib`][lib-manual] for the documentation. + +These functions include various utilities for dealing with the data types in Nix +(lists, attribute sets, strings etc.) and it is useful to at least skim through +them to familiarise yourself with what is available. + +```nix +{ pkgs ? import <nixpkgs> {} }: + +with pkgs.lib; # bring contents pkgs.lib into scope + +strings.toUpper "hello" + +# yields "HELLO" +``` + +## `pkgs` itself + +The Nix package set itself does not just contain packages, but also many useful +functions which you might run into while creating new Nix packages. + +One particular subset of these that stands out are the [trivial builders][], +which provide utilities for writing text files or shell scripts, running shell +commands and capturing their output and so on. + +```nix +{ pkgs ? import <nixpkgs> {} }: + +pkgs.writeText "hello.txt" "Hello dear reader!" + +# yields a derivation which creates a text file with the above content +``` + +# Derivations + +When a Nix expression is evaluated it may yield one or more *derivations*. +Derivations describe a single build action that, when run, places one or more +outputs (whether they be files or folders) in the Nix store. + +The builtin function `derivation` is responsible for creating derivations at a +lower level. Usually when Nix users create derivations they will use the +higher-level functions such as [stdenv.mkDerivation][smkd]. + +Please see the manual [on derivations][drv-manual] for more information, as the +general build logic is out of scope for this document. + +# Nix Idioms + +There are several idioms in Nix which are not technically part of the language +specification, but will commonly be encountered in the wild. + +This section is an (incomplete) list of them. + +## File lambdas + +It is customary to start every file with a function header that receives the +files dependencies, instead of importing them directly in the file. + +Sticking to this pattern lets users of your code easily change out, for example, +the specific version of `nixpkgs` that is used. + +A common file header pattern would look like this: + +```nix +{ pkgs ? import <nixpkgs> {} }: + +# ... 'pkgs' is then used in the code +``` + +In some sense, you might consider the function header of a file to be its "API". + +## `callPackage` + +Building on the previous pattern, there is a custom in nixpkgs of specifying the +dependencies of your file explicitly instead of accepting the entire package +set. + +For example, a file containing build instructions for a tool that needs the +standard build environment and `libsvg` might start like this: + +```nix +# my-funky-program.nix +{ stdenv, libsvg }: + +stdenv.mkDerivation { ... } +``` + +Any time a file follows this header pattern it is probably meant to be imported +using a special function called `callPackage` which is part of the top-level +package set (as well as certain subsets, such as `haskellPackages`). + +```nix +{ pkgs ? import <nixpkgs> {} }: + +let my-funky-program = pkgs.callPackage ./my-funky-program.nix {}; +in # ... something happens with my-funky-program +``` + +The `callPackage` function looks at the expected arguments (via +`builtins.functionArgs`) and passes the appropriate keys from the set in which +it is defined as the values for each corresponding argument. + +## Overrides / Overlays + +One of the most powerful features of Nix is that the representation of all build +instructions as data means that they can easily be *overridden* to get a +different result. + +For example, assuming there is a package `someProgram` which is built without +our favourite configuration flag (`--mimic-threaten-tag`) we might override it +like this: + +```nix +someProgram.overrideAttrs(old: { + configureFlags = old.configureFlags or [] ++ ["--mimic-threaten-tag"]; +}) +``` + +This pattern has a variety of applications of varying complexity. The top-level +package set itself can have an `overlays` argument passed to it which may add +new packages to the imported set. + +Note the use of the `or` operator to default to an empty list if the +original flags do not include `configureFlags`. This is required in +case a package does not set any flags by itself. + +Since this can change in a package over time, it is useful to guard +against it using `or`. + +For a slightly more advanced example, assume that we want to import `<nixpkgs>` +but have the modification above be reflected in the imported package set: + +```nix +let + overlay = (final: prev: { + someProgram = prev.someProgram.overrideAttrs(old: { + configureFlags = old.configureFlags or [] ++ ["--mimic-threaten-tag"]; + }); + }); +in import <nixpkgs> { overlays = [ overlay ]; } +``` + +The overlay function receives two arguments, `final` and `prev`. `final` is +the [fixed point][fp] of the overlay's evaluation, i.e. the package set +*including* the new packages and `prev` is the "original" package set. + +See the Nix manual sections [on overrides][] and [on overlays][] for more +details (note: the convention has moved away from using `self` in favor of +`final`, and `prev` instead of `super`, but the documentation has not been +updated to reflect this). + +[currying]: https://en.wikipedia.org/wiki/Currying +[builtins]: https://nixos.org/manual/nix/stable/language/builtins +[nixpkgs]: https://github.com/NixOS/nixpkgs +[lib-src]: https://github.com/NixOS/nixpkgs/tree/master/lib +[nixdoc]: https://github.com/tazjin/nixdoc +[lib-manual]: https://nixos.org/manual/nixpkgs/stable/#sec-functions-library +[channels]: https://nixos.org/manual/nix/stable/command-ref/files/channels +[trivial builders]: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/trivial-builders/default.nix +[smkd]: https://nixos.org/manual/nixpkgs/stable/#chap-stdenv +[drv-manual]: https://nixos.org/manual/nix/stable/language/derivations +[fp]: https://github.com/NixOS/nixpkgs/blob/master/lib/fixed-points.nix +[on overrides]: https://nixos.org/manual/nixpkgs/stable/#chap-overrides +[on overlays]: https://nixos.org/manual/nixpkgs/stable/#chap-overlays diff --git a/nix/nix-1p/default.nix b/nix/nix-1p/default.nix new file mode 100644 index 0000000000..6cc71b9548 --- /dev/null +++ b/nix/nix-1p/default.nix @@ -0,0 +1,16 @@ +# The canonical source location of nix-1p is //nix/nix-1p in the TVL +# depot: https://code.tvl.fyi/about/nix/nix-1p +# +# This file configures TVL CI to mirror the subtree to GitHub. +{ depot ? { }, pkgs ? import <nixpkgs> { }, ... }: + +(pkgs.runCommandLocal "nix-1p" { } '' + mkdir $out + cp ${./README.md} $out/README.md +'').overrideAttrs (_: { + meta.ci.extraSteps.github = depot.tools.releases.filteredGitPush { + filter = ":/nix/nix-1p"; + remote = "git@github.com:tazjin/nix-1p.git"; + ref = "refs/heads/master"; + }; +}) diff --git a/nix/readTree/README.md b/nix/readTree/README.md index f8bbe2255e..5d430d1cfc 100644 --- a/nix/readTree/README.md +++ b/nix/readTree/README.md @@ -52,14 +52,17 @@ true;` attribute merged into it. `readTree` will follow any subdirectories of a tree and import all Nix files, with some exceptions: +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it + evaluates to a set, *merged* with the children. If it evaluates to + anything other than a set, else the children are *not traversed*. +* A folder can opt out from readTree completely by containing a + `.skip-tree` file. The content of the file is not read. These + folders will be missing completely from the readTree structure. * A folder can declare that its children are off-limit by containing a `.skip-subtree` file. Since the content of the file is not checked, it can be useful to leave a note for a human in the file. -* If a folder contains a `default.nix` file, no *sibling* Nix files will be - imported - however children are traversed as normal. -* If a folder contains a `default.nix` it is loaded and, if it evaluates to a - set, *merged* with the children. If it evaluates to anything else the children - are *not traversed*. * The `default.nix` of the top-level folder on which readTree is called is **not** read to avoid infinite recursion (as, presumably, this file is where readTree itself is called). diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix index ba3363d8d6..4a745ce33c 100644 --- a/nix/readTree/default.nix +++ b/nix/readTree/default.nix @@ -2,7 +2,7 @@ # Copyright (c) 2020-2021 The TVL Authors # SPDX-License-Identifier: MIT # -# Provides a function to automatically read a a filesystem structure +# Provides a function to automatically read a filesystem structure # into a Nix attribute set. # # Called with an attribute set taking the following arguments: @@ -41,7 +41,8 @@ let readDirVisible = path: let children = readDir path; - isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + # skip hidden files, except for those that contain special instructions to readTree + isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != "."; names = filter isVisible (attrNames children); in listToAttrs (map @@ -80,22 +81,45 @@ let importFile = args: scopedArgs: path: parts: filter: let importedFile = - if scopedArgs != { } + if scopedArgs != { } && builtins ? scopedImport # For tvix then builtins.scopedImport scopedArgs path else import path; pathType = builtins.typeOf importedFile; in if pathType != "lambda" - then builtins.throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" else importedFile (filter parts (argsWithPath args parts)); nixFileName = file: let res = match "(.*)\\.nix" file; in if res == null then null else head res; - readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + # Internal implementation of readTree, which handles things like the + # skipping of trees and subtrees. + # + # This method returns an attribute sets with either of two shapes: + # + # { ok = ...; } # a tree was read successfully + # { skip = true; } # a tree was skipped + # + # The higher-level `readTree` method assembles the final attribute + # set out of these results at the top-level, and the internal + # `children` implementation unwraps and processes nested trees. + readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: let dir = readDirVisible initPath; + + # Determine whether any part of this tree should be skipped. + # + # Adding a `.skip-subtree` file will still allow the import of + # the current node's "default.nix" file, but stop recursion + # there. + # + # Adding a `.skip-tree` file will completely ignore the folder + # in which this file is located. + skipTree = hasAttr ".skip-tree" dir; + skipSubtree = skipTree || hasAttr ".skip-subtree" dir; + joinChild = c: initPath + ("/" + c); self = @@ -103,19 +127,17 @@ let then { __readTree = [ ]; } else importFile args scopedArgs initPath parts argsFilter; - # Import subdirectories of the current one, unless the special - # `.skip-subtree` file exists which makes readTree ignore the - # children. + # Import subdirectories of the current one, unless any skip + # instructions exist. # # This file can optionally contain information on why the tree # should be ignored, but its content is not inspected by # readTree filterDir = f: dir."${f}" == "directory"; - children = if hasAttr ".skip-subtree" dir then [ ] else - map + filteredChildren = map (c: { name = c; - value = readTree { + value = readTreeImpl { inherit argsFilter scopedArgs; args = args; initPath = (joinChild c); @@ -125,9 +147,15 @@ let }) (filter filterDir (attrNames dir)); + # Remove skipped children from the final set, and unwrap the + # result set. + children = + if skipSubtree then [ ] + else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren); + # Import Nix files nixFiles = - if hasAttr ".skip-subtree" dir then [ ] + if skipSubtree then [ ] else filter (f: f != null) (map nixFileName (attrNames dir)); nixChildren = map (c: @@ -154,9 +182,23 @@ let ); in - if isAttrs nodeValue - then merge nodeValue (allChildren // (marker parts allChildren)) - else nodeValue; + if skipTree + then { skip = true; } + else { + ok = + if isAttrs nodeValue + then merge nodeValue (allChildren // (marker parts allChildren)) + else nodeValue; + }; + + # Top-level implementation of readTree itself. + readTree = args: + let + tree = readTreeImpl args; + in + if tree ? skip + then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!" + else tree.ok; # Helper function to fetch subtargets from a target. This is a # temporary helper to warn on the use of the `meta.targets` diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix index fcca141714..6f9eb02eff 100644 --- a/nix/readTree/tests/default.nix +++ b/nix/readTree/tests/default.nix @@ -41,6 +41,16 @@ let }; traversal-logic = it "corresponds to the traversal logic in the README" [ + (assertEq "skip-tree/a is read" + tree-tl.skip-tree.a + "a is read normally") + (assertEq "skip-tree does not contain b" + (builtins.attrNames tree-tl.skip-tree) + [ "__readTree" "__readTreeChildren" "a" ]) + (assertEq "skip-tree children list does not contain b" + tree-tl.skip-tree.__readTreeChildren + [ "a" ]) + (assertEq "skip subtree default.nix is read" tree-tl.skip-subtree.but "the default.nix is still read") diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix new file mode 100644 index 0000000000..186488be3c --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix @@ -0,0 +1 @@ +_: "a is read normally" diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree new file mode 100644 index 0000000000..34936b45d1 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree @@ -0,0 +1 @@ +b subfolder should be skipped completely diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix new file mode 100644 index 0000000000..7903f8e95a --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix @@ -0,0 +1 @@ +throw "b is skipped completely" diff --git a/nix/renderMarkdown/default.nix b/nix/renderMarkdown/default.nix index 8d6b31cfcc..8759ada0fe 100644 --- a/nix/renderMarkdown/default.nix +++ b/nix/renderMarkdown/default.nix @@ -3,6 +3,19 @@ with depot.nix.yants; -defun [ path drv ] (file: pkgs.runCommandNoCC "${file}.rendered.html" { } '' - cat ${file} | ${depot.tools.cheddar}/bin/cheddar --about-filter ${file} > $out -'') +let + args = struct "args" { + path = path; + tagfilter = option bool; + }; +in +defun [ (either path args) drv ] + (arg: pkgs.runCommand "${arg.path or arg}.rendered.html" { } + ( + let + tagfilter = if (arg.tagfilter or true) then "" else "--no-tagfilter"; + in + '' + cat ${arg.path or arg} | ${depot.tools.cheddar}/bin/cheddar --about-filter ${tagfilter} ${arg.path or arg} > $out + '' + )) diff --git a/nix/sparseTree/OWNERS b/nix/sparseTree/OWNERS index fdf6d72040..2e95807063 100644 --- a/nix/sparseTree/OWNERS +++ b/nix/sparseTree/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni \ No newline at end of file +sterni diff --git a/nix/sparseTree/default.nix b/nix/sparseTree/default.nix index 16fc9b6103..35fa459e1c 100644 --- a/nix/sparseTree/default.nix +++ b/nix/sparseTree/default.nix @@ -2,22 +2,33 @@ # and directories if they are listed in a supplied list: # # # A very minimal depot -# sparseTree ./depot [ -# ./default.nix -# ./depot/nix/readTree/default.nix -# ./third_party/nixpkgs -# ./third_party/overlays -# ] +# sparseTree { +# root = ./depot; +# paths = [ +# ./default.nix +# ./depot/nix/readTree/default.nix +# ./third_party/nixpkgs +# ./third_party/overlays +# ]; +# } { pkgs, lib, ... }: -# root path to use as a reference point -root: -# list of paths below `root` that should be -# included in the resulting directory -# -# If path, need to refer to the actual file / directory to be included. -# If a string, it is treated as a string relative to the root. -paths: +{ + # root path to use as a reference point + root +, # list of paths below `root` that should be + # included in the resulting directory + # + # If path, need to refer to the actual file / directory to be included. + # If a string, it is treated as a string relative to the root. + paths +, # (optional) name to use for the derivation + # + # This should always be set when using roots that do not have + # controlled names, such as when passing the top-level of a git + # repository (e.g. `depot.path.origSrc`). + name ? builtins.baseNameOf root +}: let rootLength = builtins.stringLength (toString root); @@ -45,7 +56,6 @@ let let withLeading = p: if builtins.substring 0 1 p == "/" then p else "/" + p; fullPath = - /**/ if builtins.isPath path then path else if builtins.isString path then (root + withLeading path) else builtins.throw "Unsupported path type ${builtins.typeOf path}"; @@ -64,7 +74,7 @@ in # TODO(sterni): teach readTree to also read symlinked directories, # so we ln -sT instead of cp -aT. -pkgs.runCommandNoCC "sparse-${builtins.baseNameOf root}" { } ( +pkgs.runCommand "sparse-${name}" { } ( lib.concatMapStrings ({ src, dst }: '' mkdir -p "$(dirname "$out${dst}")" diff --git a/nix/stateMonad/default.nix b/nix/stateMonad/default.nix new file mode 100644 index 0000000000..209412e099 --- /dev/null +++ b/nix/stateMonad/default.nix @@ -0,0 +1,76 @@ +# Simple state monad represented as +# +# stateMonad s a = s -> { state : s; value : a } +# +{ ... }: + +rec { + # + # Monad + # + + # Type: stateMonad s a -> (a -> stateMonad s b) -> stateMonad s b + bind = action: f: state: + let + afterAction = action state; + in + (f afterAction.value) afterAction.state; + + # Type: stateMonad s a -> stateMonad s b -> stateMonad s b + after = action1: action2: state: action2 (action1 state).state; + + # Type: stateMonad s (stateMonad s a) -> stateMonad s a + join = action: bind action (action': action'); + + # Type: [a] -> (a -> stateMonad s b) -> stateMonad s null + for_ = xs: f: + builtins.foldl' + (laterAction: x: + after (f x) laterAction + ) + (pure null) + xs; + + # + # Applicative + # + + # Type: a -> stateMonad s a + pure = value: state: { inherit state value; }; + + # TODO(sterni): <*>, lift2, … + + # + # Functor + # + + # Type: (a -> b) -> stateMonad s a -> stateMonad s b + fmap = f: action: bind action (result: pure (f result)); + + # + # State Monad + # + + # Type: (s -> s) -> stateMonad s null + modify = f: state: { value = null; state = f state; }; + + # Type: stateMonad s s + get = state: { value = state; inherit state; }; + + # Type: s -> stateMonad s null + set = new: modify (_: new); + + # Type: str -> stateMonad set set.${str} + getAttr = attr: fmap (state: state.${attr}) get; + + # Type: str -> (any -> any) -> stateMonad s null + modifyAttr = attr: f: modify (state: state // { + ${attr} = f state.${attr}; + }); + + # Type: str -> any -> stateMonad s null + setAttr = attr: value: modifyAttr attr (_: value); + + # Type: s -> stateMonad s a -> a + run = state: action: (action state).value; +} diff --git a/nix/stateMonad/tests/default.nix b/nix/stateMonad/tests/default.nix new file mode 100644 index 0000000000..c3cb5c99b5 --- /dev/null +++ b/nix/stateMonad/tests/default.nix @@ -0,0 +1,110 @@ +{ depot, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + inherit (depot.nix.stateMonad) + pure + run + join + fmap + bind + get + set + modify + after + for_ + getAttr + setAttr + modifyAttr + ; + + runStateIndependent = run (throw "This should never be evaluated!"); +in + +runTestsuite "stateMonad" [ + (it "behaves correctly independent of state" [ + (assertEq "pure" (runStateIndependent (pure 21)) 21) + (assertEq "join pure" (runStateIndependent (join (pure (pure 42)))) 42) + (assertEq "fmap pure" (runStateIndependent (fmap (builtins.mul 2) (pure 21))) 42) + (assertEq "bind pure" (runStateIndependent (bind (pure 12) (x: pure x))) 12) + ]) + (it "behaves correctly with an integer state" [ + (assertEq "get" (run 42 get) 42) + (assertEq "after set get" (run 21 (after (set 42) get)) 42) + (assertEq "after modify get" (run 21 (after (modify (builtins.mul 2)) get)) 42) + (assertEq "fmap get" (run 40 (fmap (builtins.add 2) get)) 42) + (assertEq "stateful sum list" + (run 0 (after + (for_ + [ + 15 + 12 + 10 + 5 + ] + (x: modify (builtins.add x))) + get)) + 42) + ]) + (it "behaves correctly with an attr set state" [ + (assertEq "getAttr" (run { foo = 42; } (getAttr "foo")) 42) + (assertEq "after setAttr getAttr" + (run { foo = 21; } (after (setAttr "foo" 42) (getAttr "foo"))) + 42) + (assertEq "after modifyAttr getAttr" + (run { foo = 10.5; } + (after + (modifyAttr "foo" (builtins.mul 4)) + (getAttr "foo"))) + 42) + (assertEq "fmap getAttr" + (run { foo = 21; } (fmap (builtins.mul 2) (getAttr "foo"))) + 42) + (assertEq "after setAttr to insert getAttr" + (run { } (after (setAttr "foo" 42) (getAttr "foo"))) + 42) + (assertEq "insert permutations" + (run + { + a = 2; + b = 3; + c = 5; + } + (after + (bind get + (state: + let + names = builtins.attrNames state; + in + for_ names (name1: + for_ names (name2: + # this is of course a bit silly, but making it more cumbersome + # makes sure the test exercises more of the code. + (bind (getAttr name1) + (value1: + (bind (getAttr name2) + (value2: + setAttr "${name1}_${name2}" (value1 * value2))))))))) + get)) + { + a = 2; + b = 3; + c = 5; + a_a = 4; + a_b = 6; + a_c = 10; + b_a = 6; + b_b = 9; + b_c = 15; + c_c = 25; + c_a = 10; + c_b = 15; + } + ) + ]) +] diff --git a/nix/tag/default.nix b/nix/tag/default.nix index 0038404460..2955656323 100644 --- a/nix/tag/default.nix +++ b/nix/tag/default.nix @@ -78,7 +78,7 @@ let # Like `discrDef`, but fail if there is no match. discr = fs: v: let res = discrDef null fs v; in - assert lib.assertMsg (res != null) + assert lib.assertMsg (res != { }) "tag.discr: No predicate found that matches ${lib.generators.toPretty {} v}"; res; diff --git a/nix/tag/tests.nix b/nix/tag/tests.nix index bcc42c758a..e0085b4837 100644 --- a/nix/tag/tests.nix +++ b/nix/tag/tests.nix @@ -4,6 +4,7 @@ let inherit (depot.nix.runTestsuite) runTestsuite assertEq + assertThrows it ; @@ -50,6 +51,10 @@ let { int = lib.isInt; } ] "foo") { def = "foo"; }) + (assertThrows "throws failing to match" + (discr [ + { fish = x: x == 42; } + ] 21)) ]; match-test = it "can match things" [ diff --git a/nix/utils/OWNERS b/nix/utils/OWNERS index f16dd105d7..2e95807063 100644 --- a/nix/utils/OWNERS +++ b/nix/utils/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni +sterni diff --git a/nix/utils/default.nix b/nix/utils/default.nix index cabea5bbee..0c6c88fafd 100644 --- a/nix/utils/default.nix +++ b/nix/utils/default.nix @@ -53,13 +53,7 @@ let * `regular`: is a regular file, always `true` if returned * `directory`: is a directory, always `true` if returned * `missing`: path does not exist, always `true` if returned - * `symlink`: path is a symlink, value is a string describing the type - of its realpath which may be either: - - * `"directory"`: realpath of the symlink is a directory - * `"regular-or-missing`": realpath of the symlink is either a regular - file or does not exist. Due to limitations of the Nix expression - language, we can't tell which. + * `symlink`: path is a symlink, always `true` if returned Type: path(-like) -> tag @@ -73,10 +67,10 @@ let => { directory = true; } pathType ./result - => { symlink = "directory"; } + => { symlink = true; } pathType ./link-to-file - => { symlink = "regular-or-missing"; } + => { symlink = true; } pathType /does/not/exist => { missing = true; } @@ -90,12 +84,12 @@ let # Match on the result using //nix/tag nix.tag.match (nix.utils.pathType ./result) { - symlink = v: "symlink to ${v}"; + symlink = _: "symlink"; directory = _: "directory"; regular = _: "regular"; missing = _: "path does not exist"; } - => "symlink to directory" + => "symlink" # Query path type nix.tag.tagName (pathType /path) @@ -122,11 +116,7 @@ let isSymlinkDir = builtins.pathExists (path' + "/."); in { - ${thisPathType} = - /**/ - if thisPathType != "symlink" then true - else if isSymlinkDir then "directory" - else "regular-or-missing"; + ${thisPathType} = true; }; pathType' = path: @@ -144,21 +134,6 @@ let */ isDirectory = path: pathType' path ? directory; - /* Checks whether the given path is a directory or - a symlink to a directory. Throws if the path in - question doesn't exist. - - Warning: Does not throw if the target file or - directory doesn't exist, but the symlink does. - - Type: path(-like) -> bool - */ - realPathIsDirectory = path: - let - pt = pathType' path; - in - pt ? directory || pt.symlink or null == "directory"; - /* Check whether the given path is a regular file. Throws if the path in question doesn't exist. @@ -179,7 +154,6 @@ in storePathName pathType isDirectory - realPathIsDirectory isRegularFile isSymlink ; diff --git a/nix/utils/tests/default.nix b/nix/utils/tests/default.nix index 52b7ca41d2..344a1771d7 100644 --- a/nix/utils/tests/default.nix +++ b/nix/utils/tests/default.nix @@ -11,7 +11,6 @@ let inherit (depot.nix.utils) isDirectory - realPathIsDirectory isRegularFile isSymlink pathType @@ -34,16 +33,6 @@ let (assertUtilsPred "file not isDirectory" (isDirectory ./directory/file) false) - # realPathIsDirectory - (assertUtilsPred "directory realPathIsDirectory" - (realPathIsDirectory ./directory) - true) - (assertUtilsPred "symlink to directory realPathIsDirectory" - (realPathIsDirectory ./symlink-directory) - true) - (assertUtilsPred "realPathIsDirectory resolves chained symlinks" - (realPathIsDirectory ./symlink-symlink-directory) - true) # isRegularFile (assertUtilsPred "file isRegularFile" (isRegularFile ./directory/file) @@ -76,27 +65,12 @@ let # missing files throw (assertThrows "isDirectory throws on missing file" (isDirectory ./does-not-exist)) - (assertThrows "realPathIsDirectory throws on missing file" - (realPathIsDirectory ./does-not-exist)) (assertThrows "isRegularFile throws on missing file" (isRegularFile ./does-not-exist)) (assertThrows "isSymlink throws on missing file" (isSymlink ./does-not-exist)) ]); - symlinkPathTypeTests = it "correctly judges symlinks" [ - (assertEq "symlinks to directories are detected correcty" - ((pathType ./symlink-directory).symlink or null) "directory") - (assertEq "symlinks to symlinks to directories are detected correctly" - ((pathType ./symlink-symlink-directory).symlink or null) "directory") - (assertEq "symlinks to files are detected-ish" - ((pathType ./symlink-file).symlink or null) "regular-or-missing") - (assertEq "symlinks to symlinks to files are detected-ish" - ((pathType ./symlink-symlink-file).symlink or null) "regular-or-missing") - (assertEq "symlinks to nowhere are not distinguished from files" - ((pathType ./missing).symlink or null) "regular-or-missing") - ]; - cheddarStorePath = builtins.unsafeDiscardStringContext depot.tools.cheddar.outPath; @@ -121,6 +95,5 @@ in runTestsuite "nix.utils" [ pathPredicates - symlinkPathTypeTests storePathNameTests ] diff --git a/nix/writeTree/OWNERS b/nix/writeTree/OWNERS new file mode 100644 index 0000000000..b381c4e660 --- /dev/null +++ b/nix/writeTree/OWNERS @@ -0,0 +1 @@ +aspen diff --git a/nix/writeTree/default.nix b/nix/writeTree/default.nix new file mode 100644 index 0000000000..0c7c2a130f --- /dev/null +++ b/nix/writeTree/default.nix @@ -0,0 +1,43 @@ +{ depot, lib, pkgs, ... }: +let + inherit (lib) fix pipe mapAttrsToList isAttrs concatLines isString isDerivation isPath; + + # TODO(sterni): move to //nix/utils with clearer naming and alternative similar to lib.types.path + isPathLike = value: + isPath value + || isDerivation value + || (isString value && builtins.hasContext value); + + esc = s: lib.escapeShellArg /* ensure paths import into store */ "${s}"; + + writeTreeAtPath = path: tree: + '' + mkdir -p "$out/"${esc path} + '' + + pipe tree [ + (mapAttrsToList (k: v: + if isPathLike v then + "cp -R --reflink=auto ${v} \"$out/\"${esc path}/${esc k}" + else if lib.isAttrs v then + writeTreeAtPath (path + "/" + k) v + else + throw "invalid type (expected path, derivation, string with context, or attrs)")) + concatLines + ]; + + /* Create a directory tree specified by a Nix attribute set structure. + + Each value in `tree` should either be a file, a directory, or another tree + attribute set. Those paths will be written to a directory tree + corresponding to the structure of the attribute set. + + Type: string -> attrSet -> derivation + */ + writeTree = name: tree: + pkgs.runCommandLocal name { } (writeTreeAtPath "" tree); +in + +# __functor trick so readTree can add the tests attribute +{ + __functor = _: writeTree; +} diff --git a/nix/writeTree/tests/default.nix b/nix/writeTree/tests/default.nix new file mode 100644 index 0000000000..c5858ee96e --- /dev/null +++ b/nix/writeTree/tests/default.nix @@ -0,0 +1,93 @@ +{ depot, pkgs, lib, ... }: + +let + inherit (pkgs) runCommand writeText writeTextFile; + inherit (depot.nix) writeTree; + + checkTree = name: tree: expected: + runCommand "writeTree-test-${name}" + { + nativeBuildInputs = [ pkgs.buildPackages.lr ]; + passAsFile = [ "expected" ]; + inherit expected; + } '' + actualPath="$NIX_BUILD_TOP/actual" + cd ${lib.escapeShellArg (writeTree name tree)} + lr . > "$actualPath" + diff -u "$expectedPath" "$actualPath" | tee "$out" + ''; +in + +depot.nix.readTree.drvTargets { + empty = checkTree "empty" { } + '' + . + ''; + + simple-paths = checkTree "simple" + { + writeTree = { + meta = { + "owners.txt" = ../OWNERS; + }; + "code.nix" = ../default.nix; + all-tests = ./.; + nested.dirs.eval-time = builtins.toFile "owothia" '' + hold me owo + ''; + }; + } + '' + . + ./writeTree + ./writeTree/all-tests + ./writeTree/all-tests/default.nix + ./writeTree/code.nix + ./writeTree/meta + ./writeTree/meta/owners.txt + ./writeTree/nested + ./writeTree/nested/dirs + ./writeTree/nested/dirs/eval-time + ''; + + empty-dirs = checkTree "empty-dirs" + { + this.dir.is.empty = { }; + so.is.this.one = { }; + } + '' + . + ./so + ./so/is + ./so/is/this + ./so/is/this/one + ./this + ./this/dir + ./this/dir/is + ./this/dir/is/empty + ''; + + drvs = checkTree "drvs" + { + file-drv = writeText "road.txt" '' + Any road followed precisely to its end leads precisely nowhere. + ''; + dir-drv = writeTextFile { + name = "dir-of-text"; + destination = "/text/in/more/dirs.txt"; + text = '' + Climb the mountain just a little bit to test that it’s a mountain. + From the top of the mountain, you cannot see the mountain. + ''; + }; + } + '' + . + ./dir-drv + ./dir-drv/text + ./dir-drv/text/in + ./dir-drv/text/in/more + ./dir-drv/text/in/more/dirs.txt + ./file-drv + ''; +} |