diff options
Diffstat (limited to 'nix')
41 files changed, 1220 insertions, 460 deletions
diff --git a/nix/OWNERS b/nix/OWNERS index a742d0d22b..a640227914 100644 --- a/nix/OWNERS +++ b/nix/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - Profpatsch +Profpatsch diff --git a/nix/bufCheck/default.nix b/nix/bufCheck/default.nix index 039303ba68..ec98cfc376 100644 --- a/nix/bufCheck/default.nix +++ b/nix/bufCheck/default.nix @@ -1,9 +1,26 @@ -# Check protobuf syntax and breaking. +# Check protobuf breaking. Lints already happen in individual targets. # -{ depot, pkgs, ... }: +{ depot, pkgs, lib, ... }: -pkgs.writeShellScriptBin "ci-buf-check" '' - ${depot.third_party.bufbuild}/bin/buf check lint --input . - # Report-only - ${depot.third_party.bufbuild}/bin/buf check breaking --input "." --against-input "./.git#branch=canon" || true -'' +let + inherit (depot.nix) bufCheck;# self reference + + script = pkgs.writeShellScriptBin "ci-buf-check" '' + export PATH="$PATH:${pkgs.lib.makeBinPath [ pkgs.buf ]}" + # Report-only + (cd $(git rev-parse --show-toplevel) && (buf breaking . --against "./.git#ref=HEAD~1" || true)) + ''; +in + +script.overrideAttrs (old: { + meta = lib.recursiveUpdate old.meta { + # Protobuf check step executed in the buildkite pipeline which + # validates that changes to .proto files between revisions + # don't cause backwards-incompatible or otherwise flawed changes. + ci.extraSteps.protoCheck = { + alwaysRun = true; + label = ":water_buffalo: protoCheck"; + command = pkgs.writeShellScript "ci-buf-check-step" "exec ${depot.nix.bufCheck}/bin/ci-buf-check"; + }; + }; +}) diff --git a/nix/buildGo/README.md b/nix/buildGo/README.md index 37e0c06933..e9667c039a 100644 --- a/nix/buildGo/README.md +++ b/nix/buildGo/README.md @@ -2,8 +2,7 @@ buildGo.nix =========== This is an alternative [Nix][] build system for [Go][]. It supports building Go -libraries and programs, and even automatically generating Protobuf & gRPC -libraries. +libraries and programs. *Note:* This will probably end up being folded into [Nixery][]. @@ -33,7 +32,6 @@ Given a program layout like this: ├── lib <-- some library component │ ├── bar.go │ └── foo.go -├── api.proto <-- gRPC API definition ├── main.go <-- program implementation └── default.nix <-- build instructions ``` @@ -44,11 +42,6 @@ The contents of `default.nix` could look like this: { buildGo }: let - api = buildGo.grpc { - name = "someapi"; - proto = ./api.proto; - }; - lib = buildGo.package { name = "somelib"; srcs = [ @@ -58,7 +51,7 @@ let }; in buildGo.program { name = "my-program"; - deps = [ api lib ]; + deps = [ lib ]; srcs = [ ./main.go @@ -105,22 +98,6 @@ in buildGo.program { | `src` | `path` | Path to the source **directory** | yes | | `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no | - For some examples of how `buildGo.external` is used, check out - [`proto.nix`](./proto.nix). - -* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. - - | parameter | type | use | required? | - |-------------|-------------|--------------------------------------------------|-----------| - | `name` | `string` | Name for the resulting library | yes | - | `proto` | `path` | Path to the Protobuf definition file | yes | - | `path` | `string` | Import path for the resulting Go library | no | - | `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no | - -* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. - - The parameters are identical to `buildGo.proto`. - ## Current status This project is work-in-progress. Crucially it is lacking the following features: diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix index 92951b3cb2..c93642a127 100644 --- a/nix/buildGo/default.nix +++ b/nix/buildGo/default.nix @@ -22,7 +22,8 @@ let replaceStrings toString; - inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; + inherit (pkgs) lib runCommand fetchFromGitHub protobuf symlinkJoin go; + goStdlib = buildStdlib go; # Helpers for low-level Go compiler invocations spaceOut = lib.concatStringsSep " "; @@ -41,8 +42,6 @@ let xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p); - # Add an `overrideGo` attribute to a function result that works # similar to `overrideAttrs`, but is used specifically for the # arguments passed to Go builders. @@ -50,16 +49,52 @@ let overrideGo = new: makeOverridable f (orig // (new orig)); }; + buildStdlib = go: runCommand "go-stdlib-${go.version}" + { + nativeBuildInputs = [ go ]; + } '' + HOME=$NIX_BUILD_TOP/home + mkdir $HOME + + goroot="$(go env GOROOT)" + cp -R "$goroot/src" "$goroot/pkg" . + + chmod -R +w . + GODEBUG=installgoroot=all GOROOT=$NIX_BUILD_TOP go install -v --trimpath std + + mkdir $out + cp -r pkg/*_*/* $out + + find $out -name '*.a' | while read -r ARCHIVE_FULL; do + ARCHIVE="''${ARCHIVE_FULL#"$out/"}" + PACKAGE="''${ARCHIVE%.a}" + echo "packagefile $PACKAGE=$ARCHIVE_FULL" + done > $out/importcfg + ''; + + importcfgCmd = { name, deps, out ? "importcfg" }: '' + echo "# nix buildGo ${name}" > "${out}" + cat "${goStdlib}/importcfg" >> "${out}" + ${lib.concatStringsSep "\n" (map (dep: '' + find "${dep}" -name '*.a' | while read -r pkgp; do + relpath="''${pkgp#"${dep}/"}" + pkgname="''${relpath%.a}" + echo "packagefile $pkgname=$pkgp" + done >> "${out}" + '') deps)} + ''; + # High-level build functions # Build a Go program out of the specified files and dependencies. program = { name, srcs, deps ? [ ], x_defs ? { } }: let uniqueDeps = allDeps (map (d: d.gopkg) deps); in runCommand name { } '' - ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} + ${importcfgCmd { inherit name; deps = uniqueDeps; }} + ${go}/bin/go tool compile -o ${name}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p main ${includeSources uniqueDeps} ${spaceOut srcs} mkdir -p $out/bin export GOROOT_FINAL=go - ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a + ${go}/bin/go tool link -o $out/bin/${name} -importcfg=importcfg -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a ''; # Build a Go library assembled out of the specified files. @@ -76,8 +111,8 @@ let # This is required for several popular packages (e.g. x/sys). ifAsm = do: lib.optionalString (sfiles != [ ]) do; asmBuild = ifAsm '' - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} - ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} + ${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} ''; asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; asmPack = ifAsm '' @@ -88,7 +123,8 @@ let mkdir -p $out/${path} ${srcList path (map (s: "${s}") srcs)} ${asmBuild} - ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${importcfgCmd { inherit name; deps = uniqueDeps; }} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} ${asmPack} '').overrideAttrs (_: { passthru = { @@ -108,33 +144,14 @@ let # named "gopkg", and an attribute named "gobin" for binaries. external = import ./external { inherit pkgs program package; }; - # Import support libraries needed for protobuf & gRPC support - protoLibs = import ./proto.nix { - inherit external; - }; - - # Build a Go library out of the specified protobuf definition. - proto = { name, proto, path ? name, goPackage ? name, extraDeps ? [ ] }: (makeOverridable package) { - inherit name path; - deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; - srcs = lib.singleton (runCommand "goproto-${name}.pb.go" { } '' - cp ${proto} ${baseNameOf proto} - ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ - --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} - mv ./${goPackage}/*.pb.go $out - ''); - }; - - # Build a Go library out of the specified gRPC definition. - grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); - in { # Only the high-level builder functions are exposed, but made # overrideable. program = makeOverridable program; package = makeOverridable package; - proto = makeOverridable proto; - grpc = makeOverridable grpc; external = makeOverridable external; + + # re-expose the Go version used + inherit go; } diff --git a/nix/buildGo/example/default.nix b/nix/buildGo/example/default.nix index 08da075e18..6756bf39e2 100644 --- a/nix/buildGo/example/default.nix +++ b/nix/buildGo/example/default.nix @@ -19,13 +19,6 @@ let ]; }; - # Example use of buildGo.proto, which generates a Go library from a - # Protobuf definition file. - exampleProto = buildGo.proto { - name = "exampleproto"; - proto = ./thing.proto; - }; - # Example use of buildGo.program, which builds an executable using # the specified name and dependencies (which in turn must have been # created via buildGo.package etc.) @@ -39,7 +32,6 @@ buildGo.program { deps = [ examplePackage - exampleProto ]; x_defs = { diff --git a/nix/buildGo/example/thing.proto b/nix/buildGo/example/thing.proto deleted file mode 100644 index 0f6d6575e0..0000000000 --- a/nix/buildGo/example/thing.proto +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; -package example; - -message Thing { - string id = 1; - string kind_of_thing = 2; -} diff --git a/nix/buildGo/external/default.nix b/nix/buildGo/external/default.nix index f713783a58..42592c67e4 100644 --- a/nix/buildGo/external/default.nix +++ b/nix/buildGo/external/default.nix @@ -13,6 +13,7 @@ let readFile replaceStrings tail + unsafeDiscardStringContext throw; inherit (pkgs) lib runCommand go jq ripgrep; @@ -102,7 +103,9 @@ let analysisOutput = runCommand "${name}-structure.json" { } '' ${analyser}/bin/analyser -path ${path} -source ${src} > $out ''; - analysis = fromJSON (readFile analysisOutput); + # readFile adds the references of the read in file to the string context for + # Nix >= 2.6 which would break the attribute set construction in fromJSON + analysis = fromJSON (unsafeDiscardStringContext (readFile analysisOutput)); in lib.fix (self: foldl' lib.recursiveUpdate { } ( map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go index a77c43b371..4402a8eb86 100644 --- a/nix/buildGo/external/main.go +++ b/nix/buildGo/external/main.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "go/build" - "io/ioutil" "log" "os" "path" @@ -74,8 +73,8 @@ func findGoDirs(at string) ([]string, error) { } goDirs := []string{} - for k, _ := range dirSet { - goDirs = append(goDirs, k) + for goDir := range dirSet { + goDirs = append(goDirs, goDir) } return goDirs, nil @@ -148,7 +147,7 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk } func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { - f, err := ioutil.ReadFile(from) + f, err := os.ReadFile(from) if err != nil { return } diff --git a/nix/buildGo/proto.nix b/nix/buildGo/proto.nix deleted file mode 100644 index 6c37f758ce..0000000000 --- a/nix/buildGo/proto.nix +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2019 Google LLC. -# SPDX-License-Identifier: Apache-2.0 -# -# This file provides derivations for the dependencies of a gRPC -# service in Go. - -{ external }: - -let - inherit (builtins) fetchGit map; -in -rec { - goProto = external { - path = "github.com/golang/protobuf"; - src = fetchGit { - url = "https://github.com/golang/protobuf"; - rev = "ed6926b37a637426117ccab59282c3839528a700"; - }; - }; - - xnet = external { - path = "golang.org/x/net"; - - src = fetchGit { - url = "https://go.googlesource.com/net"; - rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; - }; - - deps = [ - xtext.secure.bidirule - xtext.unicode.bidi - xtext.unicode.norm - ]; - }; - - xsys = external { - path = "golang.org/x/sys"; - src = fetchGit { - url = "https://go.googlesource.com/sys"; - rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; - }; - }; - - xtext = external { - path = "golang.org/x/text"; - src = fetchGit { - url = "https://go.googlesource.com/text"; - rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; - }; - }; - - genproto = external { - path = "google.golang.org/genproto"; - src = fetchGit { - url = "https://github.com/google/go-genproto"; - # necessary because https://github.com/NixOS/nix/issues/1923 - ref = "main"; - rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; - }; - - deps = with goProto; [ - proto - ptypes.any - ]; - }; - - goGrpc = external { - path = "google.golang.org/grpc"; - deps = ([ - xnet.trace - xnet.http2 - xsys.unix - xnet.http2.hpack - genproto.googleapis.rpc.status - ] ++ (with goProto; [ - proto - ptypes - ptypes.duration - ptypes.timestamp - ])); - - src = fetchGit { - url = "https://github.com/grpc/grpc-go"; - rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; - }; - }; -} diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix index 9d6ce4edda..0d68a2818b 100644 --- a/nix/buildLisp/default.nix +++ b/nix/buildLisp/default.nix @@ -8,7 +8,7 @@ let inherit (builtins) map elemAt match filter; - inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; + inherit (pkgs) lib runCommand makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; inherit (pkgs.stdenv) targetPlatform; # @@ -154,8 +154,7 @@ let let implementation = old.implementation or defaultImplementation; brokenOn = old.brokenOn or [ ]; - # TODO(sterni): https://github.com/Clozure/ccl/issues/405 - targets = lib.subtractLists (brokenOn ++ [ "ccl" implementation.name ]) + targets = lib.subtractLists (brokenOn ++ [ implementation.name ]) (builtins.attrNames impls); in { @@ -188,7 +187,7 @@ let lispNativeDeps = allNative native lispDeps; filteredSrcs = implFilter implementation srcs; in - runCommandNoCC name + runCommand name { LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; LANG = "C.UTF-8"; @@ -476,7 +475,7 @@ let } $@ ''; - bundled = name: runCommandNoCC "${name}-cllib" + bundled = name: runCommand "${name}-cllib" { passthru = { lispName = name; @@ -514,7 +513,6 @@ let # See https://ccl.clozure.com/docs/ccl.html#building-definitions faslExt = - /**/ if targetPlatform.isPower && targetPlatform.is32bit then "pfsl" else if targetPlatform.isPower && targetPlatform.is64bit then "p64fsl" else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl" @@ -641,7 +639,7 @@ let } else null; in - lib.fix (self: runCommandNoCC "${name}-cllib" + lib.fix (self: runCommand "${name}-cllib" { LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; LANG = "C.UTF-8"; @@ -708,7 +706,7 @@ let } else null; in - lib.fix (self: runCommandNoCC "${name}" + lib.fix (self: runCommand "${name}" { nativeBuildInputs = [ makeWrapper ]; LD_LIBRARY_PATH = libPath; diff --git a/nix/buildLisp/tests/argv0.nix b/nix/buildLisp/tests/argv0.nix index bc29337d06..ca5f2b9741 100644 --- a/nix/buildLisp/tests/argv0.nix +++ b/nix/buildLisp/tests/argv0.nix @@ -1,36 +1,58 @@ -{ depot, pkgs, ... }: - -depot.nix.buildLisp.program { - name = "argv0-test"; - - srcs = [ - (pkgs.writeText "argv0-test.lisp" '' - (defpackage :argv0-test (:use :common-lisp :uiop) (:export :main)) - (in-package :argv0-test) - - (defun main () - (format t "~A~%" (uiop:argv0))) - '') - ]; - - deps = [ - { - sbcl = depot.nix.buildLisp.bundled "uiop"; - default = depot.nix.buildLisp.bundled "asdf"; - } - ]; - - passthru.meta.ci = { - extraSteps.verify = { - label = "verify argv[0] output"; - needsOutput = true; - command = pkgs.writeShellScript "check-argv0" '' - set -eux - - for invocation in "$(pwd)/result/bin/argv0-test" "./result/bin/argv0-test"; do - test "$invocation" = "$("$invocation")" - done - ''; +{ depot, pkgs, lib, ... }: + +let + # Trivial test program that outputs argv[0] and exits + prog = + depot.nix.buildLisp.program { + name = "argv0-test"; + + srcs = [ + (pkgs.writeText "argv0-test.lisp" '' + (defpackage :argv0-test (:use :common-lisp :uiop) (:export :main)) + (in-package :argv0-test) + + (defun main () + (format t "~A~%" (uiop:argv0))) + '') + ]; + + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + ]; }; - }; -} + + # Extract verify argv[0] output for given buildLisp program + checkImplementation = prog: + pkgs.runCommand "check-argv0" { } '' + set -eux + + checkInvocation() { + invocation="$1" + test "$invocation" = "$("$invocation")" + } + + checkInvocation "${prog}/bin/argv0-test" + + cd ${prog} + checkInvocation "./bin/argv0-test" + + cd bin + checkInvocation ./argv0-test + + set +x + + touch "$out" + ''; + + inherit (prog.meta.ci) targets; +in + +(checkImplementation prog).overrideAttrs (_: { + # Wire up a subtarget all (active) non-default implementations + passthru = lib.genAttrs targets (name: checkImplementation prog.${name}); + + meta.ci = { inherit targets; }; +}) diff --git a/nix/buildManPages/OWNERS b/nix/buildManPages/OWNERS index f16dd105d7..2e95807063 100644 --- a/nix/buildManPages/OWNERS +++ b/nix/buildManPages/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni +sterni diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 5c46dcb333..9abba9408a 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -25,65 +25,114 @@ let toJSON unsafeDiscardStringContext; - inherit (pkgs) lib runCommandNoCC writeText; + inherit (pkgs) lib runCommand writeText; inherit (depot.nix.readTree) mkLabel; + + inherit (depot.nix) dependency-analyzer; in rec { - # Creates a Nix expression that yields the target at the specified - # location in the repository. - # - # This makes a distinction between normal targets (which physically - # exist in the repository) and subtargets (which are "virtual" - # targets exposed by a physical one) to make it clear in the build - # output which is which. - mkBuildExpr = target: + # Create a unique key for the buildkite pipeline based on the given derivation + # or drvPath. A consequence of using such keys is that every derivation may + # only be exposed as a single, unique step in the pipeline. + keyForDrv = drvOrPath: + let + drvPath = + if lib.isDerivation drvOrPath then drvOrPath.drvPath + else if lib.isString drvOrPath then drvOrPath + else builtins.throw "keyForDrv: expected string or derivation"; + + # Only use the drv hash to prevent escaping problems. Buildkite also has a + # limit of 100 characters on keys. + in + "drv-" + (builtins.substring 0 32 + (builtins.baseNameOf (unsafeDiscardStringContext drvPath)) + ); + + # Given an arbitrary attribute path generate a Nix expression which obtains + # this from the root of depot (assumed to be ./.). Attributes may be any + # Nix strings suitable as attribute names, not just Nix literal-safe strings. + mkBuildExpr = attrPath: let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; - targetExpr = foldl' descend "import ./. {}" target.__readTree; - subtargetExpr = descend targetExpr target.__subtarget; in - if target ? __subtarget then subtargetExpr else targetExpr; + foldl' descend "import ./. {}" attrPath; # Determine whether to skip a target if it has not diverged from the # HEAD branch. - shouldSkip = parentTargetMap: label: drvPath: + shouldSkip = { parentTargetMap ? { }, label, drvPath }: if (hasAttr label parentTargetMap) && parentTargetMap."${label}".drvPath == drvPath then "Target has not changed." else false; - # Create build command for a derivation target. - mkBuildCommand = target: drvPath: concatStringsSep " " [ + # Create build command for an attribute path pointing to a derivation. + mkBuildCommand = { attrPath, drvPath, outLink ? "result" }: concatStringsSep " " [ + # If the nix build fails, the Nix command's exit status should be used. + "set -o pipefail;" + # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on # at least one of its `outPath`s, so we need to discard the string context # if we don't want to build everything during pipeline construction. - "(nix-store --realise '${drvPath}' --add-root result --indirect && readlink result)" + # + # To make this more uniform with how nix-build(1) works, we call realpath(1) + # on nix-store(1)'s output since it has the habit of printing the path of the + # out link, not the store path. + "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs -r realpath)" # Since we don't gcroot the derivation files, they may be deleted by the # garbage collector. In that case we can reevaluate and build the attribute # using nix-build. - "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)" + "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr attrPath}' --show-trace --out-link '${outLink}')" ]; + # Attribute path of a target relative to the depot root. Needs to take into + # account whether the target is a physical target (which corresponds to a path + # in the filesystem) or the subtarget of a physical target. + targetAttrPath = target: + target.__readTree + ++ lib.optionals (target ? __subtarget) [ target.__subtarget ]; + + # Given a derivation (identified by drvPath) that is part of the list of + # targets passed to mkPipeline, determine all derivations that it depends on + # and are also part of the pipeline. Finally, return the keys of the steps + # that build them. This is used to populate `depends_on` in `mkStep`. + # + # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`. + getTargetPipelineDeps = targetDepMap: drvPath: + # Sanity check: We should only call this function on targets explicitly + # passed to mkPipeline. Thus it should have been passed as a “known” drv to + # dependency-analyzer. + assert targetDepMap.${drvPath}.known; + builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps; + # Create a pipeline step from a single target. - mkStep = headBranch: parentTargetMap: target: + mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }: let label = mkLabel target; drvPath = unsafeDiscardStringContext target.drvPath; - shouldSkip' = shouldSkip parentTargetMap; in { label = ":nix: " + label; - key = hashString "sha1" label; - skip = shouldSkip' label drvPath; - command = mkBuildCommand target drvPath; + key = keyForDrv target; + skip = shouldSkip { inherit label drvPath parentTargetMap; }; + command = mkBuildCommand { + attrPath = targetAttrPath target; + inherit drvPath; + }; env.READTREE_TARGET = label; + cancel_on_build_failing = cancelOnBuildFailing; # Add a dependency on the initial static pipeline step which # always runs. This allows build steps uploaded in batches to # start running before all batches have been uploaded. - depends_on = ":init:"; - }; + depends_on = [ ":init:" ] + ++ getTargetPipelineDeps targetDepMap drvPath + ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + } // lib.optionalAttrs (target ? meta.timeout) { + timeout_in_minutes = target.meta.timeout / 60; + # Additional arguments to set on the step. + # Keep in mind these *overwrite* existing step args, not extend. Use with caution. + } // lib.optionalAttrs (target ? meta.ci.buildkiteExtraStepArgs) target.meta.ci.buildkiteExtraStepArgs; # Helper function to inelegantly divide a list into chunks of at # most n elements. @@ -142,7 +191,21 @@ rec { # # Can be used for status reporting steps and the like. postBuildSteps ? [ ] - , # Build phases that are active for this invocation (i.e. their + # The list of phases known by the current Buildkite + # pipeline. Dynamic pipeline chunks for each phase are uploaded + # to Buildkite on execution of static part of the + # pipeline. Phases selection is hard-coded in the static + # pipeline. + # + # Pipeline generation will fail when an extra step with + # unregistered phase is added. + # + # Common scenarios for different phase: + # - "build" - main phase for building all Nix targets + # - "release" - pushing artifacts to external repositories + # - "deploy" - updating external deployment configurations + , phases ? [ "build" "release" ] + # Build phases that are active for this invocation (i.e. their # steps should be generated). # # This can be used to disable outputting parts of a pipeline if, @@ -150,45 +213,50 @@ rec { # eval contexts. # # TODO(tazjin): Fail/warn if unknown phase is requested. - activePhases ? [ "build" "release" ] + , activePhases ? phases + # Setting this attribute to true cancels dynamic pipeline steps + # as soon as the build is marked as failing. + # + # To enable this feature one should enable "Fail Fast" setting + # at Buildkite pipeline or on organization level. + , cancelOnBuildFailing ? false }: let - # Currently the only known phases are 'build' (Nix builds and - # extra steps that are not post-build steps) and 'release' (all - # post-build steps). - # - # TODO(tazjin): Fully configurable set of phases? - knownPhases = [ "build" "release" ]; - # List of phases to include. - phases = lib.intersectLists activePhases knownPhases; + enabledPhases = lib.intersectLists activePhases phases; # Is the 'build' phase included? This phase is treated specially # because it always contains the plain Nix builds, and some # logic/optimisation depends on knowing whether is executing. - buildEnabled = elem "build" phases; + buildEnabled = elem "build" enabledPhases; + + # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer. + targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets); # Convert a target into all of its steps, separated by build # phase (as phases end up in different chunks). targetToSteps = target: let - step = mkStep headBranch parentTargetMap target; + mkStepArgs = { + inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing; + }; + step = mkStep mkStepArgs; # Same step, but with an override function applied. This is # used in mkExtraStep if the extra step needs to modify the # parent derivation somehow. # # Note that this will never affect the label. - overridable = f: mkStep headBranch parentTargetMap (f target); + overridable = f: mkStep (mkStepArgs // { target = (f target); }); # Split extra steps by phase. splitExtraSteps = lib.groupBy ({ phase, ... }: phase) - (attrValues (mapAttrs (normaliseExtraStep knownPhases overridable) + (attrValues (mapAttrs (normaliseExtraStep phases overridable) (target.meta.ci.extraSteps or { }))); extraSteps = mapAttrs (_: steps: - map (mkExtraStep buildEnabled) steps) + map (mkExtraStep (targetAttrPath target) buildEnabled) steps) splitExtraSteps; in if !buildEnabled then extraSteps @@ -204,7 +272,7 @@ rec { release = postBuildSteps; }; - phasesWithSteps = lib.zipAttrsWithNames phases (_: concatLists) + phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists) ((map targetToSteps drvTargets) ++ [ globalSteps ]); # Generate pipeline chunks for each phase. @@ -215,10 +283,10 @@ rec { then acc else acc ++ (pipelineChunks phase phaseSteps)) [ ] - phases; + enabledPhases; in - runCommandNoCC "buildkite-pipeline" { } '' + runCommand "buildkite-pipeline" { } '' mkdir $out echo "Generated ${toString (length chunks)} pipeline chunks" ${ @@ -238,9 +306,7 @@ rec { # Include the attrPath in the output to reconstruct the drv # without parsing the human-readable label. - attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ - target.__subtarget - ]; + attrPath = targetAttrPath target; }; }) drvTargets))); @@ -264,10 +330,6 @@ rec { # confirmation. These steps always run after the main build is # done and have no influence on CI status. # - # postBuild (optional): If set to true, this step will run after - # all primary build steps (that is, after status has been reported - # back to CI). - # # needsOutput (optional): If set to true, the parent derivation # will be built in the working directory before running the # command. Output will be available as 'result'. @@ -294,8 +356,8 @@ rec { steps = [ { - inherit (step) branches; inherit prompt; + branches = step.branches or [ ]; block = ":radio_button: Run ${label}? (from ${parent.env.READTREE_TARGET})"; } @@ -309,7 +371,7 @@ rec { # Validate and normalise extra step configuration before actually # generating build steps, in order to use user-provided metadata # during the pipeline generation. - normaliseExtraStep = knownPhases: overridableParent: key: + normaliseExtraStep = phases: overridableParent: key: { command , label ? key , needsOutput ? false @@ -317,12 +379,8 @@ rec { , branches ? null , alwaysRun ? false , prompt ? false - - # TODO(tazjin): Default to 'build' after 2022-10-01. - , phase ? if (isNull postBuild || !postBuild) then "build" else "release" - - # TODO(tazjin): Turn into hard-failure after 2022-10-01. - , postBuild ? null + , softFail ? false + , phase ? "build" , skip ? false , agents ? null }: @@ -330,12 +388,12 @@ rec { parent = overridableParent parentOverride; parentLabel = parent.env.READTREE_TARGET; - validPhase = lib.throwIfNot (elem phase knownPhases) '' + validPhase = lib.throwIfNot (elem phase phases) '' In step '${label}' (from ${parentLabel}): Phase '${phase}' is not valid. - Known phases: ${concatStringsSep ", " knownPhases} + Known phases: ${concatStringsSep ", " phases} '' phase; in @@ -349,35 +407,16 @@ rec { needsOutput parent parentLabel + softFail skip agents; - # //nix/buildkite is growing a new feature for adding different - # "build phases" which supersedes the previous `postBuild` - # boolean API. - # - # To help users transition, emit warnings if the old API is used. - phase = lib.warnIfNot (isNull postBuild) '' - In step '${label}' (from ${parentLabel}): - - Please note: The CI system is introducing support for running - steps in different build phases. - - The currently supported phases are 'build' (all Nix targets, - extra steps such as tests that feed into the build results, - etc.) and 'release' (steps that run after builds and tests - have already completed). - - This replaces the previous boolean `postBuild` API in extra - step definitions. Please remove the `postBuild` parameter from - this step and instead set `phase = ${phase};`. - '' - validPhase; + phase = validPhase; prompt = lib.throwIf (prompt != false && phase == "build") '' In step '${label}' (from ${parentLabel}): - The 'prompt' feature can only be used by steps in the "release" + The 'prompt' feature can not be used by steps in the "build" phase, because CI builds should not be gated on manual human approvals. '' @@ -386,11 +425,26 @@ rec { # Create the Buildkite configuration for an extra step, optionally # wrapping it in a gate group. - mkExtraStep = buildEnabled: cfg: + mkExtraStep = parentAttrPath: buildEnabled: cfg: let + # ATTN: needs to match an entry in .gitignore so that the tree won't get dirty + commandScriptLink = "nix-buildkite-extra-step-command-script"; + step = { + key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; - skip = if cfg.alwaysRun then false else cfg.skip or cfg.parent.skip or false; + skip = + let + # When parent doesn't have skip attribute set, default to false + parentSkip = cfg.parent.skip or false; + # Extra step skip parameter can be string explaining the + # skip reason. + extraStepSkip = if builtins.isString cfg.skip then true else cfg.skip; + # Don't run if extra step is explicitly set to skip. If + # parameter is not set or equal to false, follow parent behavior. + skip' = if extraStepSkip then cfg.skip else parentSkip; + in + if cfg.alwaysRun then false else skip'; depends_on = lib.optional (buildEnabled && !cfg.alwaysRun && !cfg.needsOutput) @@ -402,9 +456,25 @@ rec { "echo '~~~ Preparing build output of ${cfg.parentLabel}'" } ${lib.optionalString cfg.needsOutput cfg.parent.command} - echo '+++ Running extra step command' - exec ${cfg.command} + echo '--- Building extra step script' + command_script="$(${ + # Using command substitution in this way assumes the script drv only has one output + assert builtins.length cfg.command.outputs == 1; + mkBuildCommand { + # script is exposed at <parent>.meta.ci.extraSteps.<key>.command + attrPath = + parentAttrPath + ++ [ "meta" "ci" "extraSteps" cfg.key "command" ]; + drvPath = unsafeDiscardStringContext cfg.command.drvPath; + # make sure it doesn't conflict with result (from needsOutput) + outLink = commandScriptLink; + } + })" + echo '+++ Running extra step script' + exec "$command_script" ''; + + soft_fail = cfg.softFail; } // (lib.optionalAttrs (cfg.agents != null) { inherit (cfg) agents; }) // (lib.optionalAttrs (cfg.branches != null) { branches = lib.concatStringsSep " " cfg.branches; diff --git a/nix/buildkite/fetch-parent-targets.sh b/nix/buildkite/fetch-parent-targets.sh index 8afac1e5ec..08c2d1f3ab 100755 --- a/nix/buildkite/fetch-parent-targets.sh +++ b/nix/buildkite/fetch-parent-targets.sh @@ -2,43 +2,54 @@ set -ueo pipefail # Each Buildkite build stores the derivation target map as a pipeline -# artifact. This script determines the most appropriate commit (the -# fork point of the current chain from HEAD) and fetches the artifact. +# artifact. To reduce the amount of work done by CI, each CI build is +# diffed against the latest such derivation map found for the +# repository. # -# New builds can be based on HEAD before the pipeline for the last -# commit has finished, in which case it is possible that the fork -# point has no derivation map. To account for this, up to 3 commits -# prior to HEAD are also queried to find a map. +# Note that this does not take into account when the currently +# processing CL was forked off from the canonical branch, meaning that +# things like nixpkgs updates in between will cause mass rebuilds in +# any case. # # If no map is found, the failure mode is not critical: We simply # build all targets. +readonly REPO_ROOT=$(git rev-parse --show-toplevel) + : ${DRVMAP_PATH:=pipeline/drvmap.json} : ${BUILDKITE_TOKEN_PATH:=~/buildkite-token} -git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" - -FIRST=$(git merge-base FETCH_HEAD "${BUILDKITE_COMMIT}") -SECOND=$(git rev-parse "$FIRST~1") -THIRD=$(git rev-parse "$FIRST~2") - -function most_relevant_builds { +# Runs a fairly complex Buildkite GraphQL query that attempts to fetch all +# pipeline-gen steps from the default branch, as long as one appears within the +# last 50 builds or so. The query restricts build states to running or passed +# builds, which means that it *should* be unlikely that nothing is found. +# +# There is no way to filter this more loosely (e.g. by saying "any recent build +# matching these conditions"). +# +# The returned data structure is complex, and disassembled by a JQ script that +# first filters out all builds with no matching jobs (e.g. builds that are still +# in progress), and then filters those down to builds with artifacts, and then +# to drvmap artifacts specifically. +# +# If a recent drvmap was found, this returns its download URL. Otherwise, it +# returns the string "null". +function latest_drvmap_url { set -u curl 'https://graphql.buildkite.com/v1' \ --silent \ -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \ - -d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \ - jq -r '.data.pipeline.builds.edges[] | .node.uuid' + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(first: 50, branch: [\\\"%default\\\"], state: [RUNNING, PASSED]) { edges { node { jobs(passed: true, first: 1, type: [COMMAND], step: {key: [\\\"pipeline-gen\\\"]}) { edges { node { ... on JobTypeCommand { url artifacts { edges { node { downloadURL path }}}}}}}}}}}}\"}" | tee out.json | \ + jq -r '[.data.pipeline.builds.edges[] | select((.node.jobs.edges | length) > 0) | .node.jobs.edges[] | .node.artifacts[][] | select(.node.path == "pipeline/drvmap.json")][0].node.downloadURL' } -mkdir -p tmp -for build in $(most_relevant_builds); do - echo "Checking artifacts for build $build" - buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true +readonly DOWNLOAD_URL=$(latest_drvmap_url) - if [[ -f "tmp/${DRVMAP_PATH}" ]]; then - echo "Fetched target map from build ${build}" - mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json - break - fi -done +if [[ ${DOWNLOAD_URL} != "null" ]]; then + mkdir -p tmp + curl -o tmp/parent-target-map.json ${DOWNLOAD_URL} && echo "downloaded parent derivation map" \ + || echo "failed to download derivation map!" +else + echo "no derivation map found!" +fi diff --git a/nix/dependency-analyzer/default.nix b/nix/dependency-analyzer/default.nix new file mode 100644 index 0000000000..2ec8d7b5b9 --- /dev/null +++ b/nix/dependency-analyzer/default.nix @@ -0,0 +1,263 @@ +{ lib, depot, pkgs, ... }: + +let + inherit (builtins) unsafeDiscardStringContext appendContext; + + # + # Utilities + # + + # Determine all paths a derivation depends on, i.e. input derivations and + # files imported into the Nix store. + # + # Implementation for Nix < 2.6 is quite hacky at the moment. + # + # Type: str -> [str] + # + # TODO(sterni): clean this up and expose it + directDrvDeps = + let + getDeps = + if lib.versionAtLeast builtins.nixVersion "2.6" + then + # Since https://github.com/NixOS/nix/pull/1643, Nix apparently »preserves + # string context« through a readFile invocation. This has the side effect + # that it becomes possible to query the actual references a store path has. + # Not a 100% sure this is intended, but _very_ convenient for us here. + drvPath: + builtins.attrNames (builtins.getContext (builtins.readFile drvPath)) + else + # For Nix < 2.6 we have to rely on HACK, namely grepping for quoted + # store path references in the file. In the future this should be + # replaced by a proper derivation parser. + drvPath: builtins.concatLists ( + builtins.filter builtins.isList ( + builtins.split + "\"(${lib.escapeRegex builtins.storeDir}/[[:alnum:]+._?=-]+.drv)\"" + (builtins.readFile drvPath) + ) + ); + in + drvPath: + # if the passed path is not a derivation we can't necessarily get its + # dependencies, since it may not be representable as a Nix string due to + # NUL bytes, e.g. compressed patch files imported into the Nix store. + if builtins.match "^.+\\.drv$" drvPath == null + then [ ] + else getDeps drvPath; + + # Maps a list of derivation to the list of corresponding `drvPath`s. + # + # Type: [drv] -> [str] + drvsToPaths = drvs: + builtins.map (drv: builtins.unsafeDiscardOutputDependency drv.drvPath) drvs; + + # + # Calculate map of direct derivation dependencies + # + + # Create the dependency map entry for a given `drvPath` which mainly includes + # a list of other `drvPath`s it depends on. Additionally we store whether the + # derivation is `known`, i.e. part of the initial list of derivations we start + # generating the map from + # + # Type: bool -> string -> set + drvEntry = known: drvPath: + let + # key may not refer to a store path, … + key = unsafeDiscardStringContext drvPath; + # but we must read from the .drv file. + path = builtins.unsafeDiscardOutputDependency drvPath; + in + { + inherit key; + # trick so we can call listToAttrs directly on the result of genericClosure + name = key; + value = { + deps = directDrvDeps path; + inherit known; + }; + }; + + # Create an attribute set that maps every derivation in the combined + # dependency closure of the list of input derivation paths to every of their + # direct dependencies. Additionally every entry will have set their `known` + # attribute to `true` if it is in the list of input derivation paths. + # + # Type: [str] -> set + plainDrvDepMap = drvPaths: + builtins.listToAttrs ( + builtins.genericClosure { + startSet = builtins.map (drvEntry true) drvPaths; + operator = { value, ... }: builtins.map (drvEntry false) value.deps; + } + ); + + # + # Calculate closest known dependencies in the dependency map + # + + inherit (depot.nix.stateMonad) + after + bind + for_ + get + getAttr + run + setAttr + pure + ; + + # This is an action in stateMonad which expects the (initial) state to have + # been produced by `plainDrvDepMap`. Given a `drvPath`, it calculates a + # `knownDeps` list which holds the `drvPath`s of the closest derivation marked + # as `known` along every edge. This list is inserted into the dependency map + # for `drvPath` and every other derivation in its dependecy closure (unless + # the information was already present). This means that the known dependency + # information for a derivation never has to be recalculated, as long as they + # are part of the same stateful computation. + # + # The upshot is that after calling `insertKnownDeps drvPath`, + # `fmap (builtins.getAttr "knownDeps") (getAttr drvPath)` will always succeed. + # + # Type: str -> stateMonad drvDepMap null + insertKnownDeps = drvPathWithContext: + let + # We no longer need to read from the store, so context is irrelevant, but + # we need to check for attr names which requires the absence of context. + drvPath = unsafeDiscardStringContext drvPathWithContext; + in + bind get (initDepMap: + # Get the dependency map's state before we've done anything to obtain the + # entry we'll be manipulating later as well as its dependencies. + let + entryPoint = initDepMap.${drvPath}; + + # We don't need to recurse if our direct dependencies either have their + # knownDeps list already populated or are known dependencies themselves. + depsPrecalculated = + builtins.partition + (dep: + initDepMap.${dep}.known + || initDepMap.${dep} ? knownDeps + ) + entryPoint.deps; + + # If a direct dependency is known, it goes right to our known dependency + # list. If it is unknown, we can copy its knownDeps list into our own. + initiallyKnownDeps = + builtins.concatLists ( + builtins.map + (dep: + if initDepMap.${dep}.known + then [ dep ] + else initDepMap.${dep}.knownDeps + ) + depsPrecalculated.right + ); + in + + # If the information was already calculated before, we can exit right away + if entryPoint ? knownDeps + then pure null + else + after + # For all unknown direct dependencies which don't have a `knownDeps` + # list, we call ourselves recursively to populate it. Since this is + # done sequentially in the state monad, we avoid recalculating the + # list for the same derivation multiple times. + (for_ + depsPrecalculated.wrong + insertKnownDeps) + # After this we can obtain the updated dependency map which will have + # a `knownDeps` list for all our direct dependencies and update the + # entry for the input `drvPath`. + (bind + get + (populatedDepMap: + (setAttr drvPath (entryPoint // { + knownDeps = + lib.unique ( + initiallyKnownDeps + ++ builtins.concatLists ( + builtins.map + (dep: populatedDepMap.${dep}.knownDeps) + depsPrecalculated.wrong + ) + ); + })))) + ); + + # This function puts it all together and is exposed via `__functor`. + # + # For a list of `drvPath`s, calculate an attribute set which maps every + # `drvPath` to a set of the following form: + # + # { + # known = true /* if it is in the list of input derivation paths */; + # deps = [ + # /* list of derivation paths it depends on directly */ + # ]; + # knownDeps = [ + # /* list of the closest derivation paths marked as known this + # derivation depends on. + # */ + # ]; + # } + knownDrvDepMap = knownDrvPaths: + run + (plainDrvDepMap knownDrvPaths) + (after + (for_ + knownDrvPaths + insertKnownDeps) + get); + + # + # Other things based on knownDrvDepMap + # + + # Create a SVG visualizing `knownDrvDepMap`. Nodes are identified by derivation + # name, so multiple entries can be collapsed if they have the same name. + # + # Type: [drv] -> drv + knownDependencyGraph = name: drvs: + let + justName = drvPath: + builtins.substring + (builtins.stringLength builtins.storeDir + 1 + 32 + 1) + (builtins.stringLength drvPath) + (unsafeDiscardStringContext drvPath); + + gv = pkgs.writeText "${name}-dependency-analysis.gv" '' + digraph depot { + ${ + (lib.concatStringsSep "\n" + (lib.mapAttrsToList (name: value: + if !value.known then "" + else lib.concatMapStringsSep "\n" + (knownDep: " \"${justName name}\" -> \"${justName knownDep}\"") + value.knownDeps + ) + (depot.nix.dependency-analyzer ( + drvsToPaths drvs + )))) + } + } + ''; + in + + pkgs.runCommand "${name}-dependency-analysis.svg" + { + nativeBuildInputs = [ + pkgs.buildPackages.graphviz + ]; + } + "dot -Tsvg < ${gv} > $out"; +in + +{ + __functor = _: knownDrvDepMap; + + inherit knownDependencyGraph plainDrvDepMap drvsToPaths; +} diff --git a/nix/dependency-analyzer/examples/ci-targets.nix b/nix/dependency-analyzer/examples/ci-targets.nix new file mode 100644 index 0000000000..597abd4109 --- /dev/null +++ b/nix/dependency-analyzer/examples/ci-targets.nix @@ -0,0 +1,12 @@ +{ depot, lib, ... }: + +( + depot.nix.dependency-analyzer.knownDependencyGraph + "depot" + depot.ci.targets +).overrideAttrs (old: { + # Causes an infinite recursion via ci.targets otherwise + meta = lib.recursiveUpdate (old.meta or { }) { + ci.skip = true; + }; +}) diff --git a/nix/dependency-analyzer/examples/lisp.nix b/nix/dependency-analyzer/examples/lisp.nix new file mode 100644 index 0000000000..775eb9ab57 --- /dev/null +++ b/nix/dependency-analyzer/examples/lisp.nix @@ -0,0 +1,5 @@ +{ depot, lib, ... }: + +depot.nix.dependency-analyzer.knownDependencyGraph "3p-lisp" ( + builtins.filter lib.isDerivation (builtins.attrValues depot.third_party.lisp) +) diff --git a/nix/dependency-analyzer/tests/default.nix b/nix/dependency-analyzer/tests/default.nix new file mode 100644 index 0000000000..79ac127e92 --- /dev/null +++ b/nix/dependency-analyzer/tests/default.nix @@ -0,0 +1,36 @@ +{ depot, lib, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + assertEq + it + ; + + inherit (depot.nix.dependency-analyzer) + plainDrvDepMap + drvsToPaths + ; + + knownDrvs = drvsToPaths ( + builtins.filter lib.isDerivation (builtins.attrValues depot.third_party.lisp) + ); + exampleMap = plainDrvDepMap knownDrvs; + + # These will be needed to index into the attribute set which can't have context + # in the attribute names. + knownDrvsNoContext = builtins.map builtins.unsafeDiscardStringContext knownDrvs; +in + +runTestsuite "dependency-analyzer" [ + (it "checks plainDrvDepMap properties" [ + (assertEq "all known drvs are marked known" + (builtins.all (drv: exampleMap.${drv}.known) knownDrvsNoContext) + true) + (assertEq "no unknown drv is marked known" + (builtins.all (entry: !entry.known) ( + builtins.attrValues (builtins.removeAttrs exampleMap knownDrvsNoContext) + )) + true) + ]) +] diff --git a/nix/emptyDerivation/OWNERS b/nix/emptyDerivation/OWNERS index a742d0d22b..a640227914 100644 --- a/nix/emptyDerivation/OWNERS +++ b/nix/emptyDerivation/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - Profpatsch +Profpatsch diff --git a/nix/lazy-deps/default.nix b/nix/lazy-deps/default.nix index 3cce48d8a5..fbdb30b38e 100644 --- a/nix/lazy-deps/default.nix +++ b/nix/lazy-deps/default.nix @@ -9,11 +9,11 @@ # evaluation, and expects both `git` and `nix-build` to exist in the # user's $PATH. If required, this can be done in the shell # configuration invoking this function. -{ pkgs, ... }: +{ pkgs, lib, ... }: let inherit (builtins) attrNames attrValues mapAttrs; - inherit (pkgs.lib) concatStringsSep; + inherit (lib) fix concatStringsSep; # Create the case statement for a command invocations, optionally # overriding the `TARGET_TOOL` variable. @@ -28,20 +28,21 @@ let invocations = tools: concatStringsSep "\n" (attrValues (mapAttrs invoke tools)); in +fix (self: # Attribute set of tools that should be lazily-added to the $PATH. - - # The name of each attribute is used as the command name (on $PATH). - # It must contain the keys 'attr' (containing the Nix attribute path - # to the tool's derivation from the top-level), and may optionally - # contain the key 'cmd' to override the name of the binary inside the - # derivation. +# +# The name of each attribute is used as the command name (on $PATH). +# It must contain the keys 'attr' (containing the Nix attribute path +# to the tool's derivation from the top-level), and may optionally +# contain the key 'cmd' to override the name of the binary inside the +# derivation. tools: -pkgs.writeTextFile { - name = "lazy-dispatch"; - executable = true; - destination = "/bin/__dispatch"; +pkgs.runCommandNoCC "lazy-dispatch" +{ + passthru.overrideDeps = newTools: self (tools // newTools); + passthru.tools = tools; text = '' #!${pkgs.runtimeShell} @@ -68,8 +69,23 @@ pkgs.writeTextFile { exec "''${TARGET_TOOL}" "''${@}" ''; - checkPhase = '' - ${pkgs.stdenv.shellDryRun} "$target" - ${concatStringsSep "\n" (map link (attrNames tools))} - ''; + # Access this to get a compatible nix-shell + passthru.devShell = pkgs.mkShellNoCC { + name = "${self.name}-shell"; + packages = [ self ]; + }; } + '' + # Write the dispatch code + target=$out/bin/__dispatch + mkdir -p "$(dirname "$target")" + echo "$text" > $target + chmod +x $target + + # Add symlinks from all the tools to the dispatch + ${concatStringsSep "\n" (map link (attrNames tools))} + + # Check that it's working-ish + ${pkgs.stdenv.shellDryRun} $target + '' +) diff --git a/nix/nint/OWNERS b/nix/nint/OWNERS index f16dd105d7..2e95807063 100644 --- a/nix/nint/OWNERS +++ b/nix/nint/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni +sterni diff --git a/nix/nix-1p/README.md b/nix/nix-1p/README.md index bdb49c7def..309eddb51e 100644 --- a/nix/nix-1p/README.md +++ b/nix/nix-1p/README.md @@ -1,3 +1,8 @@ +> [!TIP] +> Are you interested in hacking on Nix projects for a week, together +> with other Nix users? Do you have time at the end of August? Great, +> come join us at [Volga Sprint](https://volgasprint.org/)! + Nix - A One Pager ================= @@ -9,6 +14,9 @@ Unless otherwise specified, the word "Nix" refers only to the language below. Please file an issue if something in here confuses you or you think something important is missing. +If you have Nix installed, you can try the examples below by running `nix repl` +and entering code snippets there. + <!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> **Table of Contents** @@ -16,6 +24,7 @@ important is missing. - [Language constructs](#language-constructs) - [Primitives / literals](#primitives--literals) - [Operators](#operators) + - [`//` (merge) operator](#-merge-operator) - [Variable bindings](#variable-bindings) - [Functions](#functions) - [Multiple arguments (currying)](#multiple-arguments-currying) @@ -46,8 +55,7 @@ Nix is: any dependency between operations is established by depending on *data* from previous operations. - Everything in Nix is an expression, meaning that every directive returns - some kind of data. + Any valid piece of Nix code is an *expression* that returns a value. Evaluating a Nix expression *yields a single data structure*, it does not execute a sequence of operations. @@ -110,23 +118,52 @@ rec { a = 15; b = a * 2; } Nix has several operators, most of which are unsurprising: -| Syntax | Description | -|----------------------|-----------------------------------------------------------------------------| -| `+`, `-`, `*`, `/` | Numerical operations | -| `+` | String concatenation | -| `++` | List concatenation | -| `==` | Equality | -| `>`, `>=`, `<`, `<=` | Ordering comparators | -| `&&` | Logical `AND` | -| <code>||</code> | Logical `OR` | -| `e1 -> e2` | Logical implication (i.e. <code>!e1 || e2</code>) | -| `!` | Boolean negation | -| `set.attr` | Access attribute `attr` in attribute set `set` | -| `set ? attribute` | Test whether attribute set contains an attribute | -| `left // right` | Merge `left` & `right` attribute sets, with the right set taking precedence | - -Make sure to understand the `//`-operator, as it is used quite a lot and is -probably the least familiar one. +| Syntax | Description | +|---------------------------|-----------------------------------------------------------------------------| +| `+`, `-`, `*`, `/` | Numerical operations | +| `+` | String concatenation | +| `++` | List concatenation | +| `==` | Equality | +| `>`, `>=`, `<`, `<=` | Ordering comparators | +| `&&` | Logical `AND` | +| <code>||</code> | Logical `OR` | +| `e1 -> e2` | Logical implication (i.e. <code>!e1 || e2</code>) | +| `!` | Boolean negation | +| `set.attr` | Access attribute `attr` in attribute set `set` | +| `set ? attribute` | Test whether attribute set contains an attribute | +| `left // right` | Merge `left` & `right` attribute sets, with the right set taking precedence | + + +### `//` (merge) operator + +The `//`-operator is used pervasively in Nix code. You should familiarise +yourself with it, as it is likely also the least familiar one. + +It merges the left and right attribute sets given to it: + +```nix +{ a = 1; } // { b = 2; } + +# yields { a = 1; b = 2; } +``` + +Values from the right side take precedence: + +```nix +{ a = "left"; } // { a = "right"; } + +# yields { a = "right"; } +``` + +The merge operator does *not* recursively merge attribute sets; + +```nix +{ a = { b = 1; }; } // { a = { c = 2; }; } + +# yields { a = { c = 2; }; } +``` + +Helper functions for recursive merging exist in the [`lib` library](#pkgslib). ## Variable bindings @@ -323,10 +360,18 @@ let attrs = { a = 15; b = 2; }; in with attrs; a + b # 'a' and 'b' become variables in the scope following 'with' ``` +The scope of a `with`-"block" is the expression immediately following the +semicolon, i.e.: + +```nix +let attrs = { /* some attributes */ }; +in with attrs; (/* this is the scope of the `with` */) +``` + ## `import` / `NIX_PATH` / `<entry>` -Nix files can import each other by using the `import` keyword and a literal -path: +Nix files can import each other by using the builtin `import` function and a +literal path: ```nix # assuming there is a file lib.nix with some useful functions @@ -334,6 +379,8 @@ let myLib = import ./lib.nix; in myLib.usefulFunction 42 ``` +The `import` function will read and evaluate the file, and return its Nix value. + Nix files often begin with a function header to pass parameters into the rest of the file, so you will often see imports of the form `import ./some-file { ... }`. @@ -569,31 +616,33 @@ but have the modification above be reflected in the imported package set: ```nix let - overlay = (self: super: { - someProgram = super.someProgram.overrideAttrs(old: { + overlay = (final: prev: { + someProgram = prev.someProgram.overrideAttrs(old: { configureFlags = old.configureFlags or [] ++ ["--mimic-threaten-tag"]; }); }); in import <nixpkgs> { overlays = [ overlay ]; } ``` -The overlay function receives two arguments, `self` and `super`. `self` is +The overlay function receives two arguments, `final` and `prev`. `final` is the [fixed point][fp] of the overlay's evaluation, i.e. the package set -*including* the new packages and `super` is the "original" package set. +*including* the new packages and `prev` is the "original" package set. See the Nix manual sections [on overrides][] and [on overlays][] for more -details. +details (note: the convention has moved away from using `self` in favor of +`final`, and `prev` instead of `super`, but the documentation has not been +updated to reflect this). [currying]: https://en.wikipedia.org/wiki/Currying -[builtins]: https://nixos.org/nix/manual/#ssec-builtins +[builtins]: https://nixos.org/manual/nix/stable/language/builtins [nixpkgs]: https://github.com/NixOS/nixpkgs [lib-src]: https://github.com/NixOS/nixpkgs/tree/master/lib [nixdoc]: https://github.com/tazjin/nixdoc -[lib-manual]: https://nixos.org/nixpkgs/manual/#sec-functions-library -[channels]: https://nixos.org/nix/manual/#sec-channels -[trivial builders]: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/trivial-builders.nix -[smkd]: https://nixos.org/nixpkgs/manual/#chap-stdenv -[drv-manual]: https://nixos.org/nix/manual/#ssec-derivation +[lib-manual]: https://nixos.org/manual/nixpkgs/stable/#sec-functions-library +[channels]: https://nixos.org/manual/nix/stable/command-ref/files/channels +[trivial builders]: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/trivial-builders/default.nix +[smkd]: https://nixos.org/manual/nixpkgs/stable/#chap-stdenv +[drv-manual]: https://nixos.org/manual/nix/stable/language/derivations [fp]: https://github.com/NixOS/nixpkgs/blob/master/lib/fixed-points.nix -[on overrides]: https://nixos.org/nixpkgs/manual/#sec-overrides -[on overlays]: https://nixos.org/nixpkgs/manual/#chap-overlays +[on overrides]: https://nixos.org/manual/nixpkgs/stable/#chap-overrides +[on overlays]: https://nixos.org/manual/nixpkgs/stable/#chap-overlays diff --git a/nix/readTree/README.md b/nix/readTree/README.md index f8bbe2255e..5d430d1cfc 100644 --- a/nix/readTree/README.md +++ b/nix/readTree/README.md @@ -52,14 +52,17 @@ true;` attribute merged into it. `readTree` will follow any subdirectories of a tree and import all Nix files, with some exceptions: +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it + evaluates to a set, *merged* with the children. If it evaluates to + anything other than a set, else the children are *not traversed*. +* A folder can opt out from readTree completely by containing a + `.skip-tree` file. The content of the file is not read. These + folders will be missing completely from the readTree structure. * A folder can declare that its children are off-limit by containing a `.skip-subtree` file. Since the content of the file is not checked, it can be useful to leave a note for a human in the file. -* If a folder contains a `default.nix` file, no *sibling* Nix files will be - imported - however children are traversed as normal. -* If a folder contains a `default.nix` it is loaded and, if it evaluates to a - set, *merged* with the children. If it evaluates to anything else the children - are *not traversed*. * The `default.nix` of the top-level folder on which readTree is called is **not** read to avoid infinite recursion (as, presumably, this file is where readTree itself is called). diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix index ba3363d8d6..4a745ce33c 100644 --- a/nix/readTree/default.nix +++ b/nix/readTree/default.nix @@ -2,7 +2,7 @@ # Copyright (c) 2020-2021 The TVL Authors # SPDX-License-Identifier: MIT # -# Provides a function to automatically read a a filesystem structure +# Provides a function to automatically read a filesystem structure # into a Nix attribute set. # # Called with an attribute set taking the following arguments: @@ -41,7 +41,8 @@ let readDirVisible = path: let children = readDir path; - isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + # skip hidden files, except for those that contain special instructions to readTree + isVisible = f: f == ".skip-subtree" || f == ".skip-tree" || (substring 0 1 f) != "."; names = filter isVisible (attrNames children); in listToAttrs (map @@ -80,22 +81,45 @@ let importFile = args: scopedArgs: path: parts: filter: let importedFile = - if scopedArgs != { } + if scopedArgs != { } && builtins ? scopedImport # For tvix then builtins.scopedImport scopedArgs path else import path; pathType = builtins.typeOf importedFile; in if pathType != "lambda" - then builtins.throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + then throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" else importedFile (filter parts (argsWithPath args parts)); nixFileName = file: let res = match "(.*)\\.nix" file; in if res == null then null else head res; - readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + # Internal implementation of readTree, which handles things like the + # skipping of trees and subtrees. + # + # This method returns an attribute sets with either of two shapes: + # + # { ok = ...; } # a tree was read successfully + # { skip = true; } # a tree was skipped + # + # The higher-level `readTree` method assembles the final attribute + # set out of these results at the top-level, and the internal + # `children` implementation unwraps and processes nested trees. + readTreeImpl = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: let dir = readDirVisible initPath; + + # Determine whether any part of this tree should be skipped. + # + # Adding a `.skip-subtree` file will still allow the import of + # the current node's "default.nix" file, but stop recursion + # there. + # + # Adding a `.skip-tree` file will completely ignore the folder + # in which this file is located. + skipTree = hasAttr ".skip-tree" dir; + skipSubtree = skipTree || hasAttr ".skip-subtree" dir; + joinChild = c: initPath + ("/" + c); self = @@ -103,19 +127,17 @@ let then { __readTree = [ ]; } else importFile args scopedArgs initPath parts argsFilter; - # Import subdirectories of the current one, unless the special - # `.skip-subtree` file exists which makes readTree ignore the - # children. + # Import subdirectories of the current one, unless any skip + # instructions exist. # # This file can optionally contain information on why the tree # should be ignored, but its content is not inspected by # readTree filterDir = f: dir."${f}" == "directory"; - children = if hasAttr ".skip-subtree" dir then [ ] else - map + filteredChildren = map (c: { name = c; - value = readTree { + value = readTreeImpl { inherit argsFilter scopedArgs; args = args; initPath = (joinChild c); @@ -125,9 +147,15 @@ let }) (filter filterDir (attrNames dir)); + # Remove skipped children from the final set, and unwrap the + # result set. + children = + if skipSubtree then [ ] + else map ({ name, value }: { inherit name; value = value.ok; }) (filter (child: child.value ? ok) filteredChildren); + # Import Nix files nixFiles = - if hasAttr ".skip-subtree" dir then [ ] + if skipSubtree then [ ] else filter (f: f != null) (map nixFileName (attrNames dir)); nixChildren = map (c: @@ -154,9 +182,23 @@ let ); in - if isAttrs nodeValue - then merge nodeValue (allChildren // (marker parts allChildren)) - else nodeValue; + if skipTree + then { skip = true; } + else { + ok = + if isAttrs nodeValue + then merge nodeValue (allChildren // (marker parts allChildren)) + else nodeValue; + }; + + # Top-level implementation of readTree itself. + readTree = args: + let + tree = readTreeImpl args; + in + if tree ? skip + then throw "Top-level folder has a .skip-tree marker and could not be read by readTree!" + else tree.ok; # Helper function to fetch subtargets from a target. This is a # temporary helper to warn on the use of the `meta.targets` diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix index fcca141714..6f9eb02eff 100644 --- a/nix/readTree/tests/default.nix +++ b/nix/readTree/tests/default.nix @@ -41,6 +41,16 @@ let }; traversal-logic = it "corresponds to the traversal logic in the README" [ + (assertEq "skip-tree/a is read" + tree-tl.skip-tree.a + "a is read normally") + (assertEq "skip-tree does not contain b" + (builtins.attrNames tree-tl.skip-tree) + [ "__readTree" "__readTreeChildren" "a" ]) + (assertEq "skip-tree children list does not contain b" + tree-tl.skip-tree.__readTreeChildren + [ "a" ]) + (assertEq "skip subtree default.nix is read" tree-tl.skip-subtree.but "the default.nix is still read") diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix new file mode 100644 index 0000000000..186488be3c --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/a/default.nix @@ -0,0 +1 @@ +_: "a is read normally" diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree new file mode 100644 index 0000000000..34936b45d1 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/.skip-tree @@ -0,0 +1 @@ +b subfolder should be skipped completely diff --git a/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix new file mode 100644 index 0000000000..7903f8e95a --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-tree/b/default.nix @@ -0,0 +1 @@ +throw "b is skipped completely" diff --git a/nix/renderMarkdown/default.nix b/nix/renderMarkdown/default.nix index 8d6b31cfcc..8759ada0fe 100644 --- a/nix/renderMarkdown/default.nix +++ b/nix/renderMarkdown/default.nix @@ -3,6 +3,19 @@ with depot.nix.yants; -defun [ path drv ] (file: pkgs.runCommandNoCC "${file}.rendered.html" { } '' - cat ${file} | ${depot.tools.cheddar}/bin/cheddar --about-filter ${file} > $out -'') +let + args = struct "args" { + path = path; + tagfilter = option bool; + }; +in +defun [ (either path args) drv ] + (arg: pkgs.runCommand "${arg.path or arg}.rendered.html" { } + ( + let + tagfilter = if (arg.tagfilter or true) then "" else "--no-tagfilter"; + in + '' + cat ${arg.path or arg} | ${depot.tools.cheddar}/bin/cheddar --about-filter ${tagfilter} ${arg.path or arg} > $out + '' + )) diff --git a/nix/sparseTree/OWNERS b/nix/sparseTree/OWNERS index fdf6d72040..2e95807063 100644 --- a/nix/sparseTree/OWNERS +++ b/nix/sparseTree/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni \ No newline at end of file +sterni diff --git a/nix/sparseTree/default.nix b/nix/sparseTree/default.nix index 16fc9b6103..35fa459e1c 100644 --- a/nix/sparseTree/default.nix +++ b/nix/sparseTree/default.nix @@ -2,22 +2,33 @@ # and directories if they are listed in a supplied list: # # # A very minimal depot -# sparseTree ./depot [ -# ./default.nix -# ./depot/nix/readTree/default.nix -# ./third_party/nixpkgs -# ./third_party/overlays -# ] +# sparseTree { +# root = ./depot; +# paths = [ +# ./default.nix +# ./depot/nix/readTree/default.nix +# ./third_party/nixpkgs +# ./third_party/overlays +# ]; +# } { pkgs, lib, ... }: -# root path to use as a reference point -root: -# list of paths below `root` that should be -# included in the resulting directory -# -# If path, need to refer to the actual file / directory to be included. -# If a string, it is treated as a string relative to the root. -paths: +{ + # root path to use as a reference point + root +, # list of paths below `root` that should be + # included in the resulting directory + # + # If path, need to refer to the actual file / directory to be included. + # If a string, it is treated as a string relative to the root. + paths +, # (optional) name to use for the derivation + # + # This should always be set when using roots that do not have + # controlled names, such as when passing the top-level of a git + # repository (e.g. `depot.path.origSrc`). + name ? builtins.baseNameOf root +}: let rootLength = builtins.stringLength (toString root); @@ -45,7 +56,6 @@ let let withLeading = p: if builtins.substring 0 1 p == "/" then p else "/" + p; fullPath = - /**/ if builtins.isPath path then path else if builtins.isString path then (root + withLeading path) else builtins.throw "Unsupported path type ${builtins.typeOf path}"; @@ -64,7 +74,7 @@ in # TODO(sterni): teach readTree to also read symlinked directories, # so we ln -sT instead of cp -aT. -pkgs.runCommandNoCC "sparse-${builtins.baseNameOf root}" { } ( +pkgs.runCommand "sparse-${name}" { } ( lib.concatMapStrings ({ src, dst }: '' mkdir -p "$(dirname "$out${dst}")" diff --git a/nix/stateMonad/default.nix b/nix/stateMonad/default.nix new file mode 100644 index 0000000000..209412e099 --- /dev/null +++ b/nix/stateMonad/default.nix @@ -0,0 +1,76 @@ +# Simple state monad represented as +# +# stateMonad s a = s -> { state : s; value : a } +# +{ ... }: + +rec { + # + # Monad + # + + # Type: stateMonad s a -> (a -> stateMonad s b) -> stateMonad s b + bind = action: f: state: + let + afterAction = action state; + in + (f afterAction.value) afterAction.state; + + # Type: stateMonad s a -> stateMonad s b -> stateMonad s b + after = action1: action2: state: action2 (action1 state).state; + + # Type: stateMonad s (stateMonad s a) -> stateMonad s a + join = action: bind action (action': action'); + + # Type: [a] -> (a -> stateMonad s b) -> stateMonad s null + for_ = xs: f: + builtins.foldl' + (laterAction: x: + after (f x) laterAction + ) + (pure null) + xs; + + # + # Applicative + # + + # Type: a -> stateMonad s a + pure = value: state: { inherit state value; }; + + # TODO(sterni): <*>, lift2, … + + # + # Functor + # + + # Type: (a -> b) -> stateMonad s a -> stateMonad s b + fmap = f: action: bind action (result: pure (f result)); + + # + # State Monad + # + + # Type: (s -> s) -> stateMonad s null + modify = f: state: { value = null; state = f state; }; + + # Type: stateMonad s s + get = state: { value = state; inherit state; }; + + # Type: s -> stateMonad s null + set = new: modify (_: new); + + # Type: str -> stateMonad set set.${str} + getAttr = attr: fmap (state: state.${attr}) get; + + # Type: str -> (any -> any) -> stateMonad s null + modifyAttr = attr: f: modify (state: state // { + ${attr} = f state.${attr}; + }); + + # Type: str -> any -> stateMonad s null + setAttr = attr: value: modifyAttr attr (_: value); + + # Type: s -> stateMonad s a -> a + run = state: action: (action state).value; +} diff --git a/nix/stateMonad/tests/default.nix b/nix/stateMonad/tests/default.nix new file mode 100644 index 0000000000..c3cb5c99b5 --- /dev/null +++ b/nix/stateMonad/tests/default.nix @@ -0,0 +1,110 @@ +{ depot, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + inherit (depot.nix.stateMonad) + pure + run + join + fmap + bind + get + set + modify + after + for_ + getAttr + setAttr + modifyAttr + ; + + runStateIndependent = run (throw "This should never be evaluated!"); +in + +runTestsuite "stateMonad" [ + (it "behaves correctly independent of state" [ + (assertEq "pure" (runStateIndependent (pure 21)) 21) + (assertEq "join pure" (runStateIndependent (join (pure (pure 42)))) 42) + (assertEq "fmap pure" (runStateIndependent (fmap (builtins.mul 2) (pure 21))) 42) + (assertEq "bind pure" (runStateIndependent (bind (pure 12) (x: pure x))) 12) + ]) + (it "behaves correctly with an integer state" [ + (assertEq "get" (run 42 get) 42) + (assertEq "after set get" (run 21 (after (set 42) get)) 42) + (assertEq "after modify get" (run 21 (after (modify (builtins.mul 2)) get)) 42) + (assertEq "fmap get" (run 40 (fmap (builtins.add 2) get)) 42) + (assertEq "stateful sum list" + (run 0 (after + (for_ + [ + 15 + 12 + 10 + 5 + ] + (x: modify (builtins.add x))) + get)) + 42) + ]) + (it "behaves correctly with an attr set state" [ + (assertEq "getAttr" (run { foo = 42; } (getAttr "foo")) 42) + (assertEq "after setAttr getAttr" + (run { foo = 21; } (after (setAttr "foo" 42) (getAttr "foo"))) + 42) + (assertEq "after modifyAttr getAttr" + (run { foo = 10.5; } + (after + (modifyAttr "foo" (builtins.mul 4)) + (getAttr "foo"))) + 42) + (assertEq "fmap getAttr" + (run { foo = 21; } (fmap (builtins.mul 2) (getAttr "foo"))) + 42) + (assertEq "after setAttr to insert getAttr" + (run { } (after (setAttr "foo" 42) (getAttr "foo"))) + 42) + (assertEq "insert permutations" + (run + { + a = 2; + b = 3; + c = 5; + } + (after + (bind get + (state: + let + names = builtins.attrNames state; + in + for_ names (name1: + for_ names (name2: + # this is of course a bit silly, but making it more cumbersome + # makes sure the test exercises more of the code. + (bind (getAttr name1) + (value1: + (bind (getAttr name2) + (value2: + setAttr "${name1}_${name2}" (value1 * value2))))))))) + get)) + { + a = 2; + b = 3; + c = 5; + a_a = 4; + a_b = 6; + a_c = 10; + b_a = 6; + b_b = 9; + b_c = 15; + c_c = 25; + c_a = 10; + c_b = 15; + } + ) + ]) +] diff --git a/nix/tag/default.nix b/nix/tag/default.nix index 0038404460..2955656323 100644 --- a/nix/tag/default.nix +++ b/nix/tag/default.nix @@ -78,7 +78,7 @@ let # Like `discrDef`, but fail if there is no match. discr = fs: v: let res = discrDef null fs v; in - assert lib.assertMsg (res != null) + assert lib.assertMsg (res != { }) "tag.discr: No predicate found that matches ${lib.generators.toPretty {} v}"; res; diff --git a/nix/tag/tests.nix b/nix/tag/tests.nix index bcc42c758a..e0085b4837 100644 --- a/nix/tag/tests.nix +++ b/nix/tag/tests.nix @@ -4,6 +4,7 @@ let inherit (depot.nix.runTestsuite) runTestsuite assertEq + assertThrows it ; @@ -50,6 +51,10 @@ let { int = lib.isInt; } ] "foo") { def = "foo"; }) + (assertThrows "throws failing to match" + (discr [ + { fish = x: x == 42; } + ] 21)) ]; match-test = it "can match things" [ diff --git a/nix/utils/OWNERS b/nix/utils/OWNERS index f16dd105d7..2e95807063 100644 --- a/nix/utils/OWNERS +++ b/nix/utils/OWNERS @@ -1,3 +1 @@ -inherited: true -owners: - - sterni +sterni diff --git a/nix/utils/default.nix b/nix/utils/default.nix index a29f346519..0c6c88fafd 100644 --- a/nix/utils/default.nix +++ b/nix/utils/default.nix @@ -43,21 +43,6 @@ let else builtins.throw "Don't know how to get (base)name of " + lib.generators.toPretty { } p; - /* Retrieves the drvPath attribute from a given derivation, but ensures that - the resulting string only depends on the `.drv` file in the nix store and - not on its realised outputs as well. - - Type: drv -> string - */ - onlyDrvPath = drv: - let - inherit (drv) drvPath; - unsafeDrvPath = builtins.unsafeDiscardStringContext drvPath; - in - builtins.appendContext unsafeDrvPath { - ${unsafeDrvPath} = { path = true; }; - }; - /* Query the type of a path exposing the same information as would be by `builtins.readDir`, but for a single, specific target path. @@ -167,7 +152,6 @@ in { inherit storePathName - onlyDrvPath pathType isDirectory isRegularFile diff --git a/nix/utils/tests/default.nix b/nix/utils/tests/default.nix index d5159a8433..344a1771d7 100644 --- a/nix/utils/tests/default.nix +++ b/nix/utils/tests/default.nix @@ -15,7 +15,6 @@ let isSymlink pathType storePathName - onlyDrvPath ; assertUtilsPred = msg: act: exp: [ @@ -92,19 +91,9 @@ let (storePathName cleanedSource) cleanedSource.name) ]; - - onlyDrvPathTests = it "correctly updates the string context of drvPath" [ - (assertEq "onlyDrvPath only produces path dependencies" - (builtins.all - (dep: dep.path or false) - (builtins.attrValues - (builtins.getContext (onlyDrvPath depot.tools.cheddar)))) - true) - ]; in runTestsuite "nix.utils" [ pathPredicates storePathNameTests - onlyDrvPathTests ] diff --git a/nix/writeTree/OWNERS b/nix/writeTree/OWNERS new file mode 100644 index 0000000000..b381c4e660 --- /dev/null +++ b/nix/writeTree/OWNERS @@ -0,0 +1 @@ +aspen diff --git a/nix/writeTree/default.nix b/nix/writeTree/default.nix new file mode 100644 index 0000000000..0c7c2a130f --- /dev/null +++ b/nix/writeTree/default.nix @@ -0,0 +1,43 @@ +{ depot, lib, pkgs, ... }: +let + inherit (lib) fix pipe mapAttrsToList isAttrs concatLines isString isDerivation isPath; + + # TODO(sterni): move to //nix/utils with clearer naming and alternative similar to lib.types.path + isPathLike = value: + isPath value + || isDerivation value + || (isString value && builtins.hasContext value); + + esc = s: lib.escapeShellArg /* ensure paths import into store */ "${s}"; + + writeTreeAtPath = path: tree: + '' + mkdir -p "$out/"${esc path} + '' + + pipe tree [ + (mapAttrsToList (k: v: + if isPathLike v then + "cp -R --reflink=auto ${v} \"$out/\"${esc path}/${esc k}" + else if lib.isAttrs v then + writeTreeAtPath (path + "/" + k) v + else + throw "invalid type (expected path, derivation, string with context, or attrs)")) + concatLines + ]; + + /* Create a directory tree specified by a Nix attribute set structure. + + Each value in `tree` should either be a file, a directory, or another tree + attribute set. Those paths will be written to a directory tree + corresponding to the structure of the attribute set. + + Type: string -> attrSet -> derivation + */ + writeTree = name: tree: + pkgs.runCommandLocal name { } (writeTreeAtPath "" tree); +in + +# __functor trick so readTree can add the tests attribute +{ + __functor = _: writeTree; +} diff --git a/nix/writeTree/tests/default.nix b/nix/writeTree/tests/default.nix new file mode 100644 index 0000000000..c5858ee96e --- /dev/null +++ b/nix/writeTree/tests/default.nix @@ -0,0 +1,93 @@ +{ depot, pkgs, lib, ... }: + +let + inherit (pkgs) runCommand writeText writeTextFile; + inherit (depot.nix) writeTree; + + checkTree = name: tree: expected: + runCommand "writeTree-test-${name}" + { + nativeBuildInputs = [ pkgs.buildPackages.lr ]; + passAsFile = [ "expected" ]; + inherit expected; + } '' + actualPath="$NIX_BUILD_TOP/actual" + cd ${lib.escapeShellArg (writeTree name tree)} + lr . > "$actualPath" + diff -u "$expectedPath" "$actualPath" | tee "$out" + ''; +in + +depot.nix.readTree.drvTargets { + empty = checkTree "empty" { } + '' + . + ''; + + simple-paths = checkTree "simple" + { + writeTree = { + meta = { + "owners.txt" = ../OWNERS; + }; + "code.nix" = ../default.nix; + all-tests = ./.; + nested.dirs.eval-time = builtins.toFile "owothia" '' + hold me owo + ''; + }; + } + '' + . + ./writeTree + ./writeTree/all-tests + ./writeTree/all-tests/default.nix + ./writeTree/code.nix + ./writeTree/meta + ./writeTree/meta/owners.txt + ./writeTree/nested + ./writeTree/nested/dirs + ./writeTree/nested/dirs/eval-time + ''; + + empty-dirs = checkTree "empty-dirs" + { + this.dir.is.empty = { }; + so.is.this.one = { }; + } + '' + . + ./so + ./so/is + ./so/is/this + ./so/is/this/one + ./this + ./this/dir + ./this/dir/is + ./this/dir/is/empty + ''; + + drvs = checkTree "drvs" + { + file-drv = writeText "road.txt" '' + Any road followed precisely to its end leads precisely nowhere. + ''; + dir-drv = writeTextFile { + name = "dir-of-text"; + destination = "/text/in/more/dirs.txt"; + text = '' + Climb the mountain just a little bit to test that it’s a mountain. + From the top of the mountain, you cannot see the mountain. + ''; + }; + } + '' + . + ./dir-drv + ./dir-drv/text + ./dir-drv/text/in + ./dir-drv/text/in/more + ./dir-drv/text/in/more/dirs.txt + ./file-drv + ''; +} |