diff options
Diffstat (limited to 'nix')
| -rw-r--r-- | nix/buildGo/default.nix | 2 | ||||
| -rw-r--r-- | nix/buildGo/external/main.go | 7 | ||||
| -rw-r--r-- | nix/buildkite/default.nix | 54 | ||||
| -rw-r--r-- | nix/dependency-analyzer/default.nix | 53 | ||||
| -rw-r--r-- | nix/nix-1p/README.md | 5 |
5 files changed, 84 insertions, 37 deletions
diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix index 9e0a5d6d87..c93642a127 100644 --- a/nix/buildGo/default.nix +++ b/nix/buildGo/default.nix @@ -42,8 +42,6 @@ let xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); - pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p); - # Add an `overrideGo` attribute to a function result that works # similar to `overrideAttrs`, but is used specifically for the # arguments passed to Go builders. diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go index a77c43b371..4402a8eb86 100644 --- a/nix/buildGo/external/main.go +++ b/nix/buildGo/external/main.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "go/build" - "io/ioutil" "log" "os" "path" @@ -74,8 +73,8 @@ func findGoDirs(at string) ([]string, error) { } goDirs := []string{} - for k, _ := range dirSet { - goDirs = append(goDirs, k) + for goDir := range dirSet { + goDirs = append(goDirs, goDir) } return goDirs, nil @@ -148,7 +147,7 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk } func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { - f, err := ioutil.ReadFile(from) + f, err := os.ReadFile(from) if err != nil { return } diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 6e158ae6c6..9abba9408a 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -27,8 +27,27 @@ let inherit (pkgs) lib runCommand writeText; inherit (depot.nix.readTree) mkLabel; + + inherit (depot.nix) dependency-analyzer; in rec { + # Create a unique key for the buildkite pipeline based on the given derivation + # or drvPath. A consequence of using such keys is that every derivation may + # only be exposed as a single, unique step in the pipeline. + keyForDrv = drvOrPath: + let + drvPath = + if lib.isDerivation drvOrPath then drvOrPath.drvPath + else if lib.isString drvOrPath then drvOrPath + else builtins.throw "keyForDrv: expected string or derivation"; + + # Only use the drv hash to prevent escaping problems. Buildkite also has a + # limit of 100 characters on keys. + in + "drv-" + (builtins.substring 0 32 + (builtins.baseNameOf (unsafeDiscardStringContext drvPath)) + ); + # Given an arbitrary attribute path generate a Nix expression which obtains # this from the root of depot (assumed to be ./.). Attributes may be any # Nix strings suitable as attribute names, not just Nix literal-safe strings. @@ -47,6 +66,9 @@ rec { # Create build command for an attribute path pointing to a derivation. mkBuildCommand = { attrPath, drvPath, outLink ? "result" }: concatStringsSep " " [ + # If the nix build fails, the Nix command's exit status should be used. + "set -o pipefail;" + # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on # at least one of its `outPath`s, so we need to discard the string context @@ -55,7 +77,7 @@ rec { # To make this more uniform with how nix-build(1) works, we call realpath(1) # on nix-store(1)'s output since it has the habit of printing the path of the # out link, not the store path. - "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs realpath)" + "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs -r realpath)" # Since we don't gcroot the derivation files, they may be deleted by the # garbage collector. In that case we can reevaluate and build the attribute @@ -70,15 +92,28 @@ rec { target.__readTree ++ lib.optionals (target ? __subtarget) [ target.__subtarget ]; + # Given a derivation (identified by drvPath) that is part of the list of + # targets passed to mkPipeline, determine all derivations that it depends on + # and are also part of the pipeline. Finally, return the keys of the steps + # that build them. This is used to populate `depends_on` in `mkStep`. + # + # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`. + getTargetPipelineDeps = targetDepMap: drvPath: + # Sanity check: We should only call this function on targets explicitly + # passed to mkPipeline. Thus it should have been passed as a “known” drv to + # dependency-analyzer. + assert targetDepMap.${drvPath}.known; + builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps; + # Create a pipeline step from a single target. - mkStep = { headBranch, parentTargetMap, target, cancelOnBuildFailing }: + mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }: let label = mkLabel target; drvPath = unsafeDiscardStringContext target.drvPath; in { label = ":nix: " + label; - key = hashString "sha1" label; + key = keyForDrv target; skip = shouldSkip { inherit label drvPath parentTargetMap; }; command = mkBuildCommand { attrPath = targetAttrPath target; @@ -90,7 +125,9 @@ rec { # Add a dependency on the initial static pipeline step which # always runs. This allows build steps uploaded in batches to # start running before all batches have been uploaded. - depends_on = [ ":init:" ] ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + depends_on = [ ":init:" ] + ++ getTargetPipelineDeps targetDepMap drvPath + ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; } // lib.optionalAttrs (target ? meta.timeout) { timeout_in_minutes = target.meta.timeout / 60; # Additional arguments to set on the step. @@ -193,12 +230,15 @@ rec { # logic/optimisation depends on knowing whether is executing. buildEnabled = elem "build" enabledPhases; + # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer. + targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets); + # Convert a target into all of its steps, separated by build # phase (as phases end up in different chunks). targetToSteps = target: let mkStepArgs = { - inherit headBranch parentTargetMap target cancelOnBuildFailing; + inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing; }; step = mkStep mkStepArgs; @@ -376,7 +416,7 @@ rec { prompt = lib.throwIf (prompt != false && phase == "build") '' In step '${label}' (from ${parentLabel}): - The 'prompt' feature can only be used by steps in the "release" + The 'prompt' feature can not be used by steps in the "build" phase, because CI builds should not be gated on manual human approvals. '' @@ -391,7 +431,7 @@ rec { commandScriptLink = "nix-buildkite-extra-step-command-script"; step = { - key = hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; + key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; skip = let diff --git a/nix/dependency-analyzer/default.nix b/nix/dependency-analyzer/default.nix index 54ff72912e..2ec8d7b5b9 100644 --- a/nix/dependency-analyzer/default.nix +++ b/nix/dependency-analyzer/default.nix @@ -16,30 +16,35 @@ let # # TODO(sterni): clean this up and expose it directDrvDeps = - if lib.versionAtLeast builtins.nixVersion "2.6" - then - # Since https://github.com/NixOS/nix/pull/1643, Nix apparently »preserves - # string context« through a readFile invocation. This has the side effect - # that it becomes possible to query the actual references a store path has. - # Not a 100% sure this is intended, but _very_ convenient for us here. - drvPath: - # if the passed path is not a derivation we can't necessarily get its - # dependencies, since it may not be representable as a Nix string due to - # NUL bytes, e.g. compressed patch files imported into the Nix store. - if builtins.match "^.+\\.drv$" drvPath == null - then [ ] - else builtins.attrNames (builtins.getContext (builtins.readFile drvPath)) - else - # For Nix < 2.6 we have to rely on HACK, namely grepping for quoted store - # path references in the file. In the future this should be replaced by - # a proper derivation parser. - drvPath: builtins.concatLists ( - builtins.filter builtins.isList ( - builtins.split - "\"(${lib.escapeRegex builtins.storeDir}/[[:alnum:]+._?=-]+.drv)\"" - (builtins.readFile drvPath) - ) - ); + let + getDeps = + if lib.versionAtLeast builtins.nixVersion "2.6" + then + # Since https://github.com/NixOS/nix/pull/1643, Nix apparently »preserves + # string context« through a readFile invocation. This has the side effect + # that it becomes possible to query the actual references a store path has. + # Not a 100% sure this is intended, but _very_ convenient for us here. + drvPath: + builtins.attrNames (builtins.getContext (builtins.readFile drvPath)) + else + # For Nix < 2.6 we have to rely on HACK, namely grepping for quoted + # store path references in the file. In the future this should be + # replaced by a proper derivation parser. + drvPath: builtins.concatLists ( + builtins.filter builtins.isList ( + builtins.split + "\"(${lib.escapeRegex builtins.storeDir}/[[:alnum:]+._?=-]+.drv)\"" + (builtins.readFile drvPath) + ) + ); + in + drvPath: + # if the passed path is not a derivation we can't necessarily get its + # dependencies, since it may not be representable as a Nix string due to + # NUL bytes, e.g. compressed patch files imported into the Nix store. + if builtins.match "^.+\\.drv$" drvPath == null + then [ ] + else getDeps drvPath; # Maps a list of derivation to the list of corresponding `drvPath`s. # diff --git a/nix/nix-1p/README.md b/nix/nix-1p/README.md index 45643032d0..309eddb51e 100644 --- a/nix/nix-1p/README.md +++ b/nix/nix-1p/README.md @@ -1,3 +1,8 @@ +> [!TIP] +> Are you interested in hacking on Nix projects for a week, together +> with other Nix users? Do you have time at the end of August? Great, +> come join us at [Volga Sprint](https://volgasprint.org/)! + Nix - A One Pager ================= |