diff options
Diffstat (limited to 'nix/buildkite/default.nix')
| -rw-r--r-- | nix/buildkite/default.nix | 240 |
1 files changed, 155 insertions, 85 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 5c46dcb333..9abba9408a 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -25,65 +25,114 @@ let toJSON unsafeDiscardStringContext; - inherit (pkgs) lib runCommandNoCC writeText; + inherit (pkgs) lib runCommand writeText; inherit (depot.nix.readTree) mkLabel; + + inherit (depot.nix) dependency-analyzer; in rec { - # Creates a Nix expression that yields the target at the specified - # location in the repository. - # - # This makes a distinction between normal targets (which physically - # exist in the repository) and subtargets (which are "virtual" - # targets exposed by a physical one) to make it clear in the build - # output which is which. - mkBuildExpr = target: + # Create a unique key for the buildkite pipeline based on the given derivation + # or drvPath. A consequence of using such keys is that every derivation may + # only be exposed as a single, unique step in the pipeline. + keyForDrv = drvOrPath: + let + drvPath = + if lib.isDerivation drvOrPath then drvOrPath.drvPath + else if lib.isString drvOrPath then drvOrPath + else builtins.throw "keyForDrv: expected string or derivation"; + + # Only use the drv hash to prevent escaping problems. Buildkite also has a + # limit of 100 characters on keys. + in + "drv-" + (builtins.substring 0 32 + (builtins.baseNameOf (unsafeDiscardStringContext drvPath)) + ); + + # Given an arbitrary attribute path generate a Nix expression which obtains + # this from the root of depot (assumed to be ./.). Attributes may be any + # Nix strings suitable as attribute names, not just Nix literal-safe strings. + mkBuildExpr = attrPath: let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; - targetExpr = foldl' descend "import ./. {}" target.__readTree; - subtargetExpr = descend targetExpr target.__subtarget; in - if target ? __subtarget then subtargetExpr else targetExpr; + foldl' descend "import ./. {}" attrPath; # Determine whether to skip a target if it has not diverged from the # HEAD branch. - shouldSkip = parentTargetMap: label: drvPath: + shouldSkip = { parentTargetMap ? { }, label, drvPath }: if (hasAttr label parentTargetMap) && parentTargetMap."${label}".drvPath == drvPath then "Target has not changed." else false; - # Create build command for a derivation target. - mkBuildCommand = target: drvPath: concatStringsSep " " [ + # Create build command for an attribute path pointing to a derivation. + mkBuildCommand = { attrPath, drvPath, outLink ? "result" }: concatStringsSep " " [ + # If the nix build fails, the Nix command's exit status should be used. + "set -o pipefail;" + # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on # at least one of its `outPath`s, so we need to discard the string context # if we don't want to build everything during pipeline construction. - "(nix-store --realise '${drvPath}' --add-root result --indirect && readlink result)" + # + # To make this more uniform with how nix-build(1) works, we call realpath(1) + # on nix-store(1)'s output since it has the habit of printing the path of the + # out link, not the store path. + "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs -r realpath)" # Since we don't gcroot the derivation files, they may be deleted by the # garbage collector. In that case we can reevaluate and build the attribute # using nix-build. - "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)" + "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr attrPath}' --show-trace --out-link '${outLink}')" ]; + # Attribute path of a target relative to the depot root. Needs to take into + # account whether the target is a physical target (which corresponds to a path + # in the filesystem) or the subtarget of a physical target. + targetAttrPath = target: + target.__readTree + ++ lib.optionals (target ? __subtarget) [ target.__subtarget ]; + + # Given a derivation (identified by drvPath) that is part of the list of + # targets passed to mkPipeline, determine all derivations that it depends on + # and are also part of the pipeline. Finally, return the keys of the steps + # that build them. This is used to populate `depends_on` in `mkStep`. + # + # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`. + getTargetPipelineDeps = targetDepMap: drvPath: + # Sanity check: We should only call this function on targets explicitly + # passed to mkPipeline. Thus it should have been passed as a “known” drv to + # dependency-analyzer. + assert targetDepMap.${drvPath}.known; + builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps; + # Create a pipeline step from a single target. - mkStep = headBranch: parentTargetMap: target: + mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }: let label = mkLabel target; drvPath = unsafeDiscardStringContext target.drvPath; - shouldSkip' = shouldSkip parentTargetMap; in { label = ":nix: " + label; - key = hashString "sha1" label; - skip = shouldSkip' label drvPath; - command = mkBuildCommand target drvPath; + key = keyForDrv target; + skip = shouldSkip { inherit label drvPath parentTargetMap; }; + command = mkBuildCommand { + attrPath = targetAttrPath target; + inherit drvPath; + }; env.READTREE_TARGET = label; + cancel_on_build_failing = cancelOnBuildFailing; # Add a dependency on the initial static pipeline step which # always runs. This allows build steps uploaded in batches to # start running before all batches have been uploaded. - depends_on = ":init:"; - }; + depends_on = [ ":init:" ] + ++ getTargetPipelineDeps targetDepMap drvPath + ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + } // lib.optionalAttrs (target ? meta.timeout) { + timeout_in_minutes = target.meta.timeout / 60; + # Additional arguments to set on the step. + # Keep in mind these *overwrite* existing step args, not extend. Use with caution. + } // lib.optionalAttrs (target ? meta.ci.buildkiteExtraStepArgs) target.meta.ci.buildkiteExtraStepArgs; # Helper function to inelegantly divide a list into chunks of at # most n elements. @@ -142,7 +191,21 @@ rec { # # Can be used for status reporting steps and the like. postBuildSteps ? [ ] - , # Build phases that are active for this invocation (i.e. their + # The list of phases known by the current Buildkite + # pipeline. Dynamic pipeline chunks for each phase are uploaded + # to Buildkite on execution of static part of the + # pipeline. Phases selection is hard-coded in the static + # pipeline. + # + # Pipeline generation will fail when an extra step with + # unregistered phase is added. + # + # Common scenarios for different phase: + # - "build" - main phase for building all Nix targets + # - "release" - pushing artifacts to external repositories + # - "deploy" - updating external deployment configurations + , phases ? [ "build" "release" ] + # Build phases that are active for this invocation (i.e. their # steps should be generated). # # This can be used to disable outputting parts of a pipeline if, @@ -150,45 +213,50 @@ rec { # eval contexts. # # TODO(tazjin): Fail/warn if unknown phase is requested. - activePhases ? [ "build" "release" ] + , activePhases ? phases + # Setting this attribute to true cancels dynamic pipeline steps + # as soon as the build is marked as failing. + # + # To enable this feature one should enable "Fail Fast" setting + # at Buildkite pipeline or on organization level. + , cancelOnBuildFailing ? false }: let - # Currently the only known phases are 'build' (Nix builds and - # extra steps that are not post-build steps) and 'release' (all - # post-build steps). - # - # TODO(tazjin): Fully configurable set of phases? - knownPhases = [ "build" "release" ]; - # List of phases to include. - phases = lib.intersectLists activePhases knownPhases; + enabledPhases = lib.intersectLists activePhases phases; # Is the 'build' phase included? This phase is treated specially # because it always contains the plain Nix builds, and some # logic/optimisation depends on knowing whether is executing. - buildEnabled = elem "build" phases; + buildEnabled = elem "build" enabledPhases; + + # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer. + targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets); # Convert a target into all of its steps, separated by build # phase (as phases end up in different chunks). targetToSteps = target: let - step = mkStep headBranch parentTargetMap target; + mkStepArgs = { + inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing; + }; + step = mkStep mkStepArgs; # Same step, but with an override function applied. This is # used in mkExtraStep if the extra step needs to modify the # parent derivation somehow. # # Note that this will never affect the label. - overridable = f: mkStep headBranch parentTargetMap (f target); + overridable = f: mkStep (mkStepArgs // { target = (f target); }); # Split extra steps by phase. splitExtraSteps = lib.groupBy ({ phase, ... }: phase) - (attrValues (mapAttrs (normaliseExtraStep knownPhases overridable) + (attrValues (mapAttrs (normaliseExtraStep phases overridable) (target.meta.ci.extraSteps or { }))); extraSteps = mapAttrs (_: steps: - map (mkExtraStep buildEnabled) steps) + map (mkExtraStep (targetAttrPath target) buildEnabled) steps) splitExtraSteps; in if !buildEnabled then extraSteps @@ -204,7 +272,7 @@ rec { release = postBuildSteps; }; - phasesWithSteps = lib.zipAttrsWithNames phases (_: concatLists) + phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists) ((map targetToSteps drvTargets) ++ [ globalSteps ]); # Generate pipeline chunks for each phase. @@ -215,10 +283,10 @@ rec { then acc else acc ++ (pipelineChunks phase phaseSteps)) [ ] - phases; + enabledPhases; in - runCommandNoCC "buildkite-pipeline" { } '' + runCommand "buildkite-pipeline" { } '' mkdir $out echo "Generated ${toString (length chunks)} pipeline chunks" ${ @@ -238,9 +306,7 @@ rec { # Include the attrPath in the output to reconstruct the drv # without parsing the human-readable label. - attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ - target.__subtarget - ]; + attrPath = targetAttrPath target; }; }) drvTargets))); @@ -264,10 +330,6 @@ rec { # confirmation. These steps always run after the main build is # done and have no influence on CI status. # - # postBuild (optional): If set to true, this step will run after - # all primary build steps (that is, after status has been reported - # back to CI). - # # needsOutput (optional): If set to true, the parent derivation # will be built in the working directory before running the # command. Output will be available as 'result'. @@ -294,8 +356,8 @@ rec { steps = [ { - inherit (step) branches; inherit prompt; + branches = step.branches or [ ]; block = ":radio_button: Run ${label}? (from ${parent.env.READTREE_TARGET})"; } @@ -309,7 +371,7 @@ rec { # Validate and normalise extra step configuration before actually # generating build steps, in order to use user-provided metadata # during the pipeline generation. - normaliseExtraStep = knownPhases: overridableParent: key: + normaliseExtraStep = phases: overridableParent: key: { command , label ? key , needsOutput ? false @@ -317,12 +379,8 @@ rec { , branches ? null , alwaysRun ? false , prompt ? false - - # TODO(tazjin): Default to 'build' after 2022-10-01. - , phase ? if (isNull postBuild || !postBuild) then "build" else "release" - - # TODO(tazjin): Turn into hard-failure after 2022-10-01. - , postBuild ? null + , softFail ? false + , phase ? "build" , skip ? false , agents ? null }: @@ -330,12 +388,12 @@ rec { parent = overridableParent parentOverride; parentLabel = parent.env.READTREE_TARGET; - validPhase = lib.throwIfNot (elem phase knownPhases) '' + validPhase = lib.throwIfNot (elem phase phases) '' In step '${label}' (from ${parentLabel}): Phase '${phase}' is not valid. - Known phases: ${concatStringsSep ", " knownPhases} + Known phases: ${concatStringsSep ", " phases} '' phase; in @@ -349,35 +407,16 @@ rec { needsOutput parent parentLabel + softFail skip agents; - # //nix/buildkite is growing a new feature for adding different - # "build phases" which supersedes the previous `postBuild` - # boolean API. - # - # To help users transition, emit warnings if the old API is used. - phase = lib.warnIfNot (isNull postBuild) '' - In step '${label}' (from ${parentLabel}): - - Please note: The CI system is introducing support for running - steps in different build phases. - - The currently supported phases are 'build' (all Nix targets, - extra steps such as tests that feed into the build results, - etc.) and 'release' (steps that run after builds and tests - have already completed). - - This replaces the previous boolean `postBuild` API in extra - step definitions. Please remove the `postBuild` parameter from - this step and instead set `phase = ${phase};`. - '' - validPhase; + phase = validPhase; prompt = lib.throwIf (prompt != false && phase == "build") '' In step '${label}' (from ${parentLabel}): - The 'prompt' feature can only be used by steps in the "release" + The 'prompt' feature can not be used by steps in the "build" phase, because CI builds should not be gated on manual human approvals. '' @@ -386,11 +425,26 @@ rec { # Create the Buildkite configuration for an extra step, optionally # wrapping it in a gate group. - mkExtraStep = buildEnabled: cfg: + mkExtraStep = parentAttrPath: buildEnabled: cfg: let + # ATTN: needs to match an entry in .gitignore so that the tree won't get dirty + commandScriptLink = "nix-buildkite-extra-step-command-script"; + step = { + key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; - skip = if cfg.alwaysRun then false else cfg.skip or cfg.parent.skip or false; + skip = + let + # When parent doesn't have skip attribute set, default to false + parentSkip = cfg.parent.skip or false; + # Extra step skip parameter can be string explaining the + # skip reason. + extraStepSkip = if builtins.isString cfg.skip then true else cfg.skip; + # Don't run if extra step is explicitly set to skip. If + # parameter is not set or equal to false, follow parent behavior. + skip' = if extraStepSkip then cfg.skip else parentSkip; + in + if cfg.alwaysRun then false else skip'; depends_on = lib.optional (buildEnabled && !cfg.alwaysRun && !cfg.needsOutput) @@ -402,9 +456,25 @@ rec { "echo '~~~ Preparing build output of ${cfg.parentLabel}'" } ${lib.optionalString cfg.needsOutput cfg.parent.command} - echo '+++ Running extra step command' - exec ${cfg.command} + echo '--- Building extra step script' + command_script="$(${ + # Using command substitution in this way assumes the script drv only has one output + assert builtins.length cfg.command.outputs == 1; + mkBuildCommand { + # script is exposed at <parent>.meta.ci.extraSteps.<key>.command + attrPath = + parentAttrPath + ++ [ "meta" "ci" "extraSteps" cfg.key "command" ]; + drvPath = unsafeDiscardStringContext cfg.command.drvPath; + # make sure it doesn't conflict with result (from needsOutput) + outLink = commandScriptLink; + } + })" + echo '+++ Running extra step script' + exec "$command_script" ''; + + soft_fail = cfg.softFail; } // (lib.optionalAttrs (cfg.agents != null) { inherit (cfg) agents; }) // (lib.optionalAttrs (cfg.branches != null) { branches = lib.concatStringsSep " " cfg.branches; |