diff options
Diffstat (limited to 'nix/buildkite/default.nix')
-rw-r--r-- | nix/buildkite/default.nix | 360 |
1 files changed, 258 insertions, 102 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 6a0e9d246d..9abba9408a 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -11,11 +11,10 @@ let inherit (builtins) attrValues - concatMap + concatLists concatStringsSep - filter + elem foldl' - getEnv hasAttr hashString isNull @@ -23,70 +22,117 @@ let length listToAttrs mapAttrs - partition - pathExists toJSON unsafeDiscardStringContext; - inherit (pkgs) lib runCommandNoCC writeText; + inherit (pkgs) lib runCommand writeText; inherit (depot.nix.readTree) mkLabel; + + inherit (depot.nix) dependency-analyzer; in rec { - # Creates a Nix expression that yields the target at the specified - # location in the repository. - # - # This makes a distinction between normal targets (which physically - # exist in the repository) and subtargets (which are "virtual" - # targets exposed by a physical one) to make it clear in the build - # output which is which. - mkBuildExpr = target: + # Create a unique key for the buildkite pipeline based on the given derivation + # or drvPath. A consequence of using such keys is that every derivation may + # only be exposed as a single, unique step in the pipeline. + keyForDrv = drvOrPath: + let + drvPath = + if lib.isDerivation drvOrPath then drvOrPath.drvPath + else if lib.isString drvOrPath then drvOrPath + else builtins.throw "keyForDrv: expected string or derivation"; + + # Only use the drv hash to prevent escaping problems. Buildkite also has a + # limit of 100 characters on keys. + in + "drv-" + (builtins.substring 0 32 + (builtins.baseNameOf (unsafeDiscardStringContext drvPath)) + ); + + # Given an arbitrary attribute path generate a Nix expression which obtains + # this from the root of depot (assumed to be ./.). Attributes may be any + # Nix strings suitable as attribute names, not just Nix literal-safe strings. + mkBuildExpr = attrPath: let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; - targetExpr = foldl' descend "import ./. {}" target.__readTree; - subtargetExpr = descend targetExpr target.__subtarget; in - if target ? __subtarget then subtargetExpr else targetExpr; + foldl' descend "import ./. {}" attrPath; # Determine whether to skip a target if it has not diverged from the # HEAD branch. - shouldSkip = parentTargetMap: label: drvPath: + shouldSkip = { parentTargetMap ? { }, label, drvPath }: if (hasAttr label parentTargetMap) && parentTargetMap."${label}".drvPath == drvPath then "Target has not changed." else false; - # Create build command for a derivation target. - mkBuildCommand = target: drvPath: concatStringsSep " " [ + # Create build command for an attribute path pointing to a derivation. + mkBuildCommand = { attrPath, drvPath, outLink ? "result" }: concatStringsSep " " [ + # If the nix build fails, the Nix command's exit status should be used. + "set -o pipefail;" + # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on # at least one of its `outPath`s, so we need to discard the string context # if we don't want to build everything during pipeline construction. - "(nix-store --realise '${drvPath}' --add-root result --indirect && readlink result)" + # + # To make this more uniform with how nix-build(1) works, we call realpath(1) + # on nix-store(1)'s output since it has the habit of printing the path of the + # out link, not the store path. + "(nix-store --realise '${drvPath}' --add-root '${outLink}' --indirect | xargs -r realpath)" # Since we don't gcroot the derivation files, they may be deleted by the # garbage collector. In that case we can reevaluate and build the attribute # using nix-build. - "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)" + "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr attrPath}' --show-trace --out-link '${outLink}')" ]; + # Attribute path of a target relative to the depot root. Needs to take into + # account whether the target is a physical target (which corresponds to a path + # in the filesystem) or the subtarget of a physical target. + targetAttrPath = target: + target.__readTree + ++ lib.optionals (target ? __subtarget) [ target.__subtarget ]; + + # Given a derivation (identified by drvPath) that is part of the list of + # targets passed to mkPipeline, determine all derivations that it depends on + # and are also part of the pipeline. Finally, return the keys of the steps + # that build them. This is used to populate `depends_on` in `mkStep`. + # + # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`. + getTargetPipelineDeps = targetDepMap: drvPath: + # Sanity check: We should only call this function on targets explicitly + # passed to mkPipeline. Thus it should have been passed as a “known” drv to + # dependency-analyzer. + assert targetDepMap.${drvPath}.known; + builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps; + # Create a pipeline step from a single target. - mkStep = headBranch: parentTargetMap: target: + mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }: let label = mkLabel target; drvPath = unsafeDiscardStringContext target.drvPath; - shouldSkip' = shouldSkip parentTargetMap; in { label = ":nix: " + label; - key = hashString "sha1" label; - skip = shouldSkip' label drvPath; - command = mkBuildCommand target drvPath; + key = keyForDrv target; + skip = shouldSkip { inherit label drvPath parentTargetMap; }; + command = mkBuildCommand { + attrPath = targetAttrPath target; + inherit drvPath; + }; env.READTREE_TARGET = label; + cancel_on_build_failing = cancelOnBuildFailing; # Add a dependency on the initial static pipeline step which # always runs. This allows build steps uploaded in batches to # start running before all batches have been uploaded. - depends_on = ":init:"; - }; + depends_on = [ ":init:" ] + ++ getTargetPipelineDeps targetDepMap drvPath + ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + } // lib.optionalAttrs (target ? meta.timeout) { + timeout_in_minutes = target.meta.timeout / 60; + # Additional arguments to set on the step. + # Keep in mind these *overwrite* existing step args, not extend. Use with caution. + } // lib.optionalAttrs (target ? meta.ci.buildkiteExtraStepArgs) target.meta.ci.buildkiteExtraStepArgs; # Helper function to inelegantly divide a list into chunks of at # most n elements. @@ -111,7 +157,7 @@ rec { }); }; - # Split the pipeline into chunks of at most 256 steps at once, which + # Split the pipeline into chunks of at most 192 steps at once, which # are uploaded sequentially. This is because of a limitation in the # Buildkite backend which struggles to process more than a specific # number of chunks at once. @@ -145,68 +191,102 @@ rec { # # Can be used for status reporting steps and the like. postBuildSteps ? [ ] + # The list of phases known by the current Buildkite + # pipeline. Dynamic pipeline chunks for each phase are uploaded + # to Buildkite on execution of static part of the + # pipeline. Phases selection is hard-coded in the static + # pipeline. + # + # Pipeline generation will fail when an extra step with + # unregistered phase is added. + # + # Common scenarios for different phase: + # - "build" - main phase for building all Nix targets + # - "release" - pushing artifacts to external repositories + # - "deploy" - updating external deployment configurations + , phases ? [ "build" "release" ] + # Build phases that are active for this invocation (i.e. their + # steps should be generated). + # + # This can be used to disable outputting parts of a pipeline if, + # for example, build and release phases are created in separate + # eval contexts. + # + # TODO(tazjin): Fail/warn if unknown phase is requested. + , activePhases ? phases + # Setting this attribute to true cancels dynamic pipeline steps + # as soon as the build is marked as failing. + # + # To enable this feature one should enable "Fail Fast" setting + # at Buildkite pipeline or on organization level. + , cancelOnBuildFailing ? false }: let - # Convert a target into all of its build and post-build steps, - # treated separately as they need to be in different chunks. + # List of phases to include. + enabledPhases = lib.intersectLists activePhases phases; + + # Is the 'build' phase included? This phase is treated specially + # because it always contains the plain Nix builds, and some + # logic/optimisation depends on knowing whether is executing. + buildEnabled = elem "build" enabledPhases; + + # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer. + targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets); + + # Convert a target into all of its steps, separated by build + # phase (as phases end up in different chunks). targetToSteps = target: let - step = mkStep headBranch parentTargetMap target; + mkStepArgs = { + inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing; + }; + step = mkStep mkStepArgs; # Same step, but with an override function applied. This is # used in mkExtraStep if the extra step needs to modify the # parent derivation somehow. # # Note that this will never affect the label. - overridable = f: mkStep headBranch parentTargetMap (f target); - - # Split build/post-build steps - splitExtraSteps = partition ({ postStep, ... }: postStep) - (attrValues (mapAttrs - (name: value: { - inherit name value; - postStep = (value ? prompt) || (value.postBuild or false); - }) + overridable = f: mkStep (mkStepArgs // { target = (f target); }); + + # Split extra steps by phase. + splitExtraSteps = lib.groupBy ({ phase, ... }: phase) + (attrValues (mapAttrs (normaliseExtraStep phases overridable) (target.meta.ci.extraSteps or { }))); - mkExtraStep' = { name, value, ... }: mkExtraStep overridable name value; - extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt - extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt + extraSteps = mapAttrs + (_: steps: + map (mkExtraStep (targetAttrPath target) buildEnabled) steps) + splitExtraSteps; in - { - buildSteps = [ step ] ++ extraBuildSteps; - postSteps = extraPostSteps; + if !buildEnabled then extraSteps + else extraSteps // { + build = [ step ] ++ (extraSteps.build or [ ]); }; - # Combine all target steps into separate build and post-build step lists. - steps = foldl' - (acc: t: { - buildSteps = acc.buildSteps ++ t.buildSteps; - postSteps = acc.postSteps ++ t.postSteps; - }) - { buildSteps = [ ]; postSteps = [ ]; } - (map targetToSteps drvTargets); - - buildSteps = - # Add build steps for each derivation target and their extra - # steps. - steps.buildSteps - - # Add additional steps (if set). - ++ additionalSteps; - - postSteps = - # Add post-build steps for each derivation target. - steps.postSteps - - # Add any globally defined post-build steps. - ++ postBuildSteps; - - buildChunks = pipelineChunks "build" buildSteps; - postBuildChunks = pipelineChunks "post" postSteps; - chunks = buildChunks ++ postBuildChunks; + # Combine all target steps into step lists per phase. + # + # TODO(tazjin): Refactor when configurable phases show up. + globalSteps = { + build = additionalSteps; + release = postBuildSteps; + }; + + phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists) + ((map targetToSteps drvTargets) ++ [ globalSteps ]); + + # Generate pipeline chunks for each phase. + chunks = foldl' + (acc: phase: + let phaseSteps = phasesWithSteps.${phase} or [ ]; in + if phaseSteps == [ ] + then acc + else acc ++ (pipelineChunks phase phaseSteps)) + [ ] + enabledPhases; + in - runCommandNoCC "buildkite-pipeline" { } '' + runCommand "buildkite-pipeline" { } '' mkdir $out echo "Generated ${toString (length chunks)} pipeline chunks" ${ @@ -226,9 +306,7 @@ rec { # Include the attrPath in the output to reconstruct the drv # without parsing the human-readable label. - attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ - target.__subtarget - ]; + attrPath = targetAttrPath target; }; }) drvTargets))); @@ -252,10 +330,6 @@ rec { # confirmation. These steps always run after the main build is # done and have no influence on CI status. # - # postBuild (optional): If set to true, this step will run after - # all primary build steps (that is, after status has been reported - # back to CI). - # # needsOutput (optional): If set to true, the parent derivation # will be built in the working directory before running the # command. Output will be available as 'result'. @@ -282,8 +356,8 @@ rec { steps = [ { - inherit (step) branches; inherit prompt; + branches = step.branches or [ ]; block = ":radio_button: Run ${label}? (from ${parent.env.READTREE_TARGET})"; } @@ -294,42 +368,124 @@ rec { ]; }; - # Create the Buildkite configuration for an extra step, optionally - # wrapping it in a gate group. - mkExtraStep = overridableParent: key: + # Validate and normalise extra step configuration before actually + # generating build steps, in order to use user-provided metadata + # during the pipeline generation. + normaliseExtraStep = phases: overridableParent: key: { command , label ? key - , prompt ? false , needsOutput ? false , parentOverride ? (x: x) , branches ? null , alwaysRun ? false - , postBuild ? false - }@cfg: + , prompt ? false + , softFail ? false + , phase ? "build" + , skip ? false + , agents ? null + }: let parent = overridableParent parentOverride; parentLabel = parent.env.READTREE_TARGET; + validPhase = lib.throwIfNot (elem phase phases) '' + In step '${label}' (from ${parentLabel}): + + Phase '${phase}' is not valid. + + Known phases: ${concatStringsSep ", " phases} + '' + phase; + in + { + inherit + alwaysRun + branches + command + key + label + needsOutput + parent + parentLabel + softFail + skip + agents; + + phase = validPhase; + + prompt = lib.throwIf (prompt != false && phase == "build") '' + In step '${label}' (from ${parentLabel}): + + The 'prompt' feature can not be used by steps in the "build" + phase, because CI builds should not be gated on manual human + approvals. + '' + prompt; + }; + + # Create the Buildkite configuration for an extra step, optionally + # wrapping it in a gate group. + mkExtraStep = parentAttrPath: buildEnabled: cfg: + let + # ATTN: needs to match an entry in .gitignore so that the tree won't get dirty + commandScriptLink = "nix-buildkite-extra-step-command-script"; + step = { - label = ":gear: ${label} (from ${parentLabel})"; - skip = if alwaysRun then false else parent.skip or false; - depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key; - branches = if branches != null then lib.concatStringsSep " " branches else null; + key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; + label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; + skip = + let + # When parent doesn't have skip attribute set, default to false + parentSkip = cfg.parent.skip or false; + # Extra step skip parameter can be string explaining the + # skip reason. + extraStepSkip = if builtins.isString cfg.skip then true else cfg.skip; + # Don't run if extra step is explicitly set to skip. If + # parameter is not set or equal to false, follow parent behavior. + skip' = if extraStepSkip then cfg.skip else parentSkip; + in + if cfg.alwaysRun then false else skip'; - command = pkgs.writeShellScript "${key}-script" '' + depends_on = lib.optional + (buildEnabled && !cfg.alwaysRun && !cfg.needsOutput) + cfg.parent.key; + + command = pkgs.writeShellScript "${cfg.key}-script" '' set -ueo pipefail - ${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"} - ${lib.optionalString needsOutput parent.command} - echo '+++ Running extra step command' - exec ${command} + ${lib.optionalString cfg.needsOutput + "echo '~~~ Preparing build output of ${cfg.parentLabel}'" + } + ${lib.optionalString cfg.needsOutput cfg.parent.command} + echo '--- Building extra step script' + command_script="$(${ + # Using command substitution in this way assumes the script drv only has one output + assert builtins.length cfg.command.outputs == 1; + mkBuildCommand { + # script is exposed at <parent>.meta.ci.extraSteps.<key>.command + attrPath = + parentAttrPath + ++ [ "meta" "ci" "extraSteps" cfg.key "command" ]; + drvPath = unsafeDiscardStringContext cfg.command.drvPath; + # make sure it doesn't conflict with result (from needsOutput) + outLink = commandScriptLink; + } + })" + echo '+++ Running extra step script' + exec "$command_script" ''; - }; + + soft_fail = cfg.softFail; + } // (lib.optionalAttrs (cfg.agents != null) { inherit (cfg) agents; }) + // (lib.optionalAttrs (cfg.branches != null) { + branches = lib.concatStringsSep " " cfg.branches; + }); in - if (isString prompt) + if (isString cfg.prompt) then mkGatedStep { - inherit step label parent prompt; + inherit step; + inherit (cfg) label parent prompt; } else step; } |