diff options
Diffstat (limited to 'nix/buildkite')
| -rw-r--r-- | nix/buildkite/default.nix | 426 | ||||
| -rwxr-xr-x | nix/buildkite/fetch-parent-targets.sh | 44 |
2 files changed, 470 insertions, 0 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix new file mode 100644 index 000000000000..c53d58eecdae --- /dev/null +++ b/nix/buildkite/default.nix @@ -0,0 +1,426 @@ +# Logic for generating Buildkite pipelines from Nix build targets read +# by //nix/readTree. +# +# It outputs a "YAML" (actually JSON) file which is evaluated and +# submitted to Buildkite at the start of each build. +# +# The structure of the file that is being created is documented here: +# https://buildkite.com/docs/pipelines/defining-steps +{ depot, pkgs, ... }: + +let + inherit (builtins) + attrValues + concatLists + concatStringsSep + elem + foldl' + hasAttr + hashString + isNull + isString + length + listToAttrs + mapAttrs + toJSON + unsafeDiscardStringContext; + + inherit (pkgs) lib runCommand writeText; + inherit (depot.nix.readTree) mkLabel; +in +rec { + # Creates a Nix expression that yields the target at the specified + # location in the repository. + # + # This makes a distinction between normal targets (which physically + # exist in the repository) and subtargets (which are "virtual" + # targets exposed by a physical one) to make it clear in the build + # output which is which. + mkBuildExpr = target: + let + descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; + targetExpr = foldl' descend "import ./. {}" target.__readTree; + subtargetExpr = descend targetExpr target.__subtarget; + in + if target ? __subtarget then subtargetExpr else targetExpr; + + # Determine whether to skip a target if it has not diverged from the + # HEAD branch. + shouldSkip = parentTargetMap: label: drvPath: + if (hasAttr label parentTargetMap) && parentTargetMap."${label}".drvPath == drvPath + then "Target has not changed." + else false; + + # Create build command for a derivation target. + mkBuildCommand = target: drvPath: concatStringsSep " " [ + # First try to realise the drvPath of the target so we don't evaluate twice. + # Nix has no concept of depending on a derivation file without depending on + # at least one of its `outPath`s, so we need to discard the string context + # if we don't want to build everything during pipeline construction. + "(nix-store --realise '${drvPath}' --add-root result --indirect && readlink result)" + + # Since we don't gcroot the derivation files, they may be deleted by the + # garbage collector. In that case we can reevaluate and build the attribute + # using nix-build. + "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)" + ]; + + # Create a pipeline step from a single target. + mkStep = headBranch: parentTargetMap: target: cancelOnBuildFailing: + let + label = mkLabel target; + drvPath = unsafeDiscardStringContext target.drvPath; + shouldSkip' = shouldSkip parentTargetMap; + in + { + label = ":nix: " + label; + key = hashString "sha1" label; + skip = shouldSkip' label drvPath; + command = mkBuildCommand target drvPath; + env.READTREE_TARGET = label; + cancel_on_build_failing = cancelOnBuildFailing; + + # Add a dependency on the initial static pipeline step which + # always runs. This allows build steps uploaded in batches to + # start running before all batches have been uploaded. + depends_on = [ ":init:" ] ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps; + } // lib.optionalAttrs (target ? meta.timeout) { + timeout_in_minutes = target.meta.timeout / 60; + # Additional arguments to set on the step. + # Keep in mind these *overwrite* existing step args, not extend. Use with caution. + } // lib.optionalAttrs (target ? meta.ci.buildkiteExtraStepArgs) target.meta.ci.buildkiteExtraStepArgs; + + # Helper function to inelegantly divide a list into chunks of at + # most n elements. + # + # This works by assigning each element a chunk ID based on its + # index, and then grouping all elements by their chunk ID. + chunksOf = n: list: + let + chunkId = idx: toString (idx / n + 1); + assigned = lib.imap1 (idx: value: { inherit value; chunk = chunkId idx; }) list; + unchunk = mapAttrs (_: elements: map (e: e.value) elements); + in + unchunk (lib.groupBy (e: e.chunk) assigned); + + # Define a build pipeline chunk as a JSON file, using the pipeline + # format documented on + # https://buildkite.com/docs/pipelines/defining-steps. + makePipelineChunk = name: chunkId: chunk: rec { + filename = "${name}-chunk-${chunkId}.json"; + path = writeText filename (toJSON { + steps = chunk; + }); + }; + + # Split the pipeline into chunks of at most 192 steps at once, which + # are uploaded sequentially. This is because of a limitation in the + # Buildkite backend which struggles to process more than a specific + # number of chunks at once. + pipelineChunks = name: steps: + attrValues (mapAttrs (makePipelineChunk name) (chunksOf 192 steps)); + + # Create a pipeline structure for the given targets. + mkPipeline = + { + # HEAD branch of the repository on which release steps, GC + # anchoring and other "mainline only" steps should run. + headBranch + , # List of derivations as read by readTree (in most cases just the + # output of readTree.gather) that should be built in Buildkite. + # + # These are scheduled as the first build steps and run as fast as + # possible, in order, without any concurrency restrictions. + drvTargets + , # Derivation map of a parent commit. Only targets which no longer + # correspond to the content of this map will be built. Passing an + # empty map will always build all targets. + parentTargetMap ? { } + , # A list of plain Buildkite step structures to run alongside the + # build for all drvTargets, but before proceeding with any + # post-build actions such as status reporting. + # + # Can be used for things like code formatting checks. + additionalSteps ? [ ] + , # A list of plain Buildkite step structures to run after all + # previous steps succeeded. + # + # Can be used for status reporting steps and the like. + postBuildSteps ? [ ] + # The list of phases known by the current Buildkite + # pipeline. Dynamic pipeline chunks for each phase are uploaded + # to Buildkite on execution of static part of the + # pipeline. Phases selection is hard-coded in the static + # pipeline. + # + # Pipeline generation will fail when an extra step with + # unregistered phase is added. + # + # Common scenarios for different phase: + # - "build" - main phase for building all Nix targets + # - "release" - pushing artifacts to external repositories + # - "deploy" - updating external deployment configurations + , phases ? [ "build" "release" ] + # Build phases that are active for this invocation (i.e. their + # steps should be generated). + # + # This can be used to disable outputting parts of a pipeline if, + # for example, build and release phases are created in separate + # eval contexts. + # + # TODO(tazjin): Fail/warn if unknown phase is requested. + , activePhases ? phases + # Setting this attribute to true cancels dynamic pipeline steps + # as soon as the build is marked as failing. + # + # To enable this feature one should enable "Fail Fast" setting + # at Buildkite pipeline or on organization level. + , cancelOnBuildFailing ? false + }: + let + # List of phases to include. + enabledPhases = lib.intersectLists activePhases phases; + + # Is the 'build' phase included? This phase is treated specially + # because it always contains the plain Nix builds, and some + # logic/optimisation depends on knowing whether is executing. + buildEnabled = elem "build" enabledPhases; + + # Convert a target into all of its steps, separated by build + # phase (as phases end up in different chunks). + targetToSteps = target: + let + step = mkStep headBranch parentTargetMap target cancelOnBuildFailing; + + # Same step, but with an override function applied. This is + # used in mkExtraStep if the extra step needs to modify the + # parent derivation somehow. + # + # Note that this will never affect the label. + overridable = f: mkStep headBranch parentTargetMap (f target) cancelOnBuildFailing; + + # Split extra steps by phase. + splitExtraSteps = lib.groupBy ({ phase, ... }: phase) + (attrValues (mapAttrs (normaliseExtraStep phases overridable) + (target.meta.ci.extraSteps or { }))); + + extraSteps = mapAttrs + (_: steps: + map (mkExtraStep buildEnabled) steps) + splitExtraSteps; + in + if !buildEnabled then extraSteps + else extraSteps // { + build = [ step ] ++ (extraSteps.build or [ ]); + }; + + # Combine all target steps into step lists per phase. + # + # TODO(tazjin): Refactor when configurable phases show up. + globalSteps = { + build = additionalSteps; + release = postBuildSteps; + }; + + phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists) + ((map targetToSteps drvTargets) ++ [ globalSteps ]); + + # Generate pipeline chunks for each phase. + chunks = foldl' + (acc: phase: + let phaseSteps = phasesWithSteps.${phase} or [ ]; in + if phaseSteps == [ ] + then acc + else acc ++ (pipelineChunks phase phaseSteps)) + [ ] + enabledPhases; + + in + runCommand "buildkite-pipeline" { } '' + mkdir $out + echo "Generated ${toString (length chunks)} pipeline chunks" + ${ + lib.concatMapStringsSep "\n" + (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks + } + ''; + + # Create a drvmap structure for the given targets, containing the + # mapping of all target paths to their derivations. The mapping can + # be persisted for future use. + mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map + (target: { + name = mkLabel target; + value = { + drvPath = unsafeDiscardStringContext target.drvPath; + + # Include the attrPath in the output to reconstruct the drv + # without parsing the human-readable label. + attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ + target.__subtarget + ]; + }; + }) + drvTargets))); + + # Implementation of extra step logic. + # + # Each target extra step is an attribute specified in + # `meta.ci.extraSteps`. Its attribute name will be used as the step + # name on Buildkite. + # + # command (required): A command that will be run in the depot + # checkout when this step is executed. Should be a derivation + # resulting in a single executable file, e.g. through + # pkgs.writeShellScript. + # + # label (optional): Human-readable label for this step to display + # in the Buildkite UI instead of the attribute name. + # + # prompt (optional): Setting this blocks the step until confirmed + # by a human. Should be a string which is displayed for + # confirmation. These steps always run after the main build is + # done and have no influence on CI status. + # + # needsOutput (optional): If set to true, the parent derivation + # will be built in the working directory before running the + # command. Output will be available as 'result'. + # TODO: Figure out multiple-output derivations. + # + # parentOverride (optional): A function (drv -> drv) to override + # the parent's target definition when preparing its output. Only + # used in extra steps that use needsOutput. + # + # branches (optional): Git references (branches, tags ... ) on + # which this step should be allowed to run. List of strings. + # + # alwaysRun (optional): If set to true, this step will always run, + # even if its parent has not been rebuilt. + # + # Note that gated steps are independent of each other. + + # Create a gated step in a step group, independent from any other + # steps. + mkGatedStep = { step, label, parent, prompt }: { + inherit (step) depends_on; + group = label; + skip = parent.skip or false; + + steps = [ + { + inherit prompt; + branches = step.branches or [ ]; + block = ":radio_button: Run ${label}? (from ${parent.env.READTREE_TARGET})"; + } + + # The explicit depends_on of the wrapped step must be removed, + # otherwise its dependency relationship with the gate step will + # break. + (builtins.removeAttrs step [ "depends_on" ]) + ]; + }; + + # Validate and normalise extra step configuration before actually + # generating build steps, in order to use user-provided metadata + # during the pipeline generation. + normaliseExtraStep = phases: overridableParent: key: + { command + , label ? key + , needsOutput ? false + , parentOverride ? (x: x) + , branches ? null + , alwaysRun ? false + , prompt ? false + , softFail ? false + , phase ? "build" + , skip ? false + , agents ? null + }: + let + parent = overridableParent parentOverride; + parentLabel = parent.env.READTREE_TARGET; + + validPhase = lib.throwIfNot (elem phase phases) '' + In step '${label}' (from ${parentLabel}): + + Phase '${phase}' is not valid. + + Known phases: ${concatStringsSep ", " phases} + '' + phase; + in + { + inherit + alwaysRun + branches + command + key + label + needsOutput + parent + parentLabel + softFail + skip + agents; + + phase = validPhase; + + prompt = lib.throwIf (prompt != false && phase == "build") '' + In step '${label}' (from ${parentLabel}): + + The 'prompt' feature can only be used by steps in the "release" + phase, because CI builds should not be gated on manual human + approvals. + '' + prompt; + }; + + # Create the Buildkite configuration for an extra step, optionally + # wrapping it in a gate group. + mkExtraStep = buildEnabled: cfg: + let + step = { + key = hashString "sha1" "${cfg.label}-${cfg.parentLabel}"; + label = ":gear: ${cfg.label} (from ${cfg.parentLabel})"; + skip = + let + # When parent doesn't have skip attribute set, default to false + parentSkip = cfg.parent.skip or false; + # Extra step skip parameter can be string explaining the + # skip reason. + extraStepSkip = if builtins.isString cfg.skip then true else cfg.skip; + # Don't run if extra step is explicitly set to skip. If + # parameter is not set or equal to false, follow parent behavior. + skip' = if extraStepSkip then cfg.skip else parentSkip; + in + if cfg.alwaysRun then false else skip'; + + depends_on = lib.optional + (buildEnabled && !cfg.alwaysRun && !cfg.needsOutput) + cfg.parent.key; + + command = pkgs.writeShellScript "${cfg.key}-script" '' + set -ueo pipefail + ${lib.optionalString cfg.needsOutput + "echo '~~~ Preparing build output of ${cfg.parentLabel}'" + } + ${lib.optionalString cfg.needsOutput cfg.parent.command} + echo '+++ Running extra step command' + exec ${cfg.command} + ''; + + soft_fail = cfg.softFail; + } // (lib.optionalAttrs (cfg.agents != null) { inherit (cfg) agents; }) + // (lib.optionalAttrs (cfg.branches != null) { + branches = lib.concatStringsSep " " cfg.branches; + }); + in + if (isString cfg.prompt) + then + mkGatedStep + { + inherit step; + inherit (cfg) label parent prompt; + } + else step; +} diff --git a/nix/buildkite/fetch-parent-targets.sh b/nix/buildkite/fetch-parent-targets.sh new file mode 100755 index 000000000000..8afac1e5ec04 --- /dev/null +++ b/nix/buildkite/fetch-parent-targets.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -ueo pipefail + +# Each Buildkite build stores the derivation target map as a pipeline +# artifact. This script determines the most appropriate commit (the +# fork point of the current chain from HEAD) and fetches the artifact. +# +# New builds can be based on HEAD before the pipeline for the last +# commit has finished, in which case it is possible that the fork +# point has no derivation map. To account for this, up to 3 commits +# prior to HEAD are also queried to find a map. +# +# If no map is found, the failure mode is not critical: We simply +# build all targets. + +: ${DRVMAP_PATH:=pipeline/drvmap.json} +: ${BUILDKITE_TOKEN_PATH:=~/buildkite-token} + +git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" + +FIRST=$(git merge-base FETCH_HEAD "${BUILDKITE_COMMIT}") +SECOND=$(git rev-parse "$FIRST~1") +THIRD=$(git rev-parse "$FIRST~2") + +function most_relevant_builds { + set -u + curl 'https://graphql.buildkite.com/v1' \ + --silent \ + -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \ + -d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \ + jq -r '.data.pipeline.builds.edges[] | .node.uuid' +} + +mkdir -p tmp +for build in $(most_relevant_builds); do + echo "Checking artifacts for build $build" + buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true + + if [[ -f "tmp/${DRVMAP_PATH}" ]]; then + echo "Fetched target map from build ${build}" + mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json + break + fi +done |