diff options
Diffstat (limited to 'nix/buildkite/default.nix')
| -rw-r--r-- | nix/buildkite/default.nix | 312 |
1 files changed, 164 insertions, 148 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index e0c947deae91..d17b5c86c4a6 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -29,7 +29,8 @@ let unsafeDiscardStringContext; inherit (pkgs) lib runCommandNoCC writeText; -in rec { +in +rec { # Creates a Nix expression that yields the target at the specified # location in the repository. # @@ -42,14 +43,15 @@ in rec { descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; targetExpr = foldl' descend "import ./. {}" target.__readTree; subtargetExpr = descend targetExpr target.__subtarget; - in if target ? __subtarget then subtargetExpr else targetExpr; + in + if target ? __subtarget then subtargetExpr else targetExpr; # Create a pipeline label from the target's tree location. mkLabel = target: let label = concatStringsSep "/" target.__readTree; in if target ? __subtarget - then "${label}:${target.__subtarget}" - else label; + then "${label}:${target.__subtarget}" + else label; # Determine whether to skip a target if it has not diverged from the # HEAD branch. @@ -74,33 +76,36 @@ in rec { # Create a pipeline step from a single target. mkStep = headBranch: parentTargetMap: target: - let - label = mkLabel target; - drvPath = unsafeDiscardStringContext target.drvPath; - shouldSkip' = shouldSkip parentTargetMap; - in { - label = ":nix: " + label; - key = hashString "sha1" label; - skip = shouldSkip' label drvPath; - command = mkBuildCommand target drvPath; - env.READTREE_TARGET = label; - - # Add a dependency on the initial static pipeline step which - # always runs. This allows build steps uploaded in batches to - # start running before all batches have been uploaded. - depends_on = ":init:"; - }; + let + label = mkLabel target; + drvPath = unsafeDiscardStringContext target.drvPath; + shouldSkip' = shouldSkip parentTargetMap; + in + { + label = ":nix: " + label; + key = hashString "sha1" label; + skip = shouldSkip' label drvPath; + command = mkBuildCommand target drvPath; + env.READTREE_TARGET = label; + + # Add a dependency on the initial static pipeline step which + # always runs. This allows build steps uploaded in batches to + # start running before all batches have been uploaded. + depends_on = ":init:"; + }; # Helper function to inelegantly divide a list into chunks of at # most n elements. # # This works by assigning each element a chunk ID based on its # index, and then grouping all elements by their chunk ID. - chunksOf = n: list: let - chunkId = idx: toString (idx / n + 1); - assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list; - unchunk = mapAttrs (_: elements: map (e: e.value) elements); - in unchunk (lib.groupBy (e: e.chunk) assigned); + chunksOf = n: list: + let + chunkId = idx: toString (idx / n + 1); + assigned = lib.imap1 (idx: value: { inherit value; chunk = chunkId idx; }) list; + unchunk = mapAttrs (_: elements: map (e: e.value) elements); + in + unchunk (lib.groupBy (e: e.chunk) assigned); # Define a build pipeline chunk as a JSON file, using the pipeline # format documented on @@ -120,104 +125,112 @@ in rec { attrValues (mapAttrs (makePipelineChunk name) (chunksOf 192 steps)); # Create a pipeline structure for the given targets. - mkPipeline = { - # HEAD branch of the repository on which release steps, GC - # anchoring and other "mainline only" steps should run. - headBranch, - - # List of derivations as read by readTree (in most cases just the - # output of readTree.gather) that should be built in Buildkite. - # - # These are scheduled as the first build steps and run as fast as - # possible, in order, without any concurrency restrictions. - drvTargets, - - # Derivation map of a parent commit. Only targets which no longer - # correspond to the content of this map will be built. Passing an - # empty map will always build all targets. - parentTargetMap ? {}, - - # A list of plain Buildkite step structures to run alongside the - # build for all drvTargets, but before proceeding with any - # post-build actions such as status reporting. - # - # Can be used for things like code formatting checks. - additionalSteps ? [], - - # A list of plain Buildkite step structures to run after all - # previous steps succeeded. - # - # Can be used for status reporting steps and the like. - postBuildSteps ? [] - }: let - # Convert a target into all of its build and post-build steps, - # treated separately as they need to be in different chunks. - targetToSteps = target: let - step = mkStep headBranch parentTargetMap target; - - # Split build/post-build steps - splitExtraSteps = partition ({ postStep, ... }: postStep) - (attrValues (mapAttrs (name: value: { - inherit name value; - postStep = (value ? prompt) || (value.postBuild or false); - }) (target.meta.ci.extraSteps or {}))); - - mkExtraStep' = { name, value, ... }: mkExtraStep step name value; - extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt - extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt - in { - buildSteps = [ step ] ++ extraBuildSteps; - postSteps = extraPostSteps; - }; - - # Combine all target steps into separate build and post-build step lists. - steps = foldl' (acc: t: { - buildSteps = acc.buildSteps ++ t.buildSteps; - postSteps = acc.postSteps ++ t.postSteps; - }) { buildSteps = []; postSteps = []; } (map targetToSteps drvTargets); - - buildSteps = - # Add build steps for each derivation target and their extra - # steps. - steps.buildSteps - - # Add additional steps (if set). - ++ additionalSteps; - - postSteps = - # Add post-build steps for each derivation target. - steps.postSteps - - # Add any globally defined post-build steps. - ++ postBuildSteps; - - buildChunks = pipelineChunks "build" buildSteps; - postBuildChunks = pipelineChunks "post" postSteps; - chunks = buildChunks ++ postBuildChunks; - in runCommandNoCC "buildkite-pipeline" {} '' - mkdir $out - echo "Generated ${toString (length chunks)} pipeline chunks" - ${ - lib.concatMapStringsSep "\n" - (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks - } - ''; + mkPipeline = + { + # HEAD branch of the repository on which release steps, GC + # anchoring and other "mainline only" steps should run. + headBranch + , # List of derivations as read by readTree (in most cases just the + # output of readTree.gather) that should be built in Buildkite. + # + # These are scheduled as the first build steps and run as fast as + # possible, in order, without any concurrency restrictions. + drvTargets + , # Derivation map of a parent commit. Only targets which no longer + # correspond to the content of this map will be built. Passing an + # empty map will always build all targets. + parentTargetMap ? { } + , # A list of plain Buildkite step structures to run alongside the + # build for all drvTargets, but before proceeding with any + # post-build actions such as status reporting. + # + # Can be used for things like code formatting checks. + additionalSteps ? [ ] + , # A list of plain Buildkite step structures to run after all + # previous steps succeeded. + # + # Can be used for status reporting steps and the like. + postBuildSteps ? [ ] + }: + let + # Convert a target into all of its build and post-build steps, + # treated separately as they need to be in different chunks. + targetToSteps = target: + let + step = mkStep headBranch parentTargetMap target; + + # Split build/post-build steps + splitExtraSteps = partition ({ postStep, ... }: postStep) + (attrValues (mapAttrs + (name: value: { + inherit name value; + postStep = (value ? prompt) || (value.postBuild or false); + }) + (target.meta.ci.extraSteps or { }))); + + mkExtraStep' = { name, value, ... }: mkExtraStep step name value; + extraBuildSteps = map mkExtraStep' splitExtraSteps.wrong; # 'wrong' -> no prompt + extraPostSteps = map mkExtraStep' splitExtraSteps.right; # 'right' -> has prompt + in + { + buildSteps = [ step ] ++ extraBuildSteps; + postSteps = extraPostSteps; + }; + + # Combine all target steps into separate build and post-build step lists. + steps = foldl' + (acc: t: { + buildSteps = acc.buildSteps ++ t.buildSteps; + postSteps = acc.postSteps ++ t.postSteps; + }) + { buildSteps = [ ]; postSteps = [ ]; } + (map targetToSteps drvTargets); + + buildSteps = + # Add build steps for each derivation target and their extra + # steps. + steps.buildSteps + + # Add additional steps (if set). + ++ additionalSteps; + + postSteps = + # Add post-build steps for each derivation target. + steps.postSteps + + # Add any globally defined post-build steps. + ++ postBuildSteps; + + buildChunks = pipelineChunks "build" buildSteps; + postBuildChunks = pipelineChunks "post" postSteps; + chunks = buildChunks ++ postBuildChunks; + in + runCommandNoCC "buildkite-pipeline" { } '' + mkdir $out + echo "Generated ${toString (length chunks)} pipeline chunks" + ${ + lib.concatMapStringsSep "\n" + (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks + } + ''; # Create a drvmap structure for the given targets, containing the # mapping of all target paths to their derivations. The mapping can # be persisted for future use. - mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: { - name = mkLabel target; - value = { - drvPath = unsafeDiscardStringContext target.drvPath; - - # Include the attrPath in the output to reconstruct the drv - # without parsing the human-readable label. - attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ - target.__subtarget - ]; - }; - }) drvTargets))); + mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map + (target: { + name = mkLabel target; + value = { + drvPath = unsafeDiscardStringContext target.drvPath; + + # Include the attrPath in the output to reconstruct the drv + # without parsing the human-readable label. + attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ + target.__subtarget + ]; + }; + }) + drvTargets))); # Implementation of extra step logic. # @@ -278,34 +291,37 @@ in rec { # Create the Buildkite configuration for an extra step, optionally # wrapping it in a gate group. - mkExtraStep = parent: key: { - command, - label ? key, - prompt ? false, - needsOutput ? false, - branches ? null, - alwaysRun ? false, - postBuild ? false - }@cfg: let - parentLabel = parent.env.READTREE_TARGET; - - step = { - label = ":gear: ${label} (from ${parentLabel})"; - skip = if alwaysRun then false else parent.skip or false; - depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key; - branches = if branches != null then lib.concatStringsSep " " branches else null; - - command = pkgs.writeShellScript "${key}-script" '' - set -ueo pipefail - ${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"} - ${lib.optionalString needsOutput parent.command} - echo '+++ Running extra step command' - exec ${command} - ''; - }; - in if (isString prompt) - then mkGatedStep { - inherit step label parent prompt; - } + mkExtraStep = parent: key: { command + , label ? key + , prompt ? false + , needsOutput ? false + , branches ? null + , alwaysRun ? false + , postBuild ? false + }@cfg: + let + parentLabel = parent.env.READTREE_TARGET; + + step = { + label = ":gear: ${label} (from ${parentLabel})"; + skip = if alwaysRun then false else parent.skip or false; + depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key; + branches = if branches != null then lib.concatStringsSep " " branches else null; + + command = pkgs.writeShellScript "${key}-script" '' + set -ueo pipefail + ${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"} + ${lib.optionalString needsOutput parent.command} + echo '+++ Running extra step command' + exec ${command} + ''; + }; + in + if (isString prompt) + then + mkGatedStep + { + inherit step label parent prompt; + } else step; } |