diff options
Diffstat (limited to 'ops/pipelines')
| -rw-r--r-- | ops/pipelines/depot.nix | 52 | ||||
| -rw-r--r-- | ops/pipelines/static-pipeline.yaml | 9 |
2 files changed, 51 insertions, 10 deletions
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix index 8d4f7d27578d..7b095cd71278 100644 --- a/ops/pipelines/depot.nix +++ b/ops/pipelines/depot.nix @@ -7,8 +7,16 @@ { depot, lib, pkgs, ... }: let - inherit (builtins) concatStringsSep foldl' map toJSON; - inherit (pkgs) symlinkJoin writeText; + inherit (builtins) + attrValues + concatStringsSep + foldl' + length + map + mapAttrs + toJSON; + + inherit (pkgs) runCommandNoCC symlinkJoin writeText; # Create an expression that builds the target at the specified # location. @@ -64,11 +72,8 @@ let label = ":water_buffalo:"; }; - # This defines the build pipeline, using the pipeline format - # documented on https://buildkite.com/docs/pipelines/defining-steps - # - # Pipeline steps need to stay in order. - pipeline.steps = + # All pipeline steps before batching them into smaller chunks. + allSteps = # Create build steps for each CI target (map mkStep depot.ci.targets) @@ -76,4 +81,35 @@ let # Simultaneously run protobuf checks protoCheck ]; -in (writeText "depot.yaml" (toJSON pipeline)) + + # Helper function to inelegantly divide a list into chunks of at + # most n elements. + # + # This works by assigning each element a chunk ID based on its + # index, and then grouping all elements by their chunk ID. + chunksOf = n: list: let + chunkId = idx: toString (idx / n + 1); + assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list; + unchunk = mapAttrs (_: elements: map (e: e.value) elements); + in unchunk (lib.groupBy (e: e.chunk) assigned); + + # Define a build pipeline chunk as a JSON file, using the pipeline + # format documented on + # https://buildkite.com/docs/pipelines/defining-steps. + makePipelineChunk = chunkId: chunk: rec { + filename = "chunk-${chunkId}.json"; + path = writeText filename (toJSON { + steps = chunk; + }); + }; + + pipelineChunks = attrValues (mapAttrs makePipelineChunk (chunksOf 256 allSteps)); + +in runCommandNoCC "depot-pipeline" {} '' + mkdir $out + echo "Generated ${toString (length pipelineChunks)} pipeline chunks" + ${ + lib.concatMapStringsSep "\n" + (chunk: "cp ${chunk.path} $out/${chunk.filename}") pipelineChunks + } +'' diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml index f366afe24cca..9f9a93c65814 100644 --- a/ops/pipelines/static-pipeline.yaml +++ b/ops/pipelines/static-pipeline.yaml @@ -8,8 +8,13 @@ steps: - label: ":llama:" command: | set -ue - nix-build -A ops.pipelines.depot -o depot.yaml --show-trace && \ - buildkite-agent pipeline upload depot.yaml + nix-build -A ops.pipelines.depot -o pipeline --show-trace + + # Steps need to be uploaded in reverse order because pipeline + # upload prepends instead of appending. + ls pipeline/chunk-*.json | tac | while read chunk; do + buildkite-agent pipeline upload $$chunk + done # Wait for all previous steps to complete. - wait: null |