about summary refs log tree commit diff
path: root/ops/pipelines
diff options
context:
space:
mode:
Diffstat (limited to 'ops/pipelines')
-rw-r--r--ops/pipelines/depot.nix52
-rw-r--r--ops/pipelines/static-pipeline.yaml9
2 files changed, 51 insertions, 10 deletions
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 8d4f7d27578d..7b095cd71278 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -7,8 +7,16 @@
 { depot, lib, pkgs, ... }:
 
 let
-  inherit (builtins) concatStringsSep foldl' map toJSON;
-  inherit (pkgs) symlinkJoin writeText;
+  inherit (builtins)
+    attrValues
+    concatStringsSep
+    foldl'
+    length
+    map
+    mapAttrs
+    toJSON;
+
+  inherit (pkgs) runCommandNoCC symlinkJoin writeText;
 
   # Create an expression that builds the target at the specified
   # location.
@@ -64,11 +72,8 @@ let
     label = ":water_buffalo:";
   };
 
-  # This defines the build pipeline, using the pipeline format
-  # documented on https://buildkite.com/docs/pipelines/defining-steps
-  #
-  # Pipeline steps need to stay in order.
-  pipeline.steps =
+  # All pipeline steps before batching them into smaller chunks.
+  allSteps =
     # Create build steps for each CI target
     (map mkStep depot.ci.targets)
 
@@ -76,4 +81,35 @@ let
       # Simultaneously run protobuf checks
       protoCheck
     ];
-in (writeText "depot.yaml" (toJSON pipeline))
+
+  # Helper function to inelegantly divide a list into chunks of at
+  # most n elements.
+  #
+  # This works by assigning each element a chunk ID based on its
+  # index, and then grouping all elements by their chunk ID.
+  chunksOf = n: list: let
+    chunkId = idx: toString (idx / n + 1);
+    assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list;
+    unchunk = mapAttrs (_: elements: map (e: e.value) elements);
+  in unchunk (lib.groupBy (e: e.chunk) assigned);
+
+  # Define a build pipeline chunk as a JSON file, using the pipeline
+  # format documented on
+  # https://buildkite.com/docs/pipelines/defining-steps.
+  makePipelineChunk = chunkId: chunk: rec {
+    filename = "chunk-${chunkId}.json";
+    path = writeText filename (toJSON {
+      steps = chunk;
+    });
+  };
+
+  pipelineChunks = attrValues (mapAttrs makePipelineChunk (chunksOf 256 allSteps));
+
+in runCommandNoCC "depot-pipeline" {} ''
+  mkdir $out
+  echo "Generated ${toString (length pipelineChunks)} pipeline chunks"
+  ${
+    lib.concatMapStringsSep "\n"
+      (chunk: "cp ${chunk.path} $out/${chunk.filename}") pipelineChunks
+  }
+''
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
index f366afe24cca..9f9a93c65814 100644
--- a/ops/pipelines/static-pipeline.yaml
+++ b/ops/pipelines/static-pipeline.yaml
@@ -8,8 +8,13 @@ steps:
   - label: ":llama:"
     command: |
       set -ue
-      nix-build -A ops.pipelines.depot -o depot.yaml --show-trace && \
-        buildkite-agent pipeline upload depot.yaml
+      nix-build -A ops.pipelines.depot -o pipeline --show-trace
+
+      # Steps need to be uploaded in reverse order because pipeline
+      # upload prepends instead of appending.
+      ls pipeline/chunk-*.json | tac | while read chunk; do
+        buildkite-agent pipeline upload $$chunk
+      done
 
   # Wait for all previous steps to complete.
   - wait: null