fix(ops/pipelines): Chunk build pipeline into multiple uploads r/3248

The number of jobs in the depot pipeline is reaching the limits of the
Buildkite backend's ability for a single pipeline upload. Based on a
conversation with their support my understanding is that this has to
do with internal locking mechanisms at Buildkite.

To work around this, we can instead chunk the pipeline into several
smaller chunks that are uploaded serially.

This commit introduces logic to chunk the pipeline accordingly. The
chunk size chosen is 256 for now (a multiple of our number of agents,
which is useful if we can get builds from the first chunk to start
before the next ones are uploaded).

Note that this chunk size is significantly below even the current
number of targets (~460 as of this commit), but choosing a lower chunk
size might alleviate problems we've been seeing with timeouts during
pipeline uploads.

Change-Id: I77030aaf8b874c330218b78c77d15216e13b9af7
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4332
Tested-by: BuildkiteCI
Reviewed-by: wpcarro <wpcarro@gmail.com>
Autosubmit: tazjin <mail@tazj.in>

2 files changed, 51 insertions, 10 deletions

diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 8d4f7d27578d..7b095cd71278 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -7,8 +7,16 @@
 { depot, lib, pkgs, ... }:
 
 let
-  inherit (builtins) concatStringsSep foldl' map toJSON;
-  inherit (pkgs) symlinkJoin writeText;
+  inherit (builtins)
+    attrValues
+    concatStringsSep
+    foldl'
+    length
+    map
+    mapAttrs
+    toJSON;
+
+  inherit (pkgs) runCommandNoCC symlinkJoin writeText;
 
   # Create an expression that builds the target at the specified
   # location.
@@ -64,11 +72,8 @@ let
     label = ":water_buffalo:";
   };
 
-  # This defines the build pipeline, using the pipeline format
-  # documented on https://buildkite.com/docs/pipelines/defining-steps
-  #
-  # Pipeline steps need to stay in order.
-  pipeline.steps =
+  # All pipeline steps before batching them into smaller chunks.
+  allSteps =
     # Create build steps for each CI target
     (map mkStep depot.ci.targets)
 
@@ -76,4 +81,35 @@ let
       # Simultaneously run protobuf checks
       protoCheck
     ];
-in (writeText "depot.yaml" (toJSON pipeline))
+
+  # Helper function to inelegantly divide a list into chunks of at
+  # most n elements.
+  #
+  # This works by assigning each element a chunk ID based on its
+  # index, and then grouping all elements by their chunk ID.
+  chunksOf = n: list: let
+    chunkId = idx: toString (idx / n + 1);
+    assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list;
+    unchunk = mapAttrs (_: elements: map (e: e.value) elements);
+  in unchunk (lib.groupBy (e: e.chunk) assigned);
+
+  # Define a build pipeline chunk as a JSON file, using the pipeline
+  # format documented on
+  # https://buildkite.com/docs/pipelines/defining-steps.
+  makePipelineChunk = chunkId: chunk: rec {
+    filename = "chunk-${chunkId}.json";
+    path = writeText filename (toJSON {
+      steps = chunk;
+    });
+  };
+
+  pipelineChunks = attrValues (mapAttrs makePipelineChunk (chunksOf 256 allSteps));
+
+in runCommandNoCC "depot-pipeline" {} ''
+  mkdir $out
+  echo "Generated ${toString (length pipelineChunks)} pipeline chunks"
+  ${
+    lib.concatMapStringsSep "\n"
+      (chunk: "cp ${chunk.path} $out/${chunk.filename}") pipelineChunks
+  }
+''
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
index f366afe24cca..9f9a93c65814 100644
--- a/ops/pipelines/static-pipeline.yaml
+++ b/ops/pipelines/static-pipeline.yaml
@@ -8,8 +8,13 @@ steps:
   - label: ":llama:"
     command: |
       set -ue
-      nix-build -A ops.pipelines.depot -o depot.yaml --show-trace && \
-        buildkite-agent pipeline upload depot.yaml
+      nix-build -A ops.pipelines.depot -o pipeline --show-trace
+
+      # Steps need to be uploaded in reverse order because pipeline
+      # upload prepends instead of appending.
+      ls pipeline/chunk-*.json | tac | while read chunk; do
+        buildkite-agent pipeline upload $$chunk
+      done
 
   # Wait for all previous steps to complete.
   - wait: null