refactor(ops/pipelines): Split build/post steps into separate chunks r/3657

This will create `build-chunk-$n.json` files for steps that should run
_before_ duck, and `post-chunk-$n.json` files for steps that should
run after duck.

The post steps are not yet uploaded to Buildkite, but we also don't
have any right now.

Change-Id: I7e1b59cf55a8bf1d97266f6e988aa496959077bf
Reviewed-on: https://cl.tvl.fyi/c/depot/+/5047
Tested-by: BuildkiteCI
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>
Autosubmit: tazjin <tazjin@tvl.su>

2 files changed, 10 insertions, 17 deletions

diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
index a471a30d750c..5e31cbbc118b 100644
--- a/nix/buildkite/default.nix
+++ b/nix/buildkite/default.nix
@@ -104,8 +104,8 @@ in rec {
   # Define a build pipeline chunk as a JSON file, using the pipeline
   # format documented on
   # https://buildkite.com/docs/pipelines/defining-steps.
-  makePipelineChunk = chunkId: chunk: rec {
-    filename = "chunk-${chunkId}.json";
+  makePipelineChunk = name: chunkId: chunk: rec {
+    filename = "${name}-chunk-${chunkId}.json";
     path = writeText filename (toJSON {
       steps = chunk;
     });
@@ -115,8 +115,8 @@ in rec {
   # are uploaded sequentially. This is because of a limitation in the
   # Buildkite backend which struggles to process more than a specific
   # number of chunks at once.
-  pipelineChunks = steps:
-    attrValues (mapAttrs makePipelineChunk (chunksOf 256 steps));
+  pipelineChunks = name: steps:
+    attrValues (mapAttrs (makePipelineChunk name) (chunksOf 256 steps));
 
   # Create a pipeline structure for the given targets.
   mkPipeline = {
@@ -156,24 +156,17 @@ in rec {
 
     steps = map targetToSteps drvTargets;
 
-    allSteps =
+    buildSteps =
       # Add build steps for each derivation target and their extra
       # steps.
       (lib.concatLists steps)
 
       # Add additional steps (if set).
-      ++ additionalSteps
+      ++ additionalSteps;
 
-      # Wait for all previous checks to complete
-      ++ [({
-        wait = null;
-        continue_on_failure = true;
-      })]
-
-      # Run post-build steps for status reporting and co.
-      ++ postBuildSteps;
-
-    chunks = pipelineChunks allSteps;
+    buildChunks = pipelineChunks "build" buildSteps;
+    postBuildChunks = pipelineChunks "post" postBuildSteps;
+    chunks = buildChunks ++ postBuildChunks;
   in runCommandNoCC "buildkite-pipeline" {} ''
     mkdir $out
     echo "Generated ${toString (length chunks)} pipeline chunks"
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
index cf3fc9d4fb38..4a844561e63d 100644
--- a/ops/pipelines/static-pipeline.yaml
+++ b/ops/pipelines/static-pipeline.yaml
@@ -54,7 +54,7 @@ steps:
 
       # Steps need to be uploaded in reverse order because pipeline
       # upload prepends instead of appending.
-      ls pipeline/chunk-*.json | tac | while read chunk; do
+      ls pipeline/build-chunk-*.json | tac | while read chunk; do
         buildkite-agent pipeline upload $$chunk
       done