refactor: Generalise pipeline generation in //nix/buildkite r/3491

Extracts the logic for generating our Buildkite pipeline (which has
been copy&pasted and slightly modified in some places outside of
depot) into a generic //nix/buildkite library.

This should cause no change in functionality.

Change-Id: Iad3201713945de41279b39e4f1b847f697c179f7
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4726
Autosubmit: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>

2 files changed, 172 insertions, 115 deletions

diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
new file mode 100644
index 0000000000..09fe0a741e
--- /dev/null
+++ b/nix/buildkite/default.nix
@@ -0,0 +1,165 @@
+# Logic for generating Buildkite pipelines from Nix build targets read
+# by //nix/readTree.
+#
+# It outputs a "YAML" (actually JSON) file which is evaluated and
+# submitted to Buildkite at the start of each build.
+#
+# The structure of the file that is being created is documented here:
+#   https://buildkite.com/docs/pipelines/defining-steps
+{ pkgs, ... }:
+
+let
+  inherit (builtins)
+    attrValues
+    concatMap
+    concatStringsSep
+    filter
+    foldl'
+    length
+    mapAttrs
+    toJSON;
+
+  inherit (pkgs) lib runCommandNoCC writeText;
+in rec {
+  # Creates a Nix expression that yields the target at the specified
+  # location in the repository.
+  #
+  # This makes a distinction between normal targets (which physically
+  # exist in the repository) and subtargets (which are "virtual"
+  # targets exposed by a physical one) to make it clear in the build
+  # output which is which.
+  mkBuildExpr = target:
+    let
+      descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
+      targetExpr = foldl' descend "import ./. {}" target.__readTree;
+      subtargetExpr = descend targetExpr target.__subtarget;
+    in if target ? __subtarget then subtargetExpr else targetExpr;
+
+  # Create a pipeline label from the target's tree location.
+  mkLabel = target:
+    let label = concatStringsSep "/" target.__readTree;
+    in if target ? __subtarget
+      then "${label}:${target.__subtarget}"
+      else label;
+
+  # Skip build steps if their out path has already been built.
+  skip = headBranch: target: let
+    shouldSkip = with builtins;
+      # Only skip in real Buildkite builds
+      (getEnv "BUILDKITE_BUILD_ID" != "") &&
+      # Always build everything for the canon branch.
+      (getEnv "BUILDKITE_BRANCH" != headBranch) &&
+      # Discard string context to avoid realising the store path during
+      # pipeline construction.
+      (pathExists (unsafeDiscardStringContext target.outPath));
+    in if shouldSkip then "Target was already built." else false;
+
+  # Create a pipeline step from a single target.
+  mkStep = headBranch: skipIfBuilt: target: {
+    label = ":nix: ${mkLabel target}";
+    skip = if skipIfBuilt then skip headBranch target else false;
+
+    command = let
+      drvPath = builtins.unsafeDiscardStringContext target.drvPath;
+    in concatStringsSep " " [
+      # First try to realise the drvPath of the target so we don't evaluate twice.
+      # Nix has no concept of depending on a derivation file without depending on
+      # at least one of its `outPath`s, so we need to discard the string context
+      # if we don't want to build everything during pipeline construction.
+      "nix-store --realise '${drvPath}'"
+      # Since we don't gcroot the derivation files, they may be deleted by the
+      # garbage collector. In that case we can reevaluate and build the attribute
+      # using nix-build.
+      "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)"
+    ];
+
+    # Add a dependency on the initial static pipeline step which
+    # always runs. This allows build steps uploaded in batches to
+    # start running before all batches have been uploaded.
+    depends_on = ":init:";
+  };
+
+  # Helper function to inelegantly divide a list into chunks of at
+  # most n elements.
+  #
+  # This works by assigning each element a chunk ID based on its
+  # index, and then grouping all elements by their chunk ID.
+  chunksOf = n: list: let
+    chunkId = idx: toString (idx / n + 1);
+    assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list;
+    unchunk = mapAttrs (_: elements: map (e: e.value) elements);
+  in unchunk (lib.groupBy (e: e.chunk) assigned);
+
+  # Define a build pipeline chunk as a JSON file, using the pipeline
+  # format documented on
+  # https://buildkite.com/docs/pipelines/defining-steps.
+  makePipelineChunk = chunkId: chunk: rec {
+    filename = "chunk-${chunkId}.json";
+    path = writeText filename (toJSON {
+      steps = chunk;
+    });
+  };
+
+  # Split the pipeline into chunks of at most 256 steps at once, which
+  # are uploaded sequentially. This is because of a limitation in the
+  # Buildkite backend which struggles to process more than a specific
+  # number of chunks at once.
+  pipelineChunks = steps:
+    attrValues (mapAttrs makePipelineChunk (chunksOf 256 steps));
+
+  # Create a pipeline structure for the given targets.
+  mkPipeline = {
+    # HEAD branch of the repository on which release steps, GC
+    # anchoring and other "mainline only" steps should run.
+    headBranch,
+
+    # List of derivations as read by readTree (in most cases just the
+    # output of readTree.gather) that should be built in Buildkite.
+    #
+    # These are scheduled as the first build steps and run as fast as
+    # possible, in order, without any concurrency restrictions.
+    drvTargets,
+
+    # Should build steps be skipped (on non-HEAD builds) if the output
+    # path has already been built?
+    skipIfBuilt ? false,
+
+    # A list of plain Buildkite step structures to run alongside the
+    # build for all drvTargets, but before proceeding with any
+    # post-build actions such as status reporting.
+    #
+    # Can be used for things like code formatting checks.
+    additionalSteps ? [],
+
+    # A list of plain Buildkite step structures to run after all
+    # previous steps succeeded.
+    #
+    # Can be used for status reporting steps and the like.
+    postBuildSteps ? []
+  }: let
+    mkStep' = mkStep headBranch skipIfBuilt;
+    steps =
+      # Add build steps for each derivation target.
+      (map mkStep' drvTargets)
+
+      # Add additional steps (if set).
+      ++ additionalSteps
+
+      # Wait for all previous checks to complete
+      ++ [({
+        wait = null;
+        continue_on_failure = true;
+      })]
+
+      # Run post-build steps for status reporting and co.
+      ++ postBuildSteps;
+    chunks = pipelineChunks steps;
+  in runCommandNoCC "buildkite-pipeline" {} ''
+    mkdir $out
+    echo "Generated ${toString (length chunks)} pipeline chunks"
+    ${
+      lib.concatMapStringsSep "\n"
+        (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
+    }
+  '';
+}
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 5843423d05..63b1bc067a 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -1,80 +1,8 @@
 # This file configures the primary build pipeline used for the
 # top-level list of depot targets.
-#
-# It outputs a "YAML" (actually JSON) file which is evaluated and
-# submitted to Buildkite at the start of each build. This means we can
-# dynamically configure the pipeline execution here.
-{ depot, lib, pkgs, ... }:
+{ depot, ... }:
 
 let
-  inherit (builtins)
-    attrValues
-    concatStringsSep
-    foldl'
-    length
-    map
-    mapAttrs
-    toJSON;
-
-  inherit (pkgs) runCommandNoCC symlinkJoin writeText;
-
-  # Create an expression that builds the target at the specified
-  # location.
-  mkBuildExpr = target:
-    let
-      descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
-      targetExpr = foldl' descend "import ./. {}" target.__readTree;
-      subtargetExpr = descend targetExpr target.__subtarget;
-    in if target ? __subtarget then subtargetExpr else targetExpr;
-
-  # Create a pipeline label from the targets tree location.
-  mkLabel = target:
-    let label = concatStringsSep "/" target.__readTree;
-    in if target ? __subtarget
-      then "${label}:${target.__subtarget}"
-      else label;
-
-  # Create a pipeline step from a single target.
-  mkStep = target: {
-    command = let
-      drvPath = builtins.unsafeDiscardStringContext target.drvPath;
-    in lib.concatStringsSep " " [
-      # First try to realise the drvPath of the target so we don't evaluate twice.
-      # Nix has no concept of depending on a derivation file without depending on
-      # at least one of its `outPath`s, so we need to discard the string context
-      # if we don't want to build everything during pipeline construction.
-      "nix-store --realise '${drvPath}'"
-      # Since we don't gcroot the derivation files, they may be deleted by the
-      # garbage collector. In that case we can reevaluate and build the attribute
-      # using nix-build.
-      "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)"
-    ];
-    label = ":nix: ${mkLabel target}";
-
-    # Skip build steps if their out path has already been built.
-    skip = let
-      shouldSkip = with builtins;
-        # Only skip in real Buildkite builds
-        (getEnv "BUILDKITE_BUILD_ID" != "") &&
-        # Always build everything for the canon branch.
-        (getEnv "BUILDKITE_BRANCH" != "refs/heads/canon") &&
-        # Discard string context to avoid realising the store path during
-        # pipeline construction.
-        (pathExists (unsafeDiscardStringContext target.outPath));
-      in if shouldSkip then "Target was already built." else false;
-
-    # Add a "fake" dependency on the initial static pipeline step. When
-    # uploading a pipeline dynamically, an implicit dependency on the uploading
-    # step is added to all newly created build steps. Since we are uploading in
-    # batches this stops the jobs in the first batch from running before all
-    # batches have been uploaded.
-    #
-    # By setting an explicit dependency on a step that has always completed at
-    # this point, we override that behaviour and allow the steps to start
-    # running already.
-    depends_on = ":init:";
-  };
-
   # Protobuf check step which validates that changes to .proto files
   # between revisions don't cause backwards-incompatible or otherwise
   # flawed changes.
@@ -82,45 +10,9 @@ let
     command = "${depot.nix.bufCheck}/bin/ci-buf-check";
     label = ":water_buffalo:";
   };
-
-  # All pipeline steps before batching them into smaller chunks.
-  allSteps =
-    # Create build steps for each CI target
-    (map mkStep depot.ci.targets)
-
-    ++ [
-      # Simultaneously run protobuf checks
-      protoCheck
-    ];
-
-  # Helper function to inelegantly divide a list into chunks of at
-  # most n elements.
-  #
-  # This works by assigning each element a chunk ID based on its
-  # index, and then grouping all elements by their chunk ID.
-  chunksOf = n: list: let
-    chunkId = idx: toString (idx / n + 1);
-    assigned = lib.imap1 (idx: value: { inherit value ; chunk = chunkId idx; }) list;
-    unchunk = mapAttrs (_: elements: map (e: e.value) elements);
-  in unchunk (lib.groupBy (e: e.chunk) assigned);
-
-  # Define a build pipeline chunk as a JSON file, using the pipeline
-  # format documented on
-  # https://buildkite.com/docs/pipelines/defining-steps.
-  makePipelineChunk = chunkId: chunk: rec {
-    filename = "chunk-${chunkId}.json";
-    path = writeText filename (toJSON {
-      steps = chunk;
-    });
-  };
-
-  pipelineChunks = attrValues (mapAttrs makePipelineChunk (chunksOf 256 allSteps));
-
-in runCommandNoCC "depot-pipeline" {} ''
-  mkdir $out
-  echo "Generated ${toString (length pipelineChunks)} pipeline chunks"
-  ${
-    lib.concatMapStringsSep "\n"
-      (chunk: "cp ${chunk.path} $out/${chunk.filename}") pipelineChunks
-  }
-''
+in depot.nix.buildkite.mkPipeline {
+  headBranch = "refs/heads/canon";
+  drvTargets = depot.ci.targets;
+  skipIfBuilt = true;
+  additionalSteps = [ protoCheck ];
+}