2 files changed, 167 insertions, 78 deletions
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index ec7fb81327..5eff622671 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -1,85 +1,40 @@
 # This file configures the primary build pipeline used for the
 # top-level list of depot targets.
-#
-# It outputs a "YAML" (actually JSON) file which is evaluated and
-# submitted to Buildkite at the start of each build. This means we can
-# dynamically configure the pipeline execution here.
-{ depot, lib, pkgs, ... }:
+{ depot, pkgs, externalArgs, ... }:
 
 let
-  inherit (builtins) concatStringsSep foldl' map toJSON;
-  inherit (lib) singleton;
-  inherit (pkgs) writeText;
-
-  # Create an expression that builds the target at the specified
-  # location.
-  mkBuildExpr = target:
-    let
-      descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
-      targetExpr = foldl' descend "import ./. {}" target.__readTree;
-      subtargetExpr = descend targetExpr target.__subtarget;
-    in if target ? __subtarget then subtargetExpr else targetExpr;
-
-  # Create a pipeline label from the targets tree location.
-  mkLabel = target:
-    let label = concatStringsSep "/" target.__readTree;
-    in if target ? __subtarget
-      then "${label}:${target.__subtarget}"
-      else label;
-
-  # Create a pipeline step from a single target.
-  #
-  # If the build fails, Buildkite metadata is updated to mark the
-  # pipeline as failed. Buildkite has a concept of a failed pipeline
-  # regardless, but this data is not accessible.
-  mkStep = target: {
-    command = ''
-      nix-build -E '${mkBuildExpr target}' || (buildkite-agent meta-data set "failure" "1"; exit 1)
-    '';
-    label = ":nix: ${mkLabel target}";
-  };
-
-  # Protobuf check step which validates that changes to .proto files
-  # between revisions don't cause backwards-incompatible or otherwise
-  # flawed changes.
-  protoCheck = {
-    command = "${depot.nix.bufCheck}/bin/ci-buf-check";
-    label = ":water_buffalo:";
-  };
-
-  # This defines the build pipeline, using the pipeline format
-  # documented on https://buildkite.com/docs/pipelines/defining-steps
-  #
-  # Pipeline steps need to stay in order.
-  pipeline.steps =
-    # Zero the failure status
-    [
+  pipeline = depot.nix.buildkite.mkPipeline {
+    headBranch = "refs/heads/canon";
+    drvTargets = depot.ci.targets;
+
+    parentTargetMap =
+      if (externalArgs ? parentTargetMap)
+      then builtins.fromJSON (builtins.readFile externalArgs.parentTargetMap)
+      else { };
+
+    postBuildSteps = [
+      # After successful builds, create a gcroot for builds on canon.
+      #
+      # This anchors *most* of the depot, in practice it's unimportant
+      # if there is a build race and we get +-1 of the targets.
+      #
+      # Unfortunately this requires a third evaluation of the graph, but
+      # since it happens after :duck: it should not affect the timing of
+      # status reporting back to Gerrit.
       {
-        command = "buildkite-agent meta-data set 'failure' '0'";
-        label = ":buildkite:";
+        label = ":anchor:";
+        branches = "refs/heads/canon";
+        command = ''
+          nix-build -A ci.gcroot --out-link /nix/var/nix/gcroots/depot/canon
+        '';
       }
-      { wait = null; }
-    ]
-
-    # Create build steps for each CI target
-    ++ (map mkStep depot.ci.targets)
-
-    ++ [
-      # Simultaneously run protobuf checks
-      protoCheck
-
-      # Wait for all previous checks to complete
-      ({
-        wait = null;
-        continue_on_failure = true;
-      })
-
-      # Wait for all steps to complete, then exit with success or
-      # failure depending on whether any failure status was written.
-      # This step must be :duck:! (yes, really!)
-      ({
-        command = "exit $(buildkite-agent meta-data get 'failure')";
-        label = ":duck:";
-      })
     ];
-in (writeText "depot.yaml" (toJSON pipeline))
+  };
+
+  drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets;
+in
+pkgs.runCommand "depot-pipeline" { } ''
+  mkdir $out
+  cp -r ${pipeline}/* $out
+  cp ${drvmap} $out/drvmap.json
+''
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
new file mode 100644
index 0000000000..af4f9d784e
--- /dev/null
+++ b/ops/pipelines/static-pipeline.yaml
@@ -0,0 +1,134 @@
+# This file defines the static Buildkite pipeline which attempts to
+# create the dynamic pipeline of all depot targets.
+#
+# If something fails during the creation of the pipeline, the fallback
+# is executed instead which will simply report an error to Gerrit.
+---
+env:
+  BUILDKITE_TOKEN_PATH: /run/agenix/buildkite-graphql-token
+steps:
+  # Run pipeline for tvl-kit when new commits arrive on canon. Since
+  # it is not part of the depot build tree, this is a useful
+  # verification to ensure we don't break external things (too much).
+  - trigger: "tvl-kit"
+    async: true
+    label: ":fork:"
+    branches: "refs/heads/canon"
+    build:
+      message: "Verification triggered by ${BUILDKITE_COMMIT}"
+
+  # Run pipeline for tvix when new commits arrive on canon. Since
+  # it is not part of the depot build tree, this is a useful
+  # verification to ensure we don't break external things (too much).
+  - trigger: "tvix"
+    async: true
+    label: ":fork:"
+    branches: "refs/heads/canon"
+    build:
+      message: "Verification triggered by ${BUILDKITE_COMMIT}"
+
+  # Create a revision number for the current commit for builds on
+  # canon.
+  #
+  # This writes data back to Gerrit using the Buildkite agent
+  # credentials injected through a git credentials helper.
+  #
+  # Revision numbers are defined as the number of commits in the
+  # lineage of HEAD, following only the first parent of merges.
+  #
+  # Note that git does not fetch these refs by default, instead
+  # you'll have to modify your git config using
+  # `git config --add remote.origin.fetch '+refs/r/*:refs/r/*'`.
+  # The refs are available after the next `git fetch`.
+  - label: ":git:"
+    branches: "refs/heads/canon"
+    command: |
+      git -c 'credential.helper=gerrit-creds' \
+        push origin "HEAD:refs/r/$(git rev-list --count --first-parent HEAD)"
+
+  # Generate & upload dynamic build steps
+  - label: ":llama:"
+    key: "pipeline-gen"
+    concurrency_group: 'depot-nix-eval'
+    concurrency: 5 # much more than this and whitby will OOM
+    command: |
+      set -ue
+
+      if test -n "$${GERRIT_CHANGE_URL-}"; then
+        echo "This is a build of [cl/$$GERRIT_CHANGE_ID]($$GERRIT_CHANGE_URL) (at patchset #$$GERRIT_PATCHSET)" | \
+          buildkite-agent annotate --context cl-annotation
+      fi
+
+      # Attempt to fetch a target map from a parent commit on canon,
+      # except on builds of canon itself.
+      [ "${BUILDKITE_BRANCH}" != "refs/heads/canon" ] && \
+        nix/buildkite/fetch-parent-targets.sh
+
+      PIPELINE_ARGS=""
+      if [[ -f tmp/parent-target-map.json ]]; then
+        PIPELINE_ARGS="--arg parentTargetMap tmp/parent-target-map.json"
+      fi
+
+      nix-build --option restrict-eval true --include "depot=$${PWD}" \
+        --include "store=/nix/store" \
+        --allowed-uris 'https://' \
+        -A ops.pipelines.depot \
+        -o pipeline --show-trace $$PIPELINE_ARGS
+
+      # Steps need to be uploaded in reverse order because pipeline
+      # upload prepends instead of appending.
+      ls pipeline/build-chunk-*.json | tac | while read chunk; do
+        buildkite-agent pipeline upload $$chunk
+      done
+
+      buildkite-agent artifact upload "pipeline/*"
+
+  # Wait for all previous steps to complete.
+  - wait: null
+    continue_on_failure: true
+
+  # Exit with success or failure depending on whether any other steps
+  # failed.
+  #
+  # This information is checked by querying the Buildkite GraphQL API
+  # and fetching the count of failed steps.
+  #
+  # This step must be :duck: (yes, really!) because the post-command
+  # hook will inspect this name.
+  #
+  # Note that this step has requirements for the agent environment, which
+  # are enforced in our NixOS configuration:
+  #
+  #  * curl and jq must be on the $PATH of build agents
+  #  * besadii configuration must be readable to the build agents
+  - label: ":duck:"
+    key: ":duck:"
+    command: |
+      set -ueo pipefail
+
+      readonly FAILED_JOBS=$(curl 'https://graphql.buildkite.com/v1' \
+        --silent \
+        -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \
+        -d "{\"query\": \"query BuildStatusQuery { build(uuid: \\\"$BUILDKITE_BUILD_ID\\\") { jobs(passed: false) { count } } }\"}" | \
+        jq -r '.data.build.jobs.count')
+
+      echo "$$FAILED_JOBS build jobs failed."
+
+      if (( $$FAILED_JOBS > 0 )); then
+        exit 1
+      fi
+
+  # After duck, on success, upload and run any release steps that were
+  # output by the dynamic pipeline.
+  - label: ":arrow_heading_down:"
+    depends_on:
+      - step: ":duck:"
+        allow_failure: false
+    command: |
+      set -ueo pipefail
+
+      buildkite-agent artifact download "pipeline/*" .
+
+      find ./pipeline -name 'release-chunk-*.json' | tac | while read chunk; do
+        buildkite-agent pipeline upload $$chunk
+      done