3 files changed, 170 insertions, 0 deletions
diff --git a/ops/pipelines/README.md b/ops/pipelines/README.md
new file mode 100644
index 0000000000..a3f94fd231
--- /dev/null
+++ b/ops/pipelines/README.md
@@ -0,0 +1,5 @@
+This folder contains the dynamic configuration for our [Buildkite CI
+setup](https://tvl.fyi/builds).
+
+The configuration is built and dynamically loaded by Buildkite at the start of
+each CI pipeline.
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
new file mode 100644
index 0000000000..6d9e625e04
--- /dev/null
+++ b/ops/pipelines/depot.nix
@@ -0,0 +1,49 @@
+# This file configures the primary build pipeline used for the
+# top-level list of depot targets.
+{ depot, pkgs, externalArgs, ... }:
+
+let
+  # Protobuf check step which validates that changes to .proto files
+  # between revisions don't cause backwards-incompatible or otherwise
+  # flawed changes.
+  protoCheck = {
+    command = "${depot.nix.bufCheck}/bin/ci-buf-check";
+    label = ":water_buffalo:";
+  };
+
+  pipeline = depot.nix.buildkite.mkPipeline {
+    headBranch = "refs/heads/canon";
+    drvTargets = depot.ci.targets;
+    additionalSteps = [ protoCheck ];
+
+    parentTargetMap =
+      if (externalArgs ? parentTargetMap)
+      then builtins.fromJSON (builtins.readFile externalArgs.parentTargetMap)
+      else { };
+
+    postBuildSteps = [
+      # After successful builds, create a gcroot for builds on canon.
+      #
+      # This anchors *most* of the depot, in practice it's unimportant
+      # if there is a build race and we get +-1 of the targets.
+      #
+      # Unfortunately this requires a third evaluation of the graph, but
+      # since it happens after :duck: it should not affect the timing of
+      # status reporting back to Gerrit.
+      {
+        label = ":anchor:";
+        branches = "refs/heads/canon";
+        command = ''
+          nix-build -A ci.gcroot --out-link /nix/var/nix/gcroots/depot/canon
+        '';
+      }
+    ];
+  };
+
+  drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets;
+in
+pkgs.runCommandNoCC "depot-pipeline" { } ''
+  mkdir $out
+  cp -r ${pipeline}/* $out
+  cp ${drvmap} $out/drvmap.json
+''
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
new file mode 100644
index 0000000000..2e35a8a179
--- /dev/null
+++ b/ops/pipelines/static-pipeline.yaml
@@ -0,0 +1,116 @@
+# This file defines the static Buildkite pipeline which attempts to
+# create the dynamic pipeline of all depot targets.
+#
+# If something fails during the creation of the pipeline, the fallback
+# is executed instead which will simply report an error to Gerrit.
+---
+env:
+  BUILDKITE_TOKEN_PATH: /run/agenix/buildkite-graphql-token
+steps:
+  # Run pipeline for tvl-kit when new commits arrive on canon. Since
+  # it is not part of the depot build tree, this is a useful
+  # verification to ensure we don't break external things (too much).
+  - trigger: "tvl-kit"
+    async: true
+    label: ":fork:"
+    branches: "refs/heads/canon"
+    build:
+      message: "Verification triggered by ${BUILDKITE_COMMIT}"
+
+  # Create a revision number for the current commit for builds on
+  # canon.
+  #
+  # This writes data back to Gerrit using the Buildkite agent
+  # credentials injected through a git credentials helper.
+  #
+  # Revision numbers are defined as the number of commits in the
+  # lineage of HEAD, following only the first parent of merges.
+  - label: ":git:"
+    branches: "refs/heads/canon"
+    command: |
+      git -c 'credential.helper=gerrit-creds' \
+        push origin "HEAD:refs/r/$(git rev-list --count --first-parent HEAD)"
+
+  # Generate & upload dynamic build steps
+  - label: ":llama:"
+    key: "pipeline-gen"
+    command: |
+      set -ue
+
+      if test -n "$${GERRIT_CHANGE_URL-}"; then
+        echo "This is a build of [cl/$$GERRIT_CHANGE_ID]($$GERRIT_CHANGE_URL) (at patchset #$$GERRIT_PATCHSET)" | \
+          buildkite-agent annotate
+      fi
+
+      # Attempt to fetch a target map from a parent commit on canon,
+      # except on builds of canon itself.
+      [ "${BUILDKITE_BRANCH}" != "refs/heads/canon" ] && \
+        nix/buildkite/fetch-parent-targets.sh
+
+      PIPELINE_ARGS=""
+      if [[ -f tmp/parent-target-map.json ]]; then
+        PIPELINE_ARGS="--arg parentTargetMap tmp/parent-target-map.json"
+      fi
+
+      nix-build --option restrict-eval true --include "depot=$${PWD}"\
+        --allowed-uris 'https://' \
+        -A ops.pipelines.depot \
+        -o pipeline --show-trace $$PIPELINE_ARGS
+
+      # Steps need to be uploaded in reverse order because pipeline
+      # upload prepends instead of appending.
+      ls pipeline/build-chunk-*.json | tac | while read chunk; do
+        buildkite-agent pipeline upload $$chunk
+      done
+
+      buildkite-agent artifact upload "pipeline/*"
+
+  # Wait for all previous steps to complete.
+  - wait: null
+    continue_on_failure: true
+
+  # Exit with success or failure depending on whether any other steps
+  # failed.
+  #
+  # This information is checked by querying the Buildkite GraphQL API
+  # and fetching the count of failed steps.
+  #
+  # This step must be :duck: (yes, really!) because the post-command
+  # hook will inspect this name.
+  #
+  # Note that this step has requirements for the agent environment, which
+  # are enforced in our NixOS configuration:
+  #
+  #  * curl and jq must be on the $PATH of build agents
+  #  * besadii configuration must be readable to the build agents
+  - label: ":duck:"
+    key: ":duck:"
+    command: |
+      set -ueo pipefail
+
+      readonly FAILED_JOBS=$(curl 'https://graphql.buildkite.com/v1' \
+        --silent \
+        -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \
+        -d "{\"query\": \"query BuildStatusQuery { build(uuid: \\\"$BUILDKITE_BUILD_ID\\\") { jobs(passed: false) { count } } }\"}" | \
+        jq -r '.data.build.jobs.count')
+
+      echo "$$FAILED_JOBS build jobs failed."
+
+      if (( $$FAILED_JOBS > 0 )); then
+        exit 1
+      fi
+
+  # After duck, on success, upload and run any post-build steps that
+  # were output by the dynamic pipeline.
+  - label: ":arrow_heading_down:"
+    depends_on:
+      - step: ":duck:"
+        allow_failure: false
+    command: |
+      set -ueo pipefail
+
+      buildkite-agent artifact download "pipeline/*" .
+
+      find ./pipeline -name 'post-chunk-*.json' | tac | while read chunk; do
+        buildkite-agent pipeline upload $$chunk
+      done