4 files changed, 113 insertions, 0 deletions
diff --git a/ops/pipelines/README.md b/ops/pipelines/README.md
new file mode 100644
index 000000000000..a3f94fd23143
--- /dev/null
+++ b/ops/pipelines/README.md
@@ -0,0 +1,5 @@
+This folder contains the dynamic configuration for our [Buildkite CI
+setup](https://tvl.fyi/builds).
+
+The configuration is built and dynamically loaded by Buildkite at the start of
+each CI pipeline.
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
new file mode 100644
index 000000000000..ec7fb813278b
--- /dev/null
+++ b/ops/pipelines/depot.nix
@@ -0,0 +1,85 @@
+# This file configures the primary build pipeline used for the
+# top-level list of depot targets.
+#
+# It outputs a "YAML" (actually JSON) file which is evaluated and
+# submitted to Buildkite at the start of each build. This means we can
+# dynamically configure the pipeline execution here.
+{ depot, lib, pkgs, ... }:
+
+let
+  inherit (builtins) concatStringsSep foldl' map toJSON;
+  inherit (lib) singleton;
+  inherit (pkgs) writeText;
+
+  # Create an expression that builds the target at the specified
+  # location.
+  mkBuildExpr = target:
+    let
+      descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
+      targetExpr = foldl' descend "import ./. {}" target.__readTree;
+      subtargetExpr = descend targetExpr target.__subtarget;
+    in if target ? __subtarget then subtargetExpr else targetExpr;
+
+  # Create a pipeline label from the targets tree location.
+  mkLabel = target:
+    let label = concatStringsSep "/" target.__readTree;
+    in if target ? __subtarget
+      then "${label}:${target.__subtarget}"
+      else label;
+
+  # Create a pipeline step from a single target.
+  #
+  # If the build fails, Buildkite metadata is updated to mark the
+  # pipeline as failed. Buildkite has a concept of a failed pipeline
+  # regardless, but this data is not accessible.
+  mkStep = target: {
+    command = ''
+      nix-build -E '${mkBuildExpr target}' || (buildkite-agent meta-data set "failure" "1"; exit 1)
+    '';
+    label = ":nix: ${mkLabel target}";
+  };
+
+  # Protobuf check step which validates that changes to .proto files
+  # between revisions don't cause backwards-incompatible or otherwise
+  # flawed changes.
+  protoCheck = {
+    command = "${depot.nix.bufCheck}/bin/ci-buf-check";
+    label = ":water_buffalo:";
+  };
+
+  # This defines the build pipeline, using the pipeline format
+  # documented on https://buildkite.com/docs/pipelines/defining-steps
+  #
+  # Pipeline steps need to stay in order.
+  pipeline.steps =
+    # Zero the failure status
+    [
+      {
+        command = "buildkite-agent meta-data set 'failure' '0'";
+        label = ":buildkite:";
+      }
+      { wait = null; }
+    ]
+
+    # Create build steps for each CI target
+    ++ (map mkStep depot.ci.targets)
+
+    ++ [
+      # Simultaneously run protobuf checks
+      protoCheck
+
+      # Wait for all previous checks to complete
+      ({
+        wait = null;
+        continue_on_failure = true;
+      })
+
+      # Wait for all steps to complete, then exit with success or
+      # failure depending on whether any failure status was written.
+      # This step must be :duck:! (yes, really!)
+      ({
+        command = "exit $(buildkite-agent meta-data get 'failure')";
+        label = ":duck:";
+      })
+    ];
+in (writeText "depot.yaml" (toJSON pipeline))
diff --git a/ops/pipelines/fallback.yaml b/ops/pipelines/fallback.yaml
new file mode 100644
index 000000000000..73308d937b0c
--- /dev/null
+++ b/ops/pipelines/fallback.yaml
@@ -0,0 +1,8 @@
+# This build configuration provides a fallback which marks a build as
+# failed. This is used if evaluating the build configuration fails,
+# for example because of a syntax error in Nix code.
+---
+steps:
+  - command: "echo 'Nix evaluation failed!' && exit 1"
+    # This step *must* be :duck: to trigger the correct hook.
+    label: ":duck:"
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
new file mode 100644
index 000000000000..7a6e911ac252
--- /dev/null
+++ b/ops/pipelines/static-pipeline.yaml
@@ -0,0 +1,15 @@
+# This file defines the static pipeline which is uploaded in the
+# Buildkite admin interface. These steps run at the beginning of each
+# build and cause the dynamic pipeline generation to run.
+---
+steps:
+  - label: ":llama:"
+    command: |
+      function fallback() {
+        echo 'Using fallback pipeline ...'
+        buildkite-agent pipeline upload ops/pipelines/fallback.yaml
+        exit
+      }
+
+      nix-build -A ops.pipelines.depot -o depot.yaml --show-trace || fallback
+      buildkite-agent pipeline upload depot.yaml || fallback