feat(ops/pipelines): Dynamically generate CI pipeline from targets r/1747

Create the pipeline by outputting a file that contains nix-build
invocations for each target's *derivation path*.

Each invocation has a generated Nix expression passed to it with `-E`
which fetches the correct target from the tree while correctly
handling targets with strange characters (such as in Go-packages).

This makes it possible to run target-level granular pipelines. We're
getting somewhere!

Change-Id: Ia6946e389dafd1d4926130bb8891446d6e17133b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1855
Tested-by: BuildkiteCI
Reviewed-by: glittershark <grfn@gws.fyi>
Reviewed-by: lukegb <lukegb@tvl.fyi>

2 files changed, 67 insertions, 14 deletions

diff --git a/ops/nixos/default.nix b/ops/nixos/default.nix
index 407e667f42ba..917a56b766da 100644
--- a/ops/nixos/default.nix
+++ b/ops/nixos/default.nix
@@ -47,5 +47,5 @@ rec {
   #
   # TODO(tazjin): Refactor the whole systems setup, it's a bit
   # inconsistent at the moment.
-  whitbySystem = (nixosFor whitby).system // { __readTree = true; };
+  whitbySystem = (nixosFor whitby).system;
 }
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 2e99bf8d1393..5d1f2babe123 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -4,22 +4,75 @@
 # It outputs a "YAML" (actually JSON) file which is evaluated and
 # submitted to Buildkite at the start of each build. This means we can
 # dynamically configure the pipeline execution here.
-{ depot, pkgs, ... }:
+{ depot, lib, pkgs, ... }:
 
 let
-  inherit (builtins) toJSON;
+  inherit (builtins) concatStringsSep foldl' map toJSON;
+  inherit (lib) singleton;
   inherit (pkgs) writeText;
 
+  # Create an expression that builds the target at the specified
+  # location.
+  mkBuildExpr =
+    let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
+    in foldl' descend "import ./. {}";
+
+  # Create a pipeline label from the targets tree location.
+  mkLabel = concatStringsSep "/";
+
+  # Create a pipeline step from a single target.
+  #
+  # If the build fails, Buildkite metadata is updated to mark the
+  # pipeline as failed. Buildkite has a concept of a failed pipeline
+  # regardless, but this data is not accessible.
+  mkStep = target: {
+    command = ''
+      nix-build -E '${mkBuildExpr target.__readTree}' || (buildkite-agent meta-data set "failure" "1"; exit 1)
+    '';
+    label = ":nix: ${mkLabel target.__readTree}";
+  };
+
+  # Protobuf check step which validates that changes to .proto files
+  # between revisions don't cause backwards-incompatible or otherwise
+  # flawed changes.
+  protoCheck = {
+    command = "${depot.nix.bufCheck}/bin/ci-buf-check";
+    label = ":water_buffalo:";
+  };
+
   # This defines the build pipeline, using the pipeline format
   # documented on https://buildkite.com/docs/pipelines/defining-steps
-  pipeline.steps = [
-    {
-      command = "nix-build -A ci.targets --show-trace";
-      label = ":duck:";
-    }
-    {
-      command = "${depot.nix.bufCheck}/bin/ci-buf-check";
-      label = ":water_buffalo:";
-    }
-  ];
-in writeText "depot.yaml" (toJSON pipeline)
+  #
+  # Pipeline steps need to stay in order.
+  pipeline.steps =
+    # Zero the failure status
+    [
+      {
+        command = "buildkite-agent meta-data set 'failure' '0'";
+        label = ":buildkite:";
+      }
+      { wait = null; }
+    ]
+
+    # Create build steps for each CI target
+    ++ (map mkStep depot.ci.targets)
+
+    ++ [
+      # Simultaneously run protobuf checks
+      protoCheck
+
+      # Wait for all previous checks to complete
+      ({
+        wait = null;
+        continue_on_failure = true;
+      })
+
+      # Wait for all steps to complete, then exit with success or
+      # failure depending on whether any failure status was written.
+      # This step must be :duck:! (yes, really!)
+      ({
+        command = "exit $(buildkite-agent meta-data get 'failure')";
+        label = ":duck:";
+      })
+    ];
+in (writeText "depot.yaml" (toJSON pipeline))