feat(ops/pipelines): Dynamically generate CI pipeline from targets r/1747

Create the pipeline by outputting a file that contains nix-build
invocations for each target's *derivation path*.

Each invocation has a generated Nix expression passed to it with `-E`
which fetches the correct target from the tree while correctly
handling targets with strange characters (such as in Go-packages).

This makes it possible to run target-level granular pipelines. We're
getting somewhere!

Change-Id: Ia6946e389dafd1d4926130bb8891446d6e17133b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1855
Tested-by: BuildkiteCI
Reviewed-by: glittershark <grfn@gws.fyi>
Reviewed-by: lukegb <lukegb@tvl.fyi>

6 files changed, 78 insertions, 20 deletions

diff --git a/default.nix b/default.nix
index b0b5399c4bc2..151d8987ea3d 100644
--- a/default.nix
+++ b/default.nix
@@ -73,9 +73,14 @@ in fix(self: {
 
   # List of all buildable targets, for CI purposes.
   #
-  # Note: This *must* be a nested attribute, otherwise we will get
-  # infinite recursion and everything blows up.
-  ci.targets = gather self;
+  # Note: To prevent infinite recursion, this *must* be a nested
+  # attribute set (which does not have a __readTree attribute).
+  ci.targets = gather (self // {
+    # remove the pipelines themselves from the set over which to
+    # generate pipelines because that also leads to infinite
+    # recursion.
+    ops = self.ops // { pipelines = null; };
+  });
 }
 
 # Add local packages as structured by readTree
diff --git a/ops/nixos/default.nix b/ops/nixos/default.nix
index 407e667f42ba..917a56b766da 100644
--- a/ops/nixos/default.nix
+++ b/ops/nixos/default.nix
@@ -47,5 +47,5 @@ rec {
   #
   # TODO(tazjin): Refactor the whole systems setup, it's a bit
   # inconsistent at the moment.
-  whitbySystem = (nixosFor whitby).system // { __readTree = true; };
+  whitbySystem = (nixosFor whitby).system;
 }
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 2e99bf8d1393..5d1f2babe123 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -4,22 +4,75 @@
 # It outputs a "YAML" (actually JSON) file which is evaluated and
 # submitted to Buildkite at the start of each build. This means we can
 # dynamically configure the pipeline execution here.
-{ depot, pkgs, ... }:
+{ depot, lib, pkgs, ... }:
 
 let
-  inherit (builtins) toJSON;
+  inherit (builtins) concatStringsSep foldl' map toJSON;
+  inherit (lib) singleton;
   inherit (pkgs) writeText;
 
+  # Create an expression that builds the target at the specified
+  # location.
+  mkBuildExpr =
+    let descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
+    in foldl' descend "import ./. {}";
+
+  # Create a pipeline label from the targets tree location.
+  mkLabel = concatStringsSep "/";
+
+  # Create a pipeline step from a single target.
+  #
+  # If the build fails, Buildkite metadata is updated to mark the
+  # pipeline as failed. Buildkite has a concept of a failed pipeline
+  # regardless, but this data is not accessible.
+  mkStep = target: {
+    command = ''
+      nix-build -E '${mkBuildExpr target.__readTree}' || (buildkite-agent meta-data set "failure" "1"; exit 1)
+    '';
+    label = ":nix: ${mkLabel target.__readTree}";
+  };
+
+  # Protobuf check step which validates that changes to .proto files
+  # between revisions don't cause backwards-incompatible or otherwise
+  # flawed changes.
+  protoCheck = {
+    command = "${depot.nix.bufCheck}/bin/ci-buf-check";
+    label = ":water_buffalo:";
+  };
+
   # This defines the build pipeline, using the pipeline format
   # documented on https://buildkite.com/docs/pipelines/defining-steps
-  pipeline.steps = [
-    {
-      command = "nix-build -A ci.targets --show-trace";
-      label = ":duck:";
-    }
-    {
-      command = "${depot.nix.bufCheck}/bin/ci-buf-check";
-      label = ":water_buffalo:";
-    }
-  ];
-in writeText "depot.yaml" (toJSON pipeline)
+  #
+  # Pipeline steps need to stay in order.
+  pipeline.steps =
+    # Zero the failure status
+    [
+      {
+        command = "buildkite-agent meta-data set 'failure' '0'";
+        label = ":buildkite:";
+      }
+      { wait = null; }
+    ]
+
+    # Create build steps for each CI target
+    ++ (map mkStep depot.ci.targets)
+
+    ++ [
+      # Simultaneously run protobuf checks
+      protoCheck
+
+      # Wait for all previous checks to complete
+      ({
+        wait = null;
+        continue_on_failure = true;
+      })
+
+      # Wait for all steps to complete, then exit with success or
+      # failure depending on whether any failure status was written.
+      # This step must be :duck:! (yes, really!)
+      ({
+        command = "exit $(buildkite-agent meta-data get 'failure')";
+        label = ":duck:";
+      })
+    ];
+in (writeText "depot.yaml" (toJSON pipeline))
diff --git a/users/glittershark/system/home/default.nix b/users/glittershark/system/home/default.nix
index d896ba340d59..a10e3f8dfc30 100644
--- a/users/glittershark/system/home/default.nix
+++ b/users/glittershark/system/home/default.nix
@@ -20,7 +20,7 @@ rec {
 
       lib.depot = depot;
     };
-  }) // { __readTree = true; };
+  });
 
   chupacabra = home ./machines/chupacabra.nix;
 }
diff --git a/users/glittershark/system/system/default.nix b/users/glittershark/system/system/default.nix
index f6710eff7b1b..e1946810540f 100644
--- a/users/glittershark/system/system/default.nix
+++ b/users/glittershark/system/system/default.nix
@@ -5,7 +5,7 @@ rec {
 
   chupacabraSystem = (pkgs.nixos {
     configuration = chupacabra;
-  }).system // { __readTree = true; };
+  }).system;
 
   rebuilder =
     let
diff --git a/users/tazjin/nixos/default.nix b/users/tazjin/nixos/default.nix
index 11f2eef13043..9747f5c00cc7 100644
--- a/users/tazjin/nixos/default.nix
+++ b/users/tazjin/nixos/default.nix
@@ -8,7 +8,7 @@ let
     configuration = lib.fix(config:
       foldl' lib.recursiveUpdate {} (map (c: c config) configs)
     );
-  }).system // { __readTree = true; };
+  }).system;
 
   caseFor = hostname: ''
     ${hostname})