feat(ops/pipelines): Create drvmap structure for each commit r/3601

Always create a structure that maps all targets to derivations, and
persist it as a JSON file.

This relates to some of the ideas expressed in:

https://docs.google.com/document/d/16A0a5oUxH1VoiSM8hyFyLW0WiUYpNo2e2D6FTW4BlH8/edit

The file is always uploaded to Buildkite as an artifact. This allows
for retrieving it based on the commit ID in a Buildkite GraphQL query.

By default, Buildkite stores artefacts for 6 months. Storage location
can be overridden (with custom retention) through some environment
variables, but for now at TVL the Buildkite-managed storage is fine.
See also: https://buildkite.com/docs/pipelines/artifacts

In the subsequent filtering implementation, when diffing commits
across a time-range that exceeds artefact retention time, we should
simply default to building everything.

Change-Id: I6d808461cd1c1fdd6983ba8c8ef075736d42caa7
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3662
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>

3 files changed, 40 insertions, 10 deletions

diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
index 09fe0a741e41..5a61aef3b779 100644
--- a/nix/buildkite/default.nix
+++ b/nix/buildkite/default.nix
@@ -15,9 +15,13 @@ let
     concatStringsSep
     filter
     foldl'
+    getEnv
     length
+    listToAttrs
     mapAttrs
-    toJSON;
+    pathExists
+    toJSON
+    unsafeDiscardStringContext;
 
   inherit (pkgs) lib runCommandNoCC writeText;
 in rec {
@@ -44,7 +48,7 @@ in rec {
 
   # Skip build steps if their out path has already been built.
   skip = headBranch: target: let
-    shouldSkip = with builtins;
+    shouldSkip =
       # Only skip in real Buildkite builds
       (getEnv "BUILDKITE_BUILD_ID" != "") &&
       # Always build everything for the canon branch.
@@ -60,7 +64,7 @@ in rec {
     skip = if skipIfBuilt then skip headBranch target else false;
 
     command = let
-      drvPath = builtins.unsafeDiscardStringContext target.drvPath;
+      drvPath = unsafeDiscardStringContext target.drvPath;
     in concatStringsSep " " [
       # First try to realise the drvPath of the target so we don't evaluate twice.
       # Nix has no concept of depending on a derivation file without depending on
@@ -162,4 +166,20 @@ in rec {
         (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
     }
   '';
+
+  # Create a drvmap structure for the given targets, containing the
+  # mapping of all target paths to their derivations. The mapping can
+  # be persisted for future use.
+  mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: {
+    name = mkLabel target;
+    value = {
+      drvPath = unsafeDiscardStringContext target.drvPath;
+
+      # Include the attrPath in the output to reconstruct the drv
+      # without parsing the human-readable label.
+      attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
+        target.__subtarget
+      ];
+    };
+  }) drvTargets)));
 }
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 9f7aaf8ba01f..8cc4b5691f59 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -1,6 +1,6 @@
 # This file configures the primary build pipeline used for the
 # top-level list of depot targets.
-{ depot, ... }:
+{ depot, pkgs, ... }:
 
 let
   # Protobuf check step which validates that changes to .proto files
@@ -17,9 +17,16 @@ let
     command = "${depot.tools.depotfmt.check}";
     label = ":evergreen_tree: (tools/depotfmt)";
   };
-in depot.nix.buildkite.mkPipeline {
-  headBranch = "refs/heads/canon";
-  drvTargets = depot.ci.targets;
-  skipIfBuilt = true;
-  additionalSteps = [ depotfmtCheck protoCheck ];
-}
+  pipeline = depot.nix.buildkite.mkPipeline {
+    headBranch = "refs/heads/canon";
+    drvTargets = depot.ci.targets;
+    skipIfBuilt = true;
+    additionalSteps = [ depotfmtCheck protoCheck ];
+  };
+
+  drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets;
+in pkgs.runCommandNoCC "depot-pipeline" {} ''
+  mkdir $out
+  cp -r ${pipeline}/* $out
+  cp ${drvmap} $out/drvmap.json
+''
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
index 78b53d692bb7..7fcb716b25f8 100644
--- a/ops/pipelines/static-pipeline.yaml
+++ b/ops/pipelines/static-pipeline.yaml
@@ -6,6 +6,7 @@
 ---
 steps:
   - label: ":llama:"
+    key: "pipeline-gen"
     command: |
       set -ue
 
@@ -22,6 +23,8 @@ steps:
         buildkite-agent pipeline upload $$chunk
       done
 
+      buildkite-agent artifact upload pipeline/drvmap.json
+
   # Wait for all previous steps to complete.
   - wait: null
     continue_on_failure: true