feat(ops/pipelines): Fetch parent target map for pipeline generation r/3603

Change-Id: I1c7d48fc0974549d67146a15f79ddb0b6ddfe805
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4947
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>

2 files changed, 51 insertions, 1 deletions

diff --git a/ops/pipelines/fetch-parent-targets.sh b/ops/pipelines/fetch-parent-targets.sh
new file mode 100755
index 0000000000..bc334fc35c
--- /dev/null
+++ b/ops/pipelines/fetch-parent-targets.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+
+# Each Buildkite build stores the derivation target map as a pipeline
+# artifact. This script determines the most appropriate commit (the
+# fork point of the current chain from canon) and fetches the
+# artifact.
+#
+# Since builds can be based on canon before the pipeline for the last
+# commit has finished, it is possible that the fork point has no
+# target map. To account for this, we will go up to 3 commits back in
+# time to find a map.
+#
+# If no map is found, the failure mode is not critical: We simply
+# build all targets.
+
+function most_relevant_builds {
+    git fetch -v origin canon
+    local FIRST=$(git merge-base --fork-point HEAD origin/canon)
+    local SECOND=$(git rev-parse "$FIRST~1")
+    local THIRD=$(git rev-parse "$FIRST~2")
+
+    curl 'https://graphql.buildkite.com/v1' \
+         --silent \
+         -H "Authorization: Bearer $(cat /run/agenix/buildkite-graphql-token)" \
+         -d "{\"query\": \"query { pipeline(slug: \\\"tvl/depot\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \
+         jq -r '.data.pipeline.builds.edges[] | .node.uuid'
+}
+
+mkdir -p tmp
+for build in $(most_relevant_builds); do
+    echo "Checking artifacts for build $build"
+    buildkite-agent artifact download --build "${build}" 'pipeline/drvmap.json' 'tmp/' || true
+
+    if [[ -f "tmp/pipeline/drvmap.json" ]]; then
+        echo "Fetched target map from build ${build}"
+        mv tmp/pipeline/drvmap.json parent-target-map.json
+        break
+    fi
+done
diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml
index 7fcb716b25..52f9f82039 100644
--- a/ops/pipelines/static-pipeline.yaml
+++ b/ops/pipelines/static-pipeline.yaml
@@ -15,7 +15,17 @@ steps:
           buildkite-agent annotate
       fi
 
-      nix-build -A ops.pipelines.depot -o pipeline --show-trace
+      # Attempt to fetch a target map from a parent commit on canon,
+      # except on builds of canon itself.
+      [ "${BUILDKITE_BRANCH}" != "refs/heads/canon" ] && \
+        ops/pipelines/fetch-parent-targets.sh
+
+      PIPELINE_ARGS=""
+      if [[ -f ./parent-target-map.json ]]; then
+        PIPELINE_ARGS="--arg parentTargetMap ./parent-target-map.json"
+      fi
+
+      nix-build -A ops.pipelines.depot -o pipeline --show-trace $$PIPELINE_ARGS
 
       # Steps need to be uploaded in reverse order because pipeline
       # upload prepends instead of appending.