feat(nix/buildkite): fetch drvmap from *any* default branch builds r/6868

With this change, the query we make to Buildkite changes from
explicitly querying for a specific number of *ancestor* builds, to
any latest builds of the default branch that have a drvmap.

This is not really supported by Buildkite, it seems, and the query to
do it feels very wonky and requires a lot of fiddling with `jq` to get
the output into the right shape.

We lose the information about which build we downloaded this from in
the output. Adding that information back would make the `jq` query
much more complex.

Change-Id: I9e7cecdffa9ac09f9e0339eb24d98c0e8dd82292
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9168
Tested-by: BuildkiteCI
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>

1 files changed, 36 insertions, 25 deletions

diff --git a/nix/buildkite/fetch-parent-targets.sh b/nix/buildkite/fetch-parent-targets.sh
index 8afac1e5ec..08c2d1f3ab 100755
--- a/nix/buildkite/fetch-parent-targets.sh
+++ b/nix/buildkite/fetch-parent-targets.sh
@@ -2,43 +2,54 @@
 set -ueo pipefail
 
 # Each Buildkite build stores the derivation target map as a pipeline
-# artifact. This script determines the most appropriate commit (the
-# fork point of the current chain from HEAD) and fetches the artifact.
+# artifact. To reduce the amount of work done by CI, each CI build is
+# diffed against the latest such derivation map found for the
+# repository.
 #
-# New builds can be based on HEAD before the pipeline for the last
-# commit has finished, in which case it is possible that the fork
-# point has no derivation map. To account for this, up to 3 commits
-# prior to HEAD are also queried to find a map.
+# Note that this does not take into account when the currently
+# processing CL was forked off from the canonical branch, meaning that
+# things like nixpkgs updates in between will cause mass rebuilds in
+# any case.
 #
 # If no map is found, the failure mode is not critical: We simply
 # build all targets.
 
+readonly REPO_ROOT=$(git rev-parse --show-toplevel)
+
 : ${DRVMAP_PATH:=pipeline/drvmap.json}
 : ${BUILDKITE_TOKEN_PATH:=~/buildkite-token}
 
-git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}"
-
-FIRST=$(git merge-base FETCH_HEAD "${BUILDKITE_COMMIT}")
-SECOND=$(git rev-parse "$FIRST~1")
-THIRD=$(git rev-parse "$FIRST~2")
-
-function most_relevant_builds {
+# Runs a fairly complex Buildkite GraphQL query that attempts to fetch all
+# pipeline-gen steps from the default branch, as long as one appears within the
+# last 50 builds or so. The query restricts build states to running or passed
+# builds, which means that it *should* be unlikely that nothing is found.
+#
+# There is no way to filter this more loosely (e.g. by saying "any recent build
+# matching these conditions").
+#
+# The returned data structure is complex, and disassembled by a JQ script that
+# first filters out all builds with no matching jobs (e.g. builds that are still
+# in progress), and then filters those down to builds with artifacts, and then
+# to drvmap artifacts specifically.
+#
+# If a recent drvmap was found, this returns its download URL. Otherwise, it
+# returns the string "null".
+function latest_drvmap_url {
     set -u
     curl 'https://graphql.buildkite.com/v1' \
          --silent \
          -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \
-         -d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \
-         jq -r '.data.pipeline.builds.edges[] | .node.uuid'
+         -H "Content-Type: application/json" \
+         -d "{\"query\": \"{ pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(first: 50, branch: [\\\"%default\\\"], state: [RUNNING, PASSED]) { edges { node { jobs(passed: true, first: 1, type: [COMMAND], step: {key: [\\\"pipeline-gen\\\"]}) { edges { node { ... on JobTypeCommand { url artifacts { edges { node { downloadURL path }}}}}}}}}}}}\"}" | tee out.json | \
+        jq -r '[.data.pipeline.builds.edges[] | select((.node.jobs.edges | length) > 0) | .node.jobs.edges[] | .node.artifacts[][] | select(.node.path == "pipeline/drvmap.json")][0].node.downloadURL'
 }
 
-mkdir -p tmp
-for build in $(most_relevant_builds); do
-    echo "Checking artifacts for build $build"
-    buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true
+readonly DOWNLOAD_URL=$(latest_drvmap_url)
 
-    if [[ -f "tmp/${DRVMAP_PATH}" ]]; then
-        echo "Fetched target map from build ${build}"
-        mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json
-        break
-    fi
-done
+if [[ ${DOWNLOAD_URL} != "null" ]]; then
+    mkdir -p tmp
+    curl -o tmp/parent-target-map.json ${DOWNLOAD_URL} && echo "downloaded parent derivation map" \
+            || echo "failed to download derivation map!"
+else
+    echo "no derivation map found!"
+fi