From 0652ac0ace548ba6ea4b6b78fc7bbfee277ec5d0 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Wed, 19 Jan 2022 18:01:41 +0300 Subject: refactor(nix/buildkite): Move fetch-parent-targets script here This is no longer TVL-specific and should live here with the other generalised stuff. Change-Id: I95a1b4c0321f34812162d6fd40568269abf639dd Reviewed-on: https://cl.tvl.fyi/c/depot/+/5006 Tested-by: BuildkiteCI Autosubmit: tazjin Reviewed-by: ezemtsov --- nix/buildkite/fetch-parent-targets.sh | 41 +++++++++++++++++++++++++++++++++++ ops/pipelines/fetch-parent-targets.sh | 41 ----------------------------------- ops/pipelines/static-pipeline.yaml | 2 +- 3 files changed, 42 insertions(+), 42 deletions(-) create mode 100755 nix/buildkite/fetch-parent-targets.sh delete mode 100755 ops/pipelines/fetch-parent-targets.sh diff --git a/nix/buildkite/fetch-parent-targets.sh b/nix/buildkite/fetch-parent-targets.sh new file mode 100755 index 0000000000..d34a826279 --- /dev/null +++ b/nix/buildkite/fetch-parent-targets.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -ueo pipefail + +# Each Buildkite build stores the derivation target map as a pipeline +# artifact. This script determines the most appropriate commit (the +# fork point of the current chain from HEAD) and fetches the artifact. +# +# New builds can be based on HEAD before the pipeline for the last +# commit has finished, in which case it is possible that the fork +# point has no derivation map. To account for this, up to 3 commits +# prior to HEAD are also queried to find a map. +# +# If no map is found, the failure mode is not critical: We simply +# build all targets. + +: ${DRVMAP_PATH:=pipeline/drvmap.json} + +function most_relevant_builds { + git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" + local FIRST=$(git merge-base --fork-point HEAD FETCH_HEAD) + local SECOND=$(git rev-parse "$FIRST~1") + local THIRD=$(git rev-parse "$FIRST~2") + + curl 'https://graphql.buildkite.com/v1' \ + --silent \ + -H "Authorization: Bearer $(cat /run/agenix/buildkite-graphql-token)" \ + -d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \ + jq -r '.data.pipeline.builds.edges[] | .node.uuid' +} + +mkdir -p tmp +for build in $(most_relevant_builds); do + echo "Checking artifacts for build $build" + buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true + + if [[ -f "tmp/${DRVMAP_PATH}" ]]; then + echo "Fetched target map from build ${build}" + mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json + break + fi +done diff --git a/ops/pipelines/fetch-parent-targets.sh b/ops/pipelines/fetch-parent-targets.sh deleted file mode 100755 index d34a826279..0000000000 --- a/ops/pipelines/fetch-parent-targets.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -ueo pipefail - -# Each Buildkite build stores the derivation target map as a pipeline -# artifact. This script determines the most appropriate commit (the -# fork point of the current chain from HEAD) and fetches the artifact. -# -# New builds can be based on HEAD before the pipeline for the last -# commit has finished, in which case it is possible that the fork -# point has no derivation map. To account for this, up to 3 commits -# prior to HEAD are also queried to find a map. -# -# If no map is found, the failure mode is not critical: We simply -# build all targets. - -: ${DRVMAP_PATH:=pipeline/drvmap.json} - -function most_relevant_builds { - git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" - local FIRST=$(git merge-base --fork-point HEAD FETCH_HEAD) - local SECOND=$(git rev-parse "$FIRST~1") - local THIRD=$(git rev-parse "$FIRST~2") - - curl 'https://graphql.buildkite.com/v1' \ - --silent \ - -H "Authorization: Bearer $(cat /run/agenix/buildkite-graphql-token)" \ - -d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \ - jq -r '.data.pipeline.builds.edges[] | .node.uuid' -} - -mkdir -p tmp -for build in $(most_relevant_builds); do - echo "Checking artifacts for build $build" - buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true - - if [[ -f "tmp/${DRVMAP_PATH}" ]]; then - echo "Fetched target map from build ${build}" - mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json - break - fi -done diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml index ffdb6a75c5..a4c3cf87ae 100644 --- a/ops/pipelines/static-pipeline.yaml +++ b/ops/pipelines/static-pipeline.yaml @@ -18,7 +18,7 @@ steps: # Attempt to fetch a target map from a parent commit on canon, # except on builds of canon itself. [ "${BUILDKITE_BRANCH}" != "refs/heads/canon" ] && \ - ops/pipelines/fetch-parent-targets.sh + nix/buildkite/fetch-parent-targets.sh PIPELINE_ARGS="" if [[ -f tmp/parent-target-map.json ]]; then -- cgit 1.4.1