diff options
Diffstat (limited to 'ops/pipelines')
-rw-r--r-- | ops/pipelines/README.md | 5 | ||||
-rw-r--r-- | ops/pipelines/depot.nix | 85 | ||||
-rw-r--r-- | ops/pipelines/fallback.yaml | 8 | ||||
-rw-r--r-- | ops/pipelines/static-pipeline.yaml | 15 |
4 files changed, 113 insertions, 0 deletions
diff --git a/ops/pipelines/README.md b/ops/pipelines/README.md new file mode 100644 index 000000000000..a3f94fd23143 --- /dev/null +++ b/ops/pipelines/README.md @@ -0,0 +1,5 @@ +This folder contains the dynamic configuration for our [Buildkite CI +setup](https://tvl.fyi/builds). + +The configuration is built and dynamically loaded by Buildkite at the start of +each CI pipeline. diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix new file mode 100644 index 000000000000..ec7fb813278b --- /dev/null +++ b/ops/pipelines/depot.nix @@ -0,0 +1,85 @@ +# This file configures the primary build pipeline used for the +# top-level list of depot targets. +# +# It outputs a "YAML" (actually JSON) file which is evaluated and +# submitted to Buildkite at the start of each build. This means we can +# dynamically configure the pipeline execution here. +{ depot, lib, pkgs, ... }: + +let + inherit (builtins) concatStringsSep foldl' map toJSON; + inherit (lib) singleton; + inherit (pkgs) writeText; + + # Create an expression that builds the target at the specified + # location. + mkBuildExpr = target: + let + descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; + targetExpr = foldl' descend "import ./. {}" target.__readTree; + subtargetExpr = descend targetExpr target.__subtarget; + in if target ? __subtarget then subtargetExpr else targetExpr; + + # Create a pipeline label from the targets tree location. + mkLabel = target: + let label = concatStringsSep "/" target.__readTree; + in if target ? __subtarget + then "${label}:${target.__subtarget}" + else label; + + # Create a pipeline step from a single target. + # + # If the build fails, Buildkite metadata is updated to mark the + # pipeline as failed. Buildkite has a concept of a failed pipeline + # regardless, but this data is not accessible. + mkStep = target: { + command = '' + nix-build -E '${mkBuildExpr target}' || (buildkite-agent meta-data set "failure" "1"; exit 1) + ''; + label = ":nix: ${mkLabel target}"; + }; + + # Protobuf check step which validates that changes to .proto files + # between revisions don't cause backwards-incompatible or otherwise + # flawed changes. + protoCheck = { + command = "${depot.nix.bufCheck}/bin/ci-buf-check"; + label = ":water_buffalo:"; + }; + + # This defines the build pipeline, using the pipeline format + # documented on https://buildkite.com/docs/pipelines/defining-steps + # + # Pipeline steps need to stay in order. + pipeline.steps = + # Zero the failure status + [ + { + command = "buildkite-agent meta-data set 'failure' '0'"; + label = ":buildkite:"; + } + { wait = null; } + ] + + # Create build steps for each CI target + ++ (map mkStep depot.ci.targets) + + ++ [ + # Simultaneously run protobuf checks + protoCheck + + # Wait for all previous checks to complete + ({ + wait = null; + continue_on_failure = true; + }) + + # Wait for all steps to complete, then exit with success or + # failure depending on whether any failure status was written. + # This step must be :duck:! (yes, really!) + ({ + command = "exit $(buildkite-agent meta-data get 'failure')"; + label = ":duck:"; + }) + ]; +in (writeText "depot.yaml" (toJSON pipeline)) diff --git a/ops/pipelines/fallback.yaml b/ops/pipelines/fallback.yaml new file mode 100644 index 000000000000..73308d937b0c --- /dev/null +++ b/ops/pipelines/fallback.yaml @@ -0,0 +1,8 @@ +# This build configuration provides a fallback which marks a build as +# failed. This is used if evaluating the build configuration fails, +# for example because of a syntax error in Nix code. +--- +steps: + - command: "echo 'Nix evaluation failed!' && exit 1" + # This step *must* be :duck: to trigger the correct hook. + label: ":duck:" diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml new file mode 100644 index 000000000000..7a6e911ac252 --- /dev/null +++ b/ops/pipelines/static-pipeline.yaml @@ -0,0 +1,15 @@ +# This file defines the static pipeline which is uploaded in the +# Buildkite admin interface. These steps run at the beginning of each +# build and cause the dynamic pipeline generation to run. +--- +steps: + - label: ":llama:" + command: | + function fallback() { + echo 'Using fallback pipeline ...' + buildkite-agent pipeline upload ops/pipelines/fallback.yaml + exit + } + + nix-build -A ops.pipelines.depot -o depot.yaml --show-trace || fallback + buildkite-agent pipeline upload depot.yaml || fallback |