diff options
| -rw-r--r-- | nix/buildkite/default.nix | 26 | ||||
| -rw-r--r-- | ops/pipelines/depot.nix | 21 | ||||
| -rw-r--r-- | ops/pipelines/static-pipeline.yaml | 3 |
3 files changed, 40 insertions, 10 deletions
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix index 09fe0a741e41..5a61aef3b779 100644 --- a/nix/buildkite/default.nix +++ b/nix/buildkite/default.nix @@ -15,9 +15,13 @@ let concatStringsSep filter foldl' + getEnv length + listToAttrs mapAttrs - toJSON; + pathExists + toJSON + unsafeDiscardStringContext; inherit (pkgs) lib runCommandNoCC writeText; in rec { @@ -44,7 +48,7 @@ in rec { # Skip build steps if their out path has already been built. skip = headBranch: target: let - shouldSkip = with builtins; + shouldSkip = # Only skip in real Buildkite builds (getEnv "BUILDKITE_BUILD_ID" != "") && # Always build everything for the canon branch. @@ -60,7 +64,7 @@ in rec { skip = if skipIfBuilt then skip headBranch target else false; command = let - drvPath = builtins.unsafeDiscardStringContext target.drvPath; + drvPath = unsafeDiscardStringContext target.drvPath; in concatStringsSep " " [ # First try to realise the drvPath of the target so we don't evaluate twice. # Nix has no concept of depending on a derivation file without depending on @@ -162,4 +166,20 @@ in rec { (chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks } ''; + + # Create a drvmap structure for the given targets, containing the + # mapping of all target paths to their derivations. The mapping can + # be persisted for future use. + mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: { + name = mkLabel target; + value = { + drvPath = unsafeDiscardStringContext target.drvPath; + + # Include the attrPath in the output to reconstruct the drv + # without parsing the human-readable label. + attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [ + target.__subtarget + ]; + }; + }) drvTargets))); } diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix index 9f7aaf8ba01f..8cc4b5691f59 100644 --- a/ops/pipelines/depot.nix +++ b/ops/pipelines/depot.nix @@ -1,6 +1,6 @@ # This file configures the primary build pipeline used for the # top-level list of depot targets. -{ depot, ... }: +{ depot, pkgs, ... }: let # Protobuf check step which validates that changes to .proto files @@ -17,9 +17,16 @@ let command = "${depot.tools.depotfmt.check}"; label = ":evergreen_tree: (tools/depotfmt)"; }; -in depot.nix.buildkite.mkPipeline { - headBranch = "refs/heads/canon"; - drvTargets = depot.ci.targets; - skipIfBuilt = true; - additionalSteps = [ depotfmtCheck protoCheck ]; -} + pipeline = depot.nix.buildkite.mkPipeline { + headBranch = "refs/heads/canon"; + drvTargets = depot.ci.targets; + skipIfBuilt = true; + additionalSteps = [ depotfmtCheck protoCheck ]; + }; + + drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets; +in pkgs.runCommandNoCC "depot-pipeline" {} '' + mkdir $out + cp -r ${pipeline}/* $out + cp ${drvmap} $out/drvmap.json +'' diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml index 78b53d692bb7..7fcb716b25f8 100644 --- a/ops/pipelines/static-pipeline.yaml +++ b/ops/pipelines/static-pipeline.yaml @@ -6,6 +6,7 @@ --- steps: - label: ":llama:" + key: "pipeline-gen" command: | set -ue @@ -22,6 +23,8 @@ steps: buildkite-agent pipeline upload $$chunk done + buildkite-agent artifact upload pipeline/drvmap.json + # Wait for all previous steps to complete. - wait: null continue_on_failure: true |