1 files changed, 43 insertions, 6 deletions

diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
index b7715e902f..9abba9408a 100644
--- a/nix/buildkite/default.nix
+++ b/nix/buildkite/default.nix
@@ -27,8 +27,27 @@ let
 
   inherit (pkgs) lib runCommand writeText;
   inherit (depot.nix.readTree) mkLabel;
+
+  inherit (depot.nix) dependency-analyzer;
 in
 rec {
+  # Create a unique key for the buildkite pipeline based on the given derivation
+  # or drvPath. A consequence of using such keys is that every derivation may
+  # only be exposed as a single, unique step in the pipeline.
+  keyForDrv = drvOrPath:
+    let
+      drvPath =
+        if lib.isDerivation drvOrPath then drvOrPath.drvPath
+        else if lib.isString drvOrPath then drvOrPath
+        else builtins.throw "keyForDrv: expected string or derivation";
+
+      # Only use the drv hash to prevent escaping problems. Buildkite also has a
+      # limit of 100 characters on keys.
+    in
+    "drv-" + (builtins.substring 0 32
+      (builtins.baseNameOf (unsafeDiscardStringContext drvPath))
+    );
+
   # Given an arbitrary attribute path generate a Nix expression which obtains
   # this from the root of depot (assumed to be ./.). Attributes may be any
   # Nix strings suitable as attribute names, not just Nix literal-safe strings.
@@ -73,15 +92,28 @@ rec {
     target.__readTree
     ++ lib.optionals (target ? __subtarget) [ target.__subtarget ];
 
+  # Given a derivation (identified by drvPath) that is part of the list of
+  # targets passed to mkPipeline, determine all derivations that it depends on
+  # and are also part of the pipeline. Finally, return the keys of the steps
+  # that build them. This is used to populate `depends_on` in `mkStep`.
+  #
+  # See //nix/dependency-analyzer for documentation on the structure of `targetDepMap`.
+  getTargetPipelineDeps = targetDepMap: drvPath:
+    # Sanity check: We should only call this function on targets explicitly
+    # passed to mkPipeline. Thus it should have been passed as a “known” drv to
+    # dependency-analyzer.
+    assert targetDepMap.${drvPath}.known;
+    builtins.map keyForDrv targetDepMap.${drvPath}.knownDeps;
+
   # Create a pipeline step from a single target.
-  mkStep = { headBranch, parentTargetMap, target, cancelOnBuildFailing }:
+  mkStep = { headBranch, parentTargetMap, targetDepMap, target, cancelOnBuildFailing }:
     let
       label = mkLabel target;
       drvPath = unsafeDiscardStringContext target.drvPath;
     in
     {
       label = ":nix: " + label;
-      key = hashString "sha1" label;
+      key = keyForDrv target;
       skip = shouldSkip { inherit label drvPath parentTargetMap; };
       command = mkBuildCommand {
         attrPath = targetAttrPath target;
@@ -93,7 +125,9 @@ rec {
       # Add a dependency on the initial static pipeline step which
       # always runs. This allows build steps uploaded in batches to
       # start running before all batches have been uploaded.
-      depends_on = [ ":init:" ] ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps;
+      depends_on = [ ":init:" ]
+      ++ getTargetPipelineDeps targetDepMap drvPath
+      ++ lib.optionals (target ? meta.ci.buildkiteExtraDeps) target.meta.ci.buildkiteExtraDeps;
     } // lib.optionalAttrs (target ? meta.timeout) {
       timeout_in_minutes = target.meta.timeout / 60;
       # Additional arguments to set on the step.
@@ -196,12 +230,15 @@ rec {
       # logic/optimisation depends on knowing whether is executing.
       buildEnabled = elem "build" enabledPhases;
 
+      # Dependency relations between the `drvTargets`. See also //nix/dependency-analyzer.
+      targetDepMap = dependency-analyzer (dependency-analyzer.drvsToPaths drvTargets);
+
       # Convert a target into all of its steps, separated by build
       # phase (as phases end up in different chunks).
       targetToSteps = target:
         let
           mkStepArgs = {
-            inherit headBranch parentTargetMap target cancelOnBuildFailing;
+            inherit headBranch parentTargetMap targetDepMap target cancelOnBuildFailing;
           };
           step = mkStep mkStepArgs;
 
@@ -379,7 +416,7 @@ rec {
       prompt = lib.throwIf (prompt != false && phase == "build") ''
         In step '${label}' (from ${parentLabel}):
 
-        The 'prompt' feature can only be used by steps in the "release"
+        The 'prompt' feature can not be used by steps in the "build"
         phase, because CI builds should not be gated on manual human
         approvals.
       ''
@@ -394,7 +431,7 @@ rec {
       commandScriptLink = "nix-buildkite-extra-step-command-script";
 
       step = {
-        key = hashString "sha1" "${cfg.label}-${cfg.parentLabel}";
+        key = "extra-step-" + hashString "sha1" "${cfg.label}-${cfg.parentLabel}";
         label = ":gear: ${cfg.label} (from ${cfg.parentLabel})";
         skip =
           let