about summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--default.nix4
-rw-r--r--nix/buildkite/default.nix44
-rw-r--r--ops/pipelines/depot.nix8
3 files changed, 31 insertions, 25 deletions
diff --git a/default.nix b/default.nix
index 1cf62e94afb0..db2c5035599c 100644
--- a/default.nix
+++ b/default.nix
@@ -2,7 +2,9 @@
 # (see //nix/readTree for details) and constructing a matching attribute set
 # tree.
 
-{ nixpkgsBisectPath ? null, nixpkgsConfig ? {}, ... }@args:
+{ nixpkgsBisectPath ? null
+, parentTargetMap ? null
+, nixpkgsConfig ? {}, ... }@args:
 
 let
   inherit (builtins)
diff --git a/nix/buildkite/default.nix b/nix/buildkite/default.nix
index 5a61aef3b779..c34861123c0d 100644
--- a/nix/buildkite/default.nix
+++ b/nix/buildkite/default.nix
@@ -16,6 +16,7 @@ let
     filter
     foldl'
     getEnv
+    hasAttr
     length
     listToAttrs
     mapAttrs
@@ -46,26 +47,24 @@ in rec {
       then "${label}:${target.__subtarget}"
       else label;
 
-  # Skip build steps if their out path has already been built.
-  skip = headBranch: target: let
-    shouldSkip =
-      # Only skip in real Buildkite builds
-      (getEnv "BUILDKITE_BUILD_ID" != "") &&
-      # Always build everything for the canon branch.
-      (getEnv "BUILDKITE_BRANCH" != headBranch) &&
-      # Discard string context to avoid realising the store path during
-      # pipeline construction.
-      (pathExists (unsafeDiscardStringContext target.outPath));
-    in if shouldSkip then "Target was already built." else false;
+  # Determine whether to skip a target if it has not diverged from the
+  # HEAD branch.
+  shouldSkip = parentTargetMap: label: drvPath:
+    if (hasAttr label parentTargetMap) && parentTargetMap."${label}".drvPath == drvPath
+    then "Target has not changed."
+    else false;
 
   # Create a pipeline step from a single target.
-  mkStep = headBranch: skipIfBuilt: target: {
-    label = ":nix: ${mkLabel target}";
-    skip = if skipIfBuilt then skip headBranch target else false;
-
-    command = let
-      drvPath = unsafeDiscardStringContext target.drvPath;
-    in concatStringsSep " " [
+  mkStep = headBranch: parentTargetMap: target:
+  let
+    label = mkLabel target;
+    drvPath = unsafeDiscardStringContext target.drvPath;
+    shouldSkip' = shouldSkip parentTargetMap;
+  in {
+    label = ":nix: " + label;
+    skip = shouldSkip' label drvPath;
+
+    command = concatStringsSep " " [
       # First try to realise the drvPath of the target so we don't evaluate twice.
       # Nix has no concept of depending on a derivation file without depending on
       # at least one of its `outPath`s, so we need to discard the string context
@@ -124,9 +123,10 @@ in rec {
     # possible, in order, without any concurrency restrictions.
     drvTargets,
 
-    # Should build steps be skipped (on non-HEAD builds) if the output
-    # path has already been built?
-    skipIfBuilt ? false,
+    # Derivation map of a parent commit. Only targets which no longer
+    # correspond to the content of this map will be built. Passing an
+    # empty map will always build all targets.
+    parentTargetMap ? {},
 
     # A list of plain Buildkite step structures to run alongside the
     # build for all drvTargets, but before proceeding with any
@@ -141,7 +141,7 @@ in rec {
     # Can be used for status reporting steps and the like.
     postBuildSteps ? []
   }: let
-    mkStep' = mkStep headBranch skipIfBuilt;
+    mkStep' = mkStep headBranch parentTargetMap;
     steps =
       # Add build steps for each derivation target.
       (map mkStep' drvTargets)
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index 8cc4b5691f59..232d229b90d8 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -1,6 +1,6 @@
 # This file configures the primary build pipeline used for the
 # top-level list of depot targets.
-{ depot, pkgs, ... }:
+{ depot, pkgs, externalArgs, ... }:
 
 let
   # Protobuf check step which validates that changes to .proto files
@@ -17,11 +17,15 @@ let
     command = "${depot.tools.depotfmt.check}";
     label = ":evergreen_tree: (tools/depotfmt)";
   };
+
   pipeline = depot.nix.buildkite.mkPipeline {
     headBranch = "refs/heads/canon";
     drvTargets = depot.ci.targets;
-    skipIfBuilt = true;
     additionalSteps = [ depotfmtCheck protoCheck ];
+
+    parentTargetMap = if (externalArgs ? parentTargetMap)
+      then builtins.fromJSON (builtins.readFile externalArgs.parentTargetMap)
+      else {};
   };
 
   drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets;