about summary refs log tree commit diff
diff options
context:
space:
mode:
authorVincent Ambo <mail@tazj.in>2021-04-11T10·19+0200
committertazjin <mail@tazj.in>2021-04-11T20·09+0000
commitd7b89df748fd9de8015602c1a7d9d36cddd96664 (patch)
tree570db67655dda8b0971b7ba041eee4bc6bf9545b
parent6c3585f76425687324275dbbff6098886479c6b6 (diff)
feat(ops/pipelines): Add gcroots for depot builds on canon r/2477
Adds a conditional build step that only runs on the canon branch, and
only if :duck: (the status reporting step) succeeds, which creates a
new Nix GC root for all depot targets named `depot-canon`.

In practice this might be a bit racey, as canon builds are not
guaranteed to succeed in order (though it is likely). This shouldn't
matter much in practice: We only want to prevent rebuilds of the whole
world.

This fixes b/102

Change-Id: Id3d0bf4158bffcb1ed6929888a29d31609b6ece1
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2904
Tested-by: BuildkiteCI
Reviewed-by: glittershark <grfn@gws.fyi>
-rw-r--r--default.nix6
-rw-r--r--ops/pipelines/depot.nix24
2 files changed, 28 insertions, 2 deletions
diff --git a/default.nix b/default.nix
index cc2ebcabcfba..c5db0d1e321a 100644
--- a/default.nix
+++ b/default.nix
@@ -105,6 +105,12 @@ in fix(self: {
     # remove nixpkgs from the set, for obvious reasons.
     third_party = self.third_party // { nixpkgs = null; };
   });
+
+  # Derivation that gcroots all depot targets.
+  ci.gcroot = self.third_party.nixpkgs.symlinkJoin {
+    name = "depot-gcroot";
+    paths = self.ci.targets;
+  };
 }
 
 # Add local packages as structured by readTree
diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix
index ec7fb813278b..8c03217c1e3b 100644
--- a/ops/pipelines/depot.nix
+++ b/ops/pipelines/depot.nix
@@ -8,8 +8,7 @@
 
 let
   inherit (builtins) concatStringsSep foldl' map toJSON;
-  inherit (lib) singleton;
-  inherit (pkgs) writeText;
+  inherit (pkgs) symlinkJoin writeText;
 
   # Create an expression that builds the target at the specified
   # location.
@@ -80,6 +79,27 @@ let
       ({
         command = "exit $(buildkite-agent meta-data get 'failure')";
         label = ":duck:";
+        key = ":duck:";
+      })
+
+      # After duck, on success, create a gcroot if the build branch is
+      # canon.
+      #
+      # We care that this anchors *most* of the depot, in practice
+      # it's unimportant if there is a build race and we get +-1 of
+      # the targets.
+      #
+      # Unfortunately this requires a third evaluation of the graph,
+      # but since it happens after :duck: it should not affect the
+      # timing of status reporting back to Gerrit.
+      ({
+        command = "nix-instantiate -A ci.gcroot --add-root /nix/var/nix/gcroots/depot/canon";
+        label = ":anchor:";
+        "if" = ''build.branch == "canon"'';
+        depends_on = [{
+          step = ":duck:";
+          allow_failure = false;
+        }];
       })
     ];
 in (writeText "depot.yaml" (toJSON pipeline))