From 1640d9d145b6e8d058b33e4e2bd9c5b0780b8f2e Mon Sep 17 00:00:00 2001
From: Vincent Ambo <mail@tazj.in>
Date: Wed, 24 Jun 2020 03:08:53 +0100
Subject: refactor(ci-builds): Split up CI builds into multiple buckets

These categories separate CI targets, which hopefully avoids the
out-of-space errors we have been seeing on Sourcehut.

The sets of CI build targets are made available in the depot itself so
that besadii can be updated to create a new build for each target
group.

For convenience, 'ciBuilds' contains an '__allTargets' attribute which
combines the contents of each target batch - this makes it possible to
still invoke a build for everything by using:

  nix-build -A ciBuilds.__allTargets

Note: Some targets that were previously built in CI aren't anymore,
most importantly my NixOS systems which don't fit on Sourcehut.

Change-Id: Ia15ed7b743c8add51ae08ce0827a0ddfacd637e2
Reviewed-on: https://cl.tvl.fyi/c/depot/+/570
Reviewed-by: lukegb <lukegb@tvl.fyi>
---
 ci-builds.nix | 101 +++++++++++++++++++++++++++++++++++++++-------------------
 default.nix   |   8 ++++-
 2 files changed, 75 insertions(+), 34 deletions(-)

diff --git a/ci-builds.nix b/ci-builds.nix
index 1f4f96765687..02af7a28b4be 100644
--- a/ci-builds.nix
+++ b/ci-builds.nix
@@ -1,35 +1,70 @@
 # This file defines the derivations that should be built by CI.
 #
-# The plan is still to implement recursive tree traversal
-# automatically and detect all derivations that have `meta.enableCI =
-# true`, but this is currently more effort than it would save me.
-
-with (import ./default.nix {}); [
-  fun.amsterdump
-  fun.gemma
-  fun.quinistry
-  fun.watchblob
-  fun.wcl
-  lisp.dns
-  nix.buildLisp.example
-  nix.yants.tests
-  ops."posix_mq.rs"
-  ops.besadii
-  ops.journaldriver
-  ops.kms_pass
-  ops.kontemplate
-  ops.mq_cli
-  third_party.cgit
-  third_party.git
-  third_party.lisp # will build all third-party libraries
-  third_party.nix
-  tools.cheddar
-  web.blog
-  web.cgit-taz
-  web.tvl
-
-  # tazjin's personal things
-  users.tazjin.emacs
-  users.tazjin.nixos.camdenSystem
-  users.tazjin.nixos.frogSystem
-]
+# The "categories" (i.e. attributes) below exist because we run out of
+# space on Sourcehut otherwise.
+{ depot, lib, ... }:
+
+let
+  inherit (builtins) attrNames filter foldl' getAttr substring;
+
+in lib.fix(self: {
+  __apprehendEvaluators = throw ''
+    Do not evaluate this attribute set directly. It exists only to group builds
+    for CI runs of different "project groups".
+
+    To use the depot, always start from the top-level attribute tree instead.
+  '';
+
+  # Names of all evaluatable attributes in here. This list will be
+  # used to trigger builds for each key.
+  __evaluatable = filter (key: (substring 0 2 key) != "__") (attrNames self);
+
+  # List of non-public targets, these are only used in local builds
+  # and not in CI.
+  __nonpublic = with depot; [
+    users.tazjin.emacs
+    users.tazjin.nixos.camdenSystem
+    users.tazjin.nixos.frogSystem
+  ];
+
+  # Combined list of all the targets, used for building everything locally.
+  __allTargets = foldl' (x: y: x ++ y) self.__nonpublic
+    (map (k: getAttr k self) self.__evaluatable);
+
+  fun = with depot.fun; [
+    amsterdump
+    gemma
+    quinistry
+    watchblob
+    wcl
+  ];
+
+  gitAndFriends = with depot; [
+    third_party.cgit
+    third_party.git
+    web.cgit-taz
+  ];
+
+  nix = [ depot.third_party.nix ];
+
+  ops = with depot.ops; [
+    depot.ops."posix_mq.rs"
+    besadii
+    journaldriver
+    kms_pass
+    kontemplate
+    mq_cli
+  ];
+
+  various = with depot; [
+    tools.cheddar
+    lisp.dns
+    nix.buildLisp.example
+    nix.yants.tests
+  ];
+
+  web = with depot.web; [
+    blog
+    tvl
+  ];
+})
diff --git a/default.nix b/default.nix
index e34734dc9e33..8c7a35f995bb 100644
--- a/default.nix
+++ b/default.nix
@@ -18,6 +18,9 @@ let
   config = depot: {
     inherit depot;
 
+    # Expose lib & ciBuilds attributes to packages.
+    inherit (depot) ciBuilds lib;
+
     # Pass third_party as 'pkgs' (for compatibility with external
     # imports for certain subdirectories)
     pkgs = depot.third_party;
@@ -59,10 +62,13 @@ in fix(self: {
   # Make the path to the depot available for things that might need it
   # (e.g. NixOS module inclusions)
   depotPath = ./.;
+
+  # Load CI builds in a way that can be injected into programs like besadii.
+  ciBuilds = import ./ci-builds.nix self.config;
 }
 
 # Add local packages as structured by readTree
-// (localPkgs (readTree' (self.config // { inherit (self) lib; })))
+// (localPkgs (readTree' self.config))
 
 # Load overrides into the top-level.
 #
-- 
cgit 1.4.1