From cc903bc3b045486af514b03d61102eea4db69bd4 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <mail@tazj.in>
Date: Wed, 14 Apr 2021 18:16:42 +0200
Subject: feat(ops/modules): Add module for automatically collecting garbage

Adds a module that automatically collects garbage based on disk space
thresholds, and configures it to run hourly on whitby.

This is implemented as an alternative to cl/2937, which I've been told
uses a Nix feature that doesn't actually work.

Under-the-hood this is simply a systemd timer running a shell script
which checks available disk space and runs GC when necessary.

Change-Id: I3c6b5de85b74ea52e7e16c53f2f900e0911c9805
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3014
Tested-by: BuildkiteCI
Reviewed-by: lukegb <lukegb@tvl.fyi>
---
 ops/machines/whitby/default.nix | 10 +++++
 ops/modules/automatic-gc.nix    | 90 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 ops/modules/automatic-gc.nix

diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix
index b2d3eca446..56cf2beab5 100644
--- a/ops/machines/whitby/default.nix
+++ b/ops/machines/whitby/default.nix
@@ -6,6 +6,7 @@ let
   inherit (lib) range;
 in {
   imports = [
+    "${depot.path}/ops/modules/automatic-gc.nix"
     "${depot.path}/ops/modules/clbot.nix"
     "${depot.path}/ops/modules/irccat.nix"
     "${depot.path}/ops/modules/monorepo-gerrit.nix"
@@ -187,6 +188,15 @@ in {
     challengeResponseAuthentication = false;
   };
 
+  # Automatically collect garbage from the Nix store.
+  services.depot.automatic-gc = {
+    enable = true;
+    interval = "1 hour";
+    diskThreshold = 200; # GiB
+    maxFreed = 420; # GiB
+    preserveGenerations = "90d";
+  };
+
   # Run a handful of Buildkite agents to support parallel builds.
   services.depot.buildkite = {
     enable = true;
diff --git a/ops/modules/automatic-gc.nix b/ops/modules/automatic-gc.nix
new file mode 100644
index 0000000000..6904afc673
--- /dev/null
+++ b/ops/modules/automatic-gc.nix
@@ -0,0 +1,90 @@
+# Defines a service for automatically collecting Nix garbage
+# periodically, without relying on the (ostensibly broken) Nix options
+# for min/max space available.
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.depot.automatic-gc;
+  description = "Automatically collect Nix garbage";
+
+  GiBtoKiB = n: n * 1024 * 1024;
+  GiBtoBytes = n: n * 1024 * 1024 * 1024;
+
+  gcScript = pkgs.writeShellScript "automatic-nix-gc" ''
+    set -ueo pipefail
+
+    readonly MIN_THRESHOLD_KIB="${toString (GiBtoKiB cfg.diskThreshold)}"
+    readonly MAX_FREED_BYTES="${toString (GiBtoBytes cfg.maxFreed)}"
+    readonly GEN_THRESHOLD="${cfg.preserveGenerations}"
+    readonly AVAILABLE_KIB=$(df --sync /nix --output=avail | tail -n1)
+
+    if [ "''${AVAILABLE_KIB}" -lt "''${MIN_THRESHOLD_KIB}" ]; then
+      echo "Have ''${AVAILABLE_KIB} KiB, but want ''${MIN_THRESHOLD_KIB} KiB."
+      echo "Triggering Nix garbage collection up to ''${MAX_FREED_BYTES} bytes."
+      set -x
+      nix-collect-garbage \
+        --delete-older-than "''${GEN_THRESHOLD}" \
+        --max-freed "''${MAX_FREED_BYTES}"
+    else
+      echo "Skipping GC, enough space available"
+    fi
+  '';
+in {
+  options.services.depot.automatic-gc = {
+    enable = lib.mkEnableOption description;
+
+    interval = lib.mkOption {
+      type = lib.types.str;
+      example = "1h";
+      description = ''
+        Interval between garbage collection runs, specified in
+        systemd.time(7) format.
+      '';
+    };
+
+    diskThreshold = lib.mkOption {
+      type = lib.types.int;
+      example = "100";
+      description = ''
+        Minimum amount of space that needs to be available (in GiB) on
+        the partition holding /nix. Garbage collection is triggered if
+        it falls below this.
+      '';
+    };
+
+    maxFreed = lib.mkOption {
+      type = lib.types.int;
+      example = "420";
+      description = ''
+        Maximum amount of space to free in a single GC run, in GiB.
+      '';
+    };
+
+    preserveGenerations = lib.mkOption {
+      type = lib.types.str;
+      default = "90d";
+      description = ''
+        Preserve NixOS generations younger than the specified value,
+        in the format expected by nix-collect-garbage(1).
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.automatic-gc = {
+      inherit description;
+      script = "${gcScript}";
+      serviceConfig.Type = "oneshot";
+    };
+
+    systemd.timers.automatic-gc = {
+      inherit description;
+      wantedBy = [ "multi-user.target" ];
+
+      timerConfig = {
+        OnActiveSec = "1";
+        OnUnitActiveSec = cfg.interval;
+      };
+    };
+  };
+}
-- 
cgit 1.4.1