chore(ops/whitby): retire most services on whitby r/9175

This turns off almost all of the lights. The server will be decomissioned on
2025-02-05. Until then we can keep running the Buildkite builders there for
extra capacity.

Stuff that was left in the whitby config has been migrated to nevsky.

This relates to b/433.

Change-Id: I84953e9d5e912f75b8884cb9d8edd5a1b7d5c85d
Reviewed-on: https://cl.tvl.fyi/c/depot/+/13095
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

3 files changed, 55 insertions, 367 deletions

diff --git a/ops/machines/nevsky/default.nix b/ops/machines/nevsky/default.nix
index 2ea75aba3e0d..7b18bd1ae3d7 100644
--- a/ops/machines/nevsky/default.nix
+++ b/ops/machines/nevsky/default.nix
@@ -6,10 +6,12 @@ let
 in
 {
   imports = [
+    (depot.third_party.agenix.src + "/modules/age.nix")
     (mod "builderball.nix")
     (mod "cgit.nix")
     (mod "cheddar.nix")
     (mod "clbot.nix")
+    (mod "gerrit-autosubmit.nix")
     (mod "harmonia.nix")
     (mod "irccat.nix")
     (mod "josh.nix")
@@ -22,12 +24,14 @@ in
     (mod "paroxysm.nix")
     (mod "restic.nix")
     (mod "smtprelay.nix")
+    (mod "teleirc.nix")
     (mod "tvl-buildkite.nix")
     (mod "tvl-slapd/default.nix")
     (mod "tvl-users.nix")
     (mod "www/auth.tvl.fyi.nix")
     (mod "www/b.tvl.fyi.nix")
     (mod "www/cache.tvl.fyi.nix")
+    (mod "www/cache.tvl.su.nix")
     (mod "www/cl.tvl.fyi.nix")
     (mod "www/code.tvl.fyi.nix")
     (mod "www/cs.tvl.fyi.nix")
@@ -35,7 +39,6 @@ in
     (mod "www/self-cache.tvl.fyi.nix")
     (mod "www/self-redirect.nix")
     (mod "www/status.tvl.su.nix")
-    (depot.third_party.agenix.src + "/modules/age.nix")
   ];
 
   hardware.cpu.amd.updateMicrocode = true;
@@ -420,6 +423,22 @@ in
         remote_user = "tvlbot@tazj.in";
       };
     };
+
+    # Run the Telegram<>IRC bridge for Volga Sprint.
+    teleirc.enable = true;
+
+    # Configure backups to GleSYS
+    restic = {
+      enable = true;
+      paths = [
+        "/var/backup/postgresql"
+        "/var/lib/grafana"
+        "/var/lib/znc"
+      ];
+    };
+
+    # Run autosubmit bot for Gerrit
+    gerrit-autosubmit.enable = true;
   };
 
   # Start a ZNC instance which bounces for tvlbot and owothia.
@@ -486,6 +505,12 @@ in
   tvl.cache.enable = true;
   tvl.cache.builderball = true;
 
+  # Disable background git gc system-wide, as it has a tendency to break CI.
+  environment.etc."gitconfig".source = pkgs.writeText "gitconfig" ''
+    [gc]
+    autoDetach = false
+  '';
+
   security.sudo.extraRules = [{
     groups = [ "wheel" ];
     commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
@@ -504,5 +529,31 @@ in
 
   zramSwap.enable = true;
 
+  environment.systemPackages = (with pkgs; [
+    bat
+    bb
+    curl
+    direnv
+    emacs-nox
+    fd
+    git
+    htop
+    hyperfine
+    jq
+    nano
+    nix-diff
+    nix-top
+    nvd
+    ripgrep
+    screen
+    tig
+    tree
+    unzip
+    vim
+    watchexec
+    zfs
+    zfstools
+  ]);
+
   system.stateVersion = "24.11";
 }
diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix
index 786b240004f3..c2e1044fff91 100644
--- a/ops/machines/whitby/default.nix
+++ b/ops/machines/whitby/default.nix
@@ -9,51 +9,17 @@ let
 in
 {
   imports = [
-    (mod "atward.nix")
     (mod "builderball.nix")
-    (mod "cgit.nix")
-    (mod "cheddar.nix")
-    (mod "clbot.nix")
-    (mod "gerrit-autosubmit.nix")
     (mod "harmonia.nix")
-    (mod "irccat.nix")
-    (mod "josh.nix")
     (mod "journaldriver.nix")
-    (mod "known-hosts.nix")
-    (mod "livegrep.nix")
-    (mod "monorepo-gerrit.nix")
-    (mod "owothia.nix")
-    (mod "panettone.nix")
-    (mod "paroxysm.nix")
-    (mod "restic.nix")
-    (mod "smtprelay.nix")
-    (mod "teleirc.nix")
     (mod "tvl-buildkite.nix")
-    (mod "tvl-slapd/default.nix")
     (mod "tvl-users.nix")
-    (mod "www/atward.tvl.fyi.nix")
-    (mod "www/auth.tvl.fyi.nix")
-    (mod "www/b.tvl.fyi.nix")
     (mod "www/cache.tvl.fyi.nix")
     (mod "www/cache.tvl.su.nix")
-    (mod "www/cl.tvl.fyi.nix")
-    (mod "www/code.tvl.fyi.nix")
-    (mod "www/cs.tvl.fyi.nix")
-    (mod "www/deploys.tvl.fyi.nix")
     (mod "www/self-cache.tvl.fyi.nix")
     (mod "www/self-redirect.nix")
-    (mod "www/signup.tvl.fyi.nix")
-    (mod "www/static.tvl.fyi.nix")
-    (mod "www/status.tvl.su.nix")
-    (mod "www/todo.tvl.fyi.nix")
-    (mod "www/tvix.dev.nix")
-    (mod "www/tvl.fyi.nix")
-    (mod "www/tvl.su.nix")
     (mod "www/wigglydonke.rs.nix")
 
-    # experimental!
-    (mod "www/grep.tvl.fyi.nix")
-
     (depot.third_party.agenix.src + "/modules/age.nix")
   ];
 
@@ -315,179 +281,8 @@ in
     agentCount = 32;
   };
 
-  # Run Markdown/code renderer
-  services.depot.cheddar.enable = true;
-
-  # Start a local SMTP relay to Gmail (used by gerrit)
-  services.depot.smtprelay = {
-    enable = true;
-    args = {
-      listen = ":2525";
-      remote_host = "smtp.gmail.com:587";
-      remote_auth = "plain";
-      remote_user = "tvlbot@tazj.in";
-    };
-  };
-
-  # Start a ZNC instance which bounces for tvlbot and owothia.
-  services.znc = {
-    enable = true;
-    useLegacyConfig = false;
-    config = {
-      LoadModule = [
-        "webadmin"
-        "adminlog"
-      ];
-
-      User.admin = {
-        Admin = true;
-        Pass.password = {
-          Method = "sha256";
-          Hash = "bb00aa8239de484c2925b1c3f6a196fb7612633f001daa9b674f83abe7e1103f";
-          Salt = "TiB0Ochb1CrtpMTl;2;j";
-        };
-      };
-
-      Listener.l = {
-        Host = "localhost";
-        Port = 2627; # bncr
-        SSL = false;
-      };
-    };
-  };
-
-  # Start the Gerrit->IRC bot
-  services.depot.clbot = {
-    enable = true;
-    channels = {
-      "#tvl" = { };
-      "#tvix-dev" = {
-        only_display = "tvix,nix-compat,third_party,third-party,3p";
-      };
-    };
-
-    # See //fun/clbot for details.
-    flags = {
-      gerrit_host = "cl.tvl.fyi:29418";
-      gerrit_ssh_auth_username = "clbot";
-      gerrit_ssh_auth_key = config.age.secretsDir + "/clbot-ssh";
-
-      irc_server = "localhost:${toString config.services.znc.config.Listener.l.Port}";
-      irc_user = "tvlbot";
-      irc_nick = "tvlbot";
-
-      notify_branches = "canon,refs/meta/config";
-      notify_repo = "depot";
-
-      # This secret is read from an environment variable, which is
-      # populated by a systemd EnvironmentFile.
-      irc_pass = "$CLBOT_PASS";
-    };
-  };
-
-  services.depot = {
-    # Run a livegrep code search instance
-    livegrep.enable = true;
-
-    # Run Nix cache proxy
-    builderball.enable = true;
-
-    # Run the Panettone issue tracker
-    panettone = {
-      enable = true;
-      dbUser = "panettone";
-      dbName = "panettone";
-      irccatChannel = "#tvl";
-    };
-
-    # Run the first cursed bot (quote bot)
-    paroxysm.enable = true;
-
-    # Run the second cursed bot
-    owothia = {
-      enable = true;
-      ircServer = "localhost";
-      ircPort = config.services.znc.config.Listener.l.Port;
-    };
-
-    # Run irccat to forward messages to IRC
-    irccat = {
-      enable = true;
-      config = {
-        tcp.listen = ":4722"; # "ircc"
-        irc = {
-          server = "localhost:${toString config.services.znc.config.Listener.l.Port}";
-          tls = false;
-          nick = "tvlbot";
-          # Note: irccat means 'ident' where it says 'realname', so
-          # this is critical for connecting to ZNC.
-          realname = "tvlbot";
-          channels = [
-            "#tvl"
-          ];
-        };
-      };
-    };
-
-    # Run the Telegram<>IRC bridge for Volga Sprint.
-    teleirc.enable = true;
-
-    # Run atward, the search engine redirection thing.
-    atward.enable = true;
-
-    # Run cgit & josh to serve git
-    cgit = {
-      enable = true;
-      user = "git"; # run as the same user as gerrit
-    };
-
-    josh.enable = true;
-
-    # Configure backups to GleSYS
-    restic = {
-      enable = true;
-      paths = [
-        "/var/backup/postgresql"
-        "/var/lib/grafana"
-        "/var/lib/znc"
-      ];
-    };
-
-    # Run autosubmit bot for Gerrit
-    gerrit-autosubmit.enable = true;
-  };
-
-  services.postgresql = {
-    enable = true;
-    enableTCPIP = true;
-    package = pkgs.postgresql_16;
-
-    authentication = lib.mkForce ''
-      local all all trust
-      host all all 127.0.0.1/32 password
-      host all all ::1/128 password
-      hostnossl all all 127.0.0.1/32 password
-      hostnossl all all ::1/128  password
-    '';
-
-    ensureDatabases = [
-      "panettone"
-    ];
-
-    ensureUsers = [{
-      name = "panettone";
-      ensureDBOwnership = true;
-    }];
-  };
-
-  services.postgresqlBackup = {
-    enable = true;
-    databases = [
-      "keycloak"
-      "panettone"
-      "tvldb"
-    ];
-  };
+  # Run Nix cache proxy
+  services.depot.builderball.enable = true;
 
   # Run a Harmonia binary cache.
   #
@@ -501,159 +296,12 @@ in
 
   services.fail2ban.enable = true;
 
-  environment.systemPackages = (with pkgs; [
-    bat
-    bb
-    curl
-    direnv
-    emacs-nox
-    fd
-    git
-    htop
-    hyperfine
-    jq
-    nano
-    nvd
-    ripgrep
-    tree
-    unzip
-    vim
-    zfs
-    zfstools
-  ]) ++ (with depot; [
-    ops.deploy-whitby
-  ]);
-
-  # Required for prometheus to be able to scrape stats
-  services.nginx.statusPage = true;
-
-  # Configure Prometheus & Grafana. Exporter configuration for
-  # Prometheus is inside the respective service modules.
-  services.prometheus = {
-    enable = true;
-    retentionTime = "90d";
-
-    exporters = {
-      node = {
-        enable = true;
-
-        enabledCollectors = [
-          "logind"
-          "processes"
-          "systemd"
-        ];
-      };
-
-      nginx = {
-        enable = true;
-        sslVerify = false;
-        constLabels = [ "host=whitby" ];
-      };
-    };
-
-    scrapeConfigs = [{
-      job_name = "node";
-      scrape_interval = "5s";
-      static_configs = [{
-        targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
-      }];
-    }
-      {
-        job_name = "nginx";
-        scrape_interval = "5s";
-        static_configs = [{
-          targets = [ "localhost:${toString config.services.prometheus.exporters.nginx.port}" ];
-        }];
-      }];
-  };
-
-  services.grafana = {
-    enable = true;
-
-    settings = {
-      server = {
-        http_port = 4723; # "graf" on phone keyboard
-        domain = "status.tvl.su";
-        root_url = "https://status.tvl.su";
-      };
-
-      analytics.reporting_enabled = false;
-
-      "auth.generic_oauth" = {
-        enabled = true;
-        client_id = "grafana";
-        scopes = "openid profile email";
-        name = "TVL";
-        email_attribute_path = "mail";
-        login_attribute_path = "sub";
-        name_attribute_path = "displayName";
-        auth_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/auth";
-        token_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/token";
-        api_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/userinfo";
-
-        # Give lukegb, aspen, tazjin "Admin" rights.
-        role_attribute_path = "((sub == 'lukegb' || sub == 'aspen' || sub == 'tazjin') && 'Admin') || 'Editor'";
-
-        # Allow creating new Grafana accounts from OAuth accounts.
-        allow_sign_up = true;
-      };
-
-      "auth.anonymous" = {
-        enabled = true;
-        org_name = "The Virus Lounge";
-        org_role = "Viewer";
-      };
-
-      "auth.basic".enabled = false;
-
-      auth = {
-        oauth_auto_login = true;
-        disable_login_form = true;
-      };
-    };
-
-    provision = {
-      enable = true;
-      datasources.settings.datasources = [{
-        name = "Prometheus";
-        type = "prometheus";
-        url = "http://localhost:9090";
-      }];
-    };
-  };
-
-  # Contains GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET.
-  systemd.services.grafana.serviceConfig.EnvironmentFile = config.age.secretsDir + "/grafana";
-
-  services.keycloak = {
-    enable = true;
-
-    settings = {
-      http-port = 5925; # kycl
-      hostname = "auth.tvl.fyi";
-      http-relative-path = "/auth";
-      proxy-headers = "xforwarded";
-      http-enabled = true;
-    };
-
-    database = {
-      type = "postgresql";
-      passwordFile = config.age.secretsDir + "/keycloak-db";
-      createLocally = false;
-    };
-  };
-
   # Join TVL Tailscale network at net.tvl.fyi
   services.tailscale = {
     enable = true;
     useRoutingFeatures = "server"; # for exit-node usage
   };
 
-  # Allow Keycloak access to the LDAP module by forcing in the JVM
-  # configuration
-  systemd.services.keycloak.environment.PREPEND_JAVA_OPTS =
-    "--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED";
-
   security.sudo.extraRules = [
     {
       groups = [ "wheel" ];
@@ -661,17 +309,6 @@ in
     }
   ];
 
-  users = {
-    # Set up a user & group for git shenanigans
-    groups.git = { };
-    users.git = {
-      group = "git";
-      isSystemUser = true;
-      createHome = true;
-      home = "/var/lib/git";
-    };
-  };
-
   zramSwap.enable = true;
 
   # Use TVL cache locally through the proxy; for cross-builder substitution.
diff --git a/ops/modules/teleirc.nix b/ops/modules/teleirc.nix
index 9f9ac059ce38..6b076a2dd28d 100644
--- a/ops/modules/teleirc.nix
+++ b/ops/modules/teleirc.nix
@@ -5,7 +5,7 @@
 { depot, config, lib, pkgs, ... }:
 
 let
-  cfg = config.services.depot.owothia;
+  cfg = config.services.depot.teleirc;
   description = "IRC<>Telegram sync for Volga Sprint channel";
   configFile = builtins.toFile "teleirc.env" ''
     # connect through tvlbot's ZNC bouncer