feat(whitby): Enable Prometheus instance on whitby r/2493

Enables Prometheus with a local node exporter, and nothing else for
now.

Some additional collectors have been enabled for things that might be
relevant on whitby:

* systemd: all our services run in systemd
* processes: might be interesting for build-related stats
* logind: might be interesting for interactive usage stats

Change-Id: I48dacdd9c68b4be9edff7b3cb6256dad562498c4
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2930
Tested-by: BuildkiteCI
Reviewed-by: grfn <grfn@gws.fyi>
Reviewed-by: lukegb <lukegb@tvl.fyi>

1 files changed, 24 insertions, 1 deletions

diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix
index eb93aef69d..df280b96f7 100644
--- a/ops/machines/whitby/default.nix
+++ b/ops/machines/whitby/default.nix
@@ -1,5 +1,5 @@
 { depot, lib, pkgs, ... }: # readTree options
-config: # passed by module system
+{ config, ... }: # passed by module system
 
 let
   inherit (builtins) listToAttrs;
@@ -357,6 +357,29 @@ in lib.fix(self: {
     applicationCredentials = "/var/lib/journaldriver/key.json";
   };
 
+  # Configure Prometheus & Grafana. Exporter configuration for
+  # Prometheus is inside the respective service modules.
+  services.prometheus = {
+    enable = true;
+    exporters.node = {
+      enable = true;
+
+      enabledCollectors = [
+        "logind"
+        "processes"
+        "systemd"
+      ];
+    };
+
+    scrapeConfigs = [{
+      job_name = "node";
+      scrape_interval = "5s";
+      static_configs = [{
+        targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+      }];
+    }];
+  };
+
   security.sudo.extraRules = [
     {
       groups = ["wheel"];