feat(gs/mugwump): Add prometheus+grafana r/1839

Add config for prometheus+grafana to mugwump, served at metrics.gws.fyi
with an Acme SSL cert.

Change-Id: Icc22b5079a24edbc4469233e938f926d92f63eb3
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2024
Reviewed-by: glittershark <grfn@gws.fyi>
Tested-by: BuildkiteCI

1 files changed, 77 insertions, 2 deletions

diff --git a/users/glittershark/system/system/machines/mugwump.nix b/users/glittershark/system/system/machines/mugwump.nix
index 9521f578846d..a2c207c4b095 100644
--- a/users/glittershark/system/system/machines/mugwump.nix
+++ b/users/glittershark/system/system/machines/mugwump.nix
@@ -1,9 +1,10 @@
 { config, lib, pkgs, modulesPath, ... }:
 
+with lib;
+
 {
   imports = [
     ../modules/common.nix
-    ../modules/tvl.nix
     (modulesPath + "/installer/scan/not-detected.nix")
   ];
 
@@ -52,7 +53,7 @@
   };
 
   networking.firewall.enable = true;
-  networking.firewall.allowedTCPPorts = [ 22 ];
+  networking.firewall.allowedTCPPorts = [ 22 80 443 ];
 
   security.sudo.extraRules = [{
     groups = ["wheel"];
@@ -71,4 +72,78 @@
     passwordAuthentication = false;
     permitRootLogin = "no";
   };
+
+  services.grafana = {
+    enable = true;
+    port = 3000;
+    domain = "metrics.gws.fyi";
+    rootUrl = "https://metrics.gws.fyi";
+    dataDir = "/var/lib/grafana";
+    analytics.reporting.enable = false;
+
+    provision = {
+      enable = true;
+      datasources = [{
+        name = "Prometheus";
+        type = "prometheus";
+        url = "localhost:9090";
+      }];
+    };
+  };
+
+  security.acme.email = "root@gws.fyi";
+  security.acme.acceptTerms = true;
+
+  services.nginx = {
+    enable = true;
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedTlsSettings = true;
+
+    virtualHosts = {
+      "metrics.gws.fyi" = {
+        enableACME = true;
+        forceSSL = true;
+        locations."/" = {
+          proxyPass = "http://localhost:${toString config.services.grafana.port}";
+        };
+      };
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+    exporters = {
+      node = {
+        enable = true;
+        openFirewall = false;
+
+        enabledCollectors = [
+          "processes"
+          "systemd"
+          "tcpstat"
+          "wifi"
+        ];
+      };
+
+      nginx = {
+        enable = true;
+        openFirewall = true;
+      };
+    };
+
+    scrapeConfigs = [{
+      job_name = "node";
+      scrape_interval = "5s";
+      static_configs = [{
+        targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+      }];
+    }];
+  };
+
+  security.acme.certs."metrics.gws.fyi" = {
+    dnsProvider = "namecheap";
+    credentialsFile = "/etc/secrets/namecheap.env";
+    webroot = mkForce null;
+  };
 }