feat(aspen/system): Move metrics to ogopogo, refresh r/8707

Change-Id: I93ddc961b473e15febe22a16879875dbd926236a
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12501
Autosubmit: aspen <root@gws.fyi>
Tested-by: BuildkiteCI
Reviewed-by: aspen <root@gws.fyi>

13 files changed, 240 insertions, 151 deletions

diff --git a/ops/machines/all-systems.nix b/ops/machines/all-systems.nix
index 5df09fa0bd30..14a8b6b26a11 100644
--- a/ops/machines/all-systems.nix
+++ b/ops/machines/all-systems.nix
@@ -4,7 +4,6 @@
   sanduny
   whitby
   nixery-01
-  volgasprint-cache
 ]) ++
 
 (with depot.users.tazjin.nixos; [
diff --git a/users/aspen/secrets/bbbg.age b/users/aspen/secrets/bbbg.age
index d8294b047191..379441b74f5c 100644
--- a/users/aspen/secrets/bbbg.age
+++ b/users/aspen/secrets/bbbg.age
Binary files differdiff --git a/users/aspen/secrets/buildkite-ssh-key.age b/users/aspen/secrets/buildkite-ssh-key.age
index 062be3b9bd98..61ad416385c6 100644
--- a/users/aspen/secrets/buildkite-ssh-key.age
+++ b/users/aspen/secrets/buildkite-ssh-key.age
Binary files differdiff --git a/users/aspen/secrets/buildkite-token.age b/users/aspen/secrets/buildkite-token.age
index f55b31fb08ed..5bd4923de34f 100644
--- a/users/aspen/secrets/buildkite-token.age
+++ b/users/aspen/secrets/buildkite-token.age
Binary files differdiff --git a/users/aspen/secrets/cloudflare.age b/users/aspen/secrets/cloudflare.age
index 6b3974ec7ab6..c94fef706c4c 100644
--- a/users/aspen/secrets/cloudflare.age
+++ b/users/aspen/secrets/cloudflare.age
Binary files differdiff --git a/users/aspen/secrets/ddclient-password.age b/users/aspen/secrets/ddclient-password.age
index bc82063c3a28..3bbc2e51ffd3 100644
--- a/users/aspen/secrets/ddclient-password.age
+++ b/users/aspen/secrets/ddclient-password.age
Binary files differdiff --git a/users/aspen/secrets/secrets.nix b/users/aspen/secrets/secrets.nix
index 778b8ebd6e0a..76126f811d02 100644
--- a/users/aspen/secrets/secrets.nix
+++ b/users/aspen/secrets/secrets.nix
@@ -7,7 +7,7 @@ in
 
 {
   "bbbg.age".publicKeys = [ grfn mugwump bbbg ];
-  "cloudflare.age".publicKeys = [ grfn mugwump ];
+  "cloudflare.age".publicKeys = [ grfn mugwump ogopogo ];
   "ddclient-password.age".publicKeys = [ grfn ogopogo ];
   "buildkite-ssh-key.age".publicKeys = [ grfn mugwump ogopogo ];
   "buildkite-token.age".publicKeys = [ grfn mugwump ogopogo ];
diff --git a/users/aspen/secrets/windtunnel-bot-github-token.age b/users/aspen/secrets/windtunnel-bot-github-token.age
index 84e852f4c1f1..39fd7cb3a476 100644
--- a/users/aspen/secrets/windtunnel-bot-github-token.age
+++ b/users/aspen/secrets/windtunnel-bot-github-token.age
@@ -1,9 +1,9 @@
 age-encryption.org/v1
--> ssh-ed25519 CpJBgQ qVlQpHyewtBSfFIdU8GihXC7JhGbcvQ61ZsJC20wSH4
-mZXwiTICzrG+3aCL67cO6cTWMgHkxhDyBi7tZ8l+QMA
--> ssh-ed25519 LfBFbQ 78NQxflRkRMW5vSP1BEvASSQU2pZAfMwd7T2+6W7NQs
-u0x986pFtnD9ZqfL3KnRrdYS5z9LRUPJhcmc8FQOuGo
--> ssh-ed25519 GeE7sQ aqFQGCywSimHNbN5si0PzmESUXwROjrpTe/5UdTyYw4
-X2thEJIyOnNUsA746VwqZhH+44XBfCTvh7VOEg/zew0
---- ndSgjJv5Tel6ovKl+SBdDHZHlszgsEhOY1HHpNDvf1s
-��I��ʵu*1��t�(����/�X��˕3Ȓ����VG��T|�@���K�<}��)se�9`��*z
\ No newline at end of file
+-> ssh-ed25519 CpJBgQ PiY6IidA+GRbpjL91BVe9UdejWvi02SRcijiMOjXcm4
+XegOhgjdEdzXtz31PsGVyOZ10gH6P82Q1/txZcSxjIY
+-> ssh-ed25519 LfBFbQ uqRF0nKMk1GrK+6pEBdmyHKu2ewDFlWwlKC+myey4gc
+dgnX4eprSolXxCDNoVmGzGK9xLEmtmeg/cJihD4/8sU
+-> ssh-ed25519 GeE7sQ ikAIyFR/qH1a+aa5mumiiDwa5o5aLsQeJKwQwMzgs1M
+8htzhM5t2VnjRBrC+VrL23f9chlQjVGzjxMaFB7Arrs
+--- Qm16HTo5wGUBKS0ly3OZDWp2etLyDS/zlxOHxPjS8PI
+��7��NY6�k|�p2�'��&�=�m��q`5�T� �N�9�N�����)RV�U-�•)��M���(%p�
\ No newline at end of file
diff --git a/users/aspen/system/system/machines/lusca.nix b/users/aspen/system/system/machines/lusca.nix
index 16dabbd2ef7d..4a9202187dd0 100644
--- a/users/aspen/system/system/machines/lusca.nix
+++ b/users/aspen/system/system/machines/lusca.nix
@@ -10,6 +10,7 @@
     ../modules/sound.nix
     ../modules/tvl.nix
     ../modules/development.nix
+    ../modules/prometheus-exporter.nix
   ];
 
   networking.hostName = "lusca";
diff --git a/users/aspen/system/system/machines/mugwump.nix b/users/aspen/system/system/machines/mugwump.nix
index 1daa92f25f42..4b72a247601f 100644
--- a/users/aspen/system/system/machines/mugwump.nix
+++ b/users/aspen/system/system/machines/mugwump.nix
@@ -117,149 +117,9 @@ with lib;
     };
   };
 
-  services.grafana = {
-    enable = true;
-    dataDir = "/var/lib/grafana";
-
-    settings = {
-      server = {
-        http_port = 3000;
-        root_url = "https://metrics.gws.fyi";
-        domain = "metrics.gws.fyi";
-      };
-      analytics.reporting_enabled = false;
-    };
-
-    provision = {
-      enable = true;
-      datasources.settings.datasources = [{
-        name = "Prometheus";
-        type = "prometheus";
-        url = "http://localhost:9090";
-      }];
-    };
-  };
-
   security.acme.defaults.email = "root@gws.fyi";
   security.acme.acceptTerms = true;
 
-  services.nginx = {
-    enable = true;
-    statusPage = true;
-    recommendedGzipSettings = true;
-    recommendedOptimisation = true;
-    recommendedTlsSettings = true;
-    recommendedProxySettings = true;
-
-    virtualHosts = {
-      "metrics.gws.fyi" = {
-        enableACME = true;
-        forceSSL = true;
-        locations."/" = {
-          proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
-        };
-      };
-    };
-  };
-
-  security.acme.certs."metrics.gws.fyi" = {
-    dnsProvider = "cloudflare";
-    credentialsFile = config.age.secretsDir + "/cloudflare";
-    webroot = mkForce null;
-  };
-
-  services.prometheus = {
-    enable = true;
-    exporters = {
-      node = {
-        enable = true;
-        openFirewall = false;
-
-        enabledCollectors = [
-          "processes"
-          "systemd"
-          "tcpstat"
-          "wifi"
-        ];
-      };
-
-      nginx = {
-        enable = true;
-        openFirewall = true;
-        sslVerify = false;
-        constLabels = [ "host=mugwump" ];
-      };
-
-      blackbox = {
-        enable = true;
-        openFirewall = true;
-        configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON {
-          modules = {
-            https_2xx = {
-              prober = "http";
-              http = {
-                method = "GET";
-                fail_if_ssl = false;
-                fail_if_not_ssl = true;
-                preferred_ip_protocol = "ip4";
-              };
-            };
-          };
-        });
-      };
-    };
-
-    scrapeConfigs = [
-      {
-        job_name = "node";
-        scrape_interval = "5s";
-        static_configs = [{
-          targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
-        }];
-      }
-      {
-        job_name = "nginx";
-        scrape_interval = "5s";
-        static_configs = [{
-          targets = [ "localhost:${toString config.services.prometheus.exporters.nginx.port}" ];
-        }];
-      }
-      {
-        job_name = "xanthous_server";
-        scrape_interval = "1s";
-        static_configs = [{
-          targets = [ "localhost:${toString config.services.xanthous-server.metricsPort}" ];
-        }];
-      }
-      {
-        job_name = "blackbox";
-        metrics_path = "/probe";
-        params.module = [ "https_2xx" ];
-        scrape_interval = "5s";
-        static_configs = [{
-          targets = [
-            "https://gws.fyi"
-            "https://windtunnel.ci"
-            "https://app.windtunnel.ci"
-            "https://metrics.gws.fyi"
-          ];
-        }];
-        relabel_configs = [{
-          source_labels = [ "__address__" ];
-          target_label = "__param_target";
-        }
-          {
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            target_label = "__address__";
-            replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
-          }];
-      }
-    ];
-  };
-
   services.xanthous-server.enable = true;
 
   virtualisation.docker = {
diff --git a/users/aspen/system/system/machines/ogopogo.nix b/users/aspen/system/system/machines/ogopogo.nix
index 4dbb3d14e6ce..3d41a839e17b 100644
--- a/users/aspen/system/system/machines/ogopogo.nix
+++ b/users/aspen/system/system/machines/ogopogo.nix
@@ -11,6 +11,8 @@
     ../modules/tvl.nix
     ../modules/development.nix
     ../modules/wireshark.nix
+    ../modules/metrics.nix
+    ../modules/prometheus-exporter.nix
   ];
 
   networking.hostName = "ogopogo";
@@ -92,7 +94,6 @@
     dataDir = "/data/postgresql";
     package = pkgs.postgresql_15;
     settings = {
-      port = 5431;
       wal_level = "logical";
     };
   };
diff --git a/users/aspen/system/system/modules/metrics.nix b/users/aspen/system/system/modules/metrics.nix
new file mode 100644
index 000000000000..0abfb27eeeb5
--- /dev/null
+++ b/users/aspen/system/system/modules/metrics.nix
@@ -0,0 +1,197 @@
+{ depot, config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  nodesToScrape = [
+    "ogopogo"
+    # "dobharchu"
+    "mugwump"
+    # "yeren"
+    "lusca"
+  ];
+
+  nodesRunningNginx = [
+    "ogopogo"
+    "mugwump"
+  ];
+
+  nodesRunningPostgres = [
+    "ogopogo"
+  ];
+
+  blackboxTargets = [
+    "https://gws.fyi"
+    "https://windtunnel.ci"
+    "https://app.windtunnel.ci"
+    "https://metrics.gws.fyi"
+  ];
+in
+{
+  imports = [
+    (depot.third_party.agenix.src + "/modules/age.nix")
+  ];
+
+  config = {
+    services.postgresql = {
+      ensureUsers = [{
+        name = config.services.grafana.settings.database.user;
+        ensureDBOwnership = true;
+      }];
+
+      ensureDatabases = [
+        config.services.grafana.settings.database.name
+      ];
+    };
+
+    services.grafana = {
+      enable = true;
+      dataDir = "/var/lib/grafana";
+
+      settings = {
+        server = {
+          http_port = 3000;
+          root_url = "https://metrics.gws.fyi";
+          domain = "metrics.gws.fyi";
+        };
+        analytics.reporting_enabled = false;
+
+        database = {
+          type = "postgres";
+          user = "grafana";
+          name = "grafana";
+          host = "/run/postgresql";
+        };
+      };
+
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [{
+          name = "Prometheus";
+          type = "prometheus";
+          url = "http://localhost:9090";
+        }];
+      };
+    };
+
+    security.acme.defaults.email = "root@gws.fyi";
+    security.acme.acceptTerms = true;
+
+    services.nginx = {
+      enable = true;
+      statusPage = true;
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedTlsSettings = true;
+      recommendedProxySettings = true;
+
+      virtualHosts = {
+        "metrics.gws.fyi" = {
+          enableACME = true;
+          forceSSL = true;
+          locations."/" = {
+            proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
+          };
+        };
+      };
+    };
+
+    age.secrets = {
+      cloudflare.file = depot.users.aspen.secrets."cloudflare.age";
+    };
+
+    security.acme.certs."metrics.gws.fyi" = {
+      dnsProvider = "cloudflare";
+      credentialsFile = config.age.secretsDir + "/cloudflare";
+      webroot = mkForce null;
+    };
+
+    services.prometheus = {
+      enable = true;
+      retentionTime = "30d";
+      exporters = {
+        blackbox = {
+          enable = true;
+          openFirewall = true;
+          configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON {
+            modules = {
+              https_2xx = {
+                prober = "http";
+                http = {
+                  method = "GET";
+                  fail_if_ssl = false;
+                  fail_if_not_ssl = true;
+                  preferred_ip_protocol = "ip4";
+                };
+              };
+            };
+          });
+        };
+      };
+
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          scrape_interval = "5s";
+          static_configs =
+            map
+              (node: {
+                targets = [ "${node}:${toString config.services.prometheus.exporters.node.port}" ];
+                labels.node = node;
+              })
+              nodesToScrape;
+        }
+        {
+          job_name = "nginx";
+          scrape_interval = "5s";
+          static_configs =
+            map
+              (node: {
+                targets = [ "${node}:${toString config.services.prometheus.exporters.nginx.port}" ];
+                labels.node = node;
+              })
+              nodesRunningNginx;
+        }
+        {
+          job_name = "postgres";
+          scrape_interval = "5s";
+          static_configs =
+            map
+              (node: {
+                targets = [ "${node}:${toString config.services.prometheus.exporters.postgres.port}" ];
+                labels.node = node;
+              })
+              nodesRunningPostgres;
+        }
+        {
+          job_name = "blackbox";
+          metrics_path = "/probe";
+          params.module = [ "https_2xx" ];
+          scrape_interval = "5s";
+          static_configs = [{
+            targets = [
+              "https://gws.fyi"
+              "https://windtunnel.ci"
+              "https://app.windtunnel.ci"
+              "https://metrics.gws.fyi"
+            ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
+            }
+          ];
+        }
+      ];
+    };
+  };
+}
diff --git a/users/aspen/system/system/modules/prometheus-exporter.nix b/users/aspen/system/system/modules/prometheus-exporter.nix
new file mode 100644
index 000000000000..2916fc70ef96
--- /dev/null
+++ b/users/aspen/system/system/modules/prometheus-exporter.nix
@@ -0,0 +1,31 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      openFirewall = false;
+
+      enabledCollectors = [
+        "processes"
+        "systemd"
+        "tcpstat"
+        "wifi"
+      ];
+    };
+
+    nginx = mkIf config.services.nginx.enable {
+      enable = true;
+      openFirewall = true;
+      sslVerify = false;
+      constLabels = [ "host=${config.networking.hostName}" ];
+    };
+
+    postgres = mkIf config.services.postgresql.enable {
+      enable = true;
+      runAsLocalSuperUser = true;
+    };
+  };
+}