feat(gs/mugwump): Add blackbox prometheus exporters r/2363

Add blackbox prometheus exporters to Mugwump with config for scraping
gws.fyi, windtunnel.ci, and app.windtunnel.ci

Change-Id: Ied9e329d44b506763b600e4978f65a5a3abcf5df
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2702
Reviewed-by: glittershark <grfn@gws.fyi>
Tested-by: BuildkiteCI

1 files changed, 40 insertions, 0 deletions

diff --git a/users/glittershark/system/system/machines/mugwump.nix b/users/glittershark/system/system/machines/mugwump.nix
index 64ee36ca72f3..22d9e7cd06bd 100644
--- a/users/glittershark/system/system/machines/mugwump.nix
+++ b/users/glittershark/system/system/machines/mugwump.nix
@@ -146,6 +146,24 @@ with lib;
         sslVerify = false;
         constLabels = [ "host=mugwump" ];
       };
+
+      blackbox = {
+        enable = true;
+        openFirewall = true;
+        configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON {
+          modules = {
+            https_2xx = {
+              prober = "http";
+              http = {
+                method = "GET";
+                fail_if_ssl = false;
+                fail_if_not_ssl = true;
+                preferred_ip_protocol = "ip4";
+              };
+            };
+          };
+        });
+      };
     };
 
     scrapeConfigs = [{
@@ -160,6 +178,28 @@ with lib;
       static_configs = [{
         targets = ["localhost:${toString config.services.prometheus.exporters.nginx.port}"];
       }];
+    } {
+      job_name = "blackbox";
+      metrics_path = "/probe";
+      params.module = ["https_2xx"];
+      scrape_interval = "5s";
+      static_configs = [{
+        targets = [
+          "https://www.gws.fyi"
+          "https://windtunnel.ci"
+          "https://app.windtunnel.ci"
+        ];
+      }];
+      relabel_configs = [{
+        source_labels = ["__address__"];
+        target_label = "__param_target";
+      } {
+        source_labels = ["__param_target"];
+        target_label = "instance";
+      } {
+        target_label = "__address__";
+        replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
+      }];
     }];
   };