feat(whitby): switch from nix-serve to harmonia for the cache r/8721

Harmonia is, ostensibly, faster and better and, most importantly, not a giant
pile of wonky Perl.

I've tested locally that Harmonia works with Nix 2.3 (on both ends), so I think
we should be good to go here.

We have a vendored copy of the upstream module for now. We need to fix Nix 2.3
compatibility in upstream for the module, but the service itself works fine.

Change-Id: I3897bb02b83bd466b6fe7077c05728ac49ea4406
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12517
Tested-by: BuildkiteCI
Autosubmit: tazjin <tazjin@tvl.su>
Reviewed-by: sterni <sternenseemann@systemli.org>

4 files changed, 125 insertions, 15 deletions

diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix
index 3181ccde451c..06add2168160 100644
--- a/ops/machines/whitby/default.nix
+++ b/ops/machines/whitby/default.nix
@@ -14,6 +14,7 @@ in
     (mod "cheddar.nix")
     (mod "clbot.nix")
     (mod "gerrit-autosubmit.nix")
+    (mod "harmonia.nix")
     (mod "irccat.nix")
     (mod "josh.nix")
     (mod "journaldriver.nix")
@@ -229,12 +230,17 @@ in
       grafana.file = secretFile "grafana";
       irccat.file = secretFile "irccat";
       keycloak-db.file = secretFile "keycloak-db";
-      nix-cache-priv.file = secretFile "nix-cache-priv";
       owothia.file = secretFile "owothia";
       panettone.file = secretFile "panettone";
       smtprelay.file = secretFile "smtprelay";
       teleirc.file = secretFile "teleirc";
 
+      nix-cache-priv = {
+        file = secretFile "nix-cache-priv";
+        mode = "0440";
+        group = "harmonia";
+      };
+
       buildkite-agent-token = {
         file = secretFile "buildkite-agent-token";
         mode = "0440";
@@ -477,11 +483,14 @@ in
     ];
   };
 
-  services.nix-serve = {
+  # Run a Harmonia binary cache.
+  #
+  # TODO(tazjin): switch to upstream module after fix for Nix 2.3
+  services.depot.harmonia = {
     enable = true;
-    port = 6443;
-    secretKeyFile = config.age.secretsDir + "/nix-cache-priv";
-    bindAddress = "localhost";
+    signKeyPaths = [ (config.age.secretsDir + "/nix-cache-priv") ];
+    settings.bind = "127.0.0.1:6443";
+    settings.priority = 50;
   };
 
   services.fail2ban.enable = true;
diff --git a/ops/modules/harmonia.nix b/ops/modules/harmonia.nix
new file mode 100644
index 000000000000..ae0bdc2cf01e
--- /dev/null
+++ b/ops/modules/harmonia.nix
@@ -0,0 +1,110 @@
+# This is a fork of the nixpkgs module for Harmonia, which adds compatibility
+# with Nix 2.3.
+#
+# We will upstream this eventually.
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.depot.harmonia;
+  format = pkgs.formats.toml { };
+
+  credentials = lib.imap0
+    (i: signKeyPath: {
+      id = "sign-key-${builtins.toString i}";
+      path = signKeyPath;
+    })
+    cfg.signKeyPaths;
+in
+{
+  options = {
+    services.depot.harmonia = {
+      enable = lib.mkEnableOption "Harmonia: Nix binary cache written in Rust";
+
+      signKeyPaths = lib.mkOption {
+        type = lib.types.listOf lib.types.path;
+        default = [ ];
+        description = "Paths to the signing keys to use for signing the cache";
+      };
+
+      package = lib.mkPackageOption pkgs "harmonia" { };
+
+      settings = lib.mkOption {
+        inherit (format) type;
+        default = { };
+        description = ''
+          Settings to merge with the default configuration.
+          For the list of the default configuration, see <https://github.com/nix-community/harmonia/tree/master#configuration>.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users.harmonia = {
+      isSystemUser = true;
+      group = "harmonia";
+    };
+    users.groups.harmonia = { };
+
+    systemd.services.harmonia = {
+      description = "harmonia binary cache service";
+
+      requires = [ "nix-daemon.socket" ];
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment = {
+        CONFIG_FILE = format.generate "harmonia.toml" cfg.settings;
+        SIGN_KEY_PATHS = lib.strings.concatMapStringsSep " "
+          (
+            credential: "%d/${credential.id}"
+          )
+          credentials;
+        # Note: it's important to set this for nix-store, because it wants to use
+        # $HOME in order to use a temporary cache dir. bizarre failures will occur
+        # otherwise
+        HOME = "/run/harmonia";
+      };
+
+      serviceConfig = {
+        ExecStart = lib.getExe cfg.package;
+        User = "harmonia";
+        Group = "harmonia";
+        Restart = "on-failure";
+        PrivateUsers = true;
+        DeviceAllow = [ "" ];
+        UMask = "0066";
+        RuntimeDirectory = "harmonia";
+        LoadCredential = builtins.map (credential: "${credential.id}:${credential.path}") credentials;
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+          "~@resources"
+        ];
+        CapabilityBoundingSet = "";
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectKernelLogs = true;
+        ProtectHostname = true;
+        ProtectClock = true;
+        RestrictRealtime = true;
+        MemoryDenyWriteExecute = true;
+        ProcSubset = "pid";
+        ProtectProc = "invisible";
+        RestrictNamespaces = true;
+        SystemCallArchitectures = "native";
+        PrivateNetwork = false;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        LockPersonality = true;
+        RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
+        LimitNOFILE = 65536;
+      };
+    };
+  };
+}
+
diff --git a/ops/modules/www/cache.tvl.su.nix b/ops/modules/www/cache.tvl.su.nix
index 99bc008cd6a5..27d1c06dd3a7 100644
--- a/ops/modules/www/cache.tvl.su.nix
+++ b/ops/modules/www/cache.tvl.su.nix
@@ -17,13 +17,8 @@
           alias /run/agenix/nix-cache-pub;
         }
 
-        location = /nix-cache-info {
-          add_header Content-Type text/plain;
-          return 200 "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 50\n";
-        }
-
         location / {
-          proxy_pass http://localhost:${toString config.services.nix-serve.port};
+          proxy_pass http://${config.services.depot.harmonia.settings.bind};
         }
       '';
     };
diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix
index a7cc68b2d994..7b3157854163 100644
--- a/third_party/overlays/tvl.nix
+++ b/third_party/overlays/tvl.nix
@@ -79,10 +79,6 @@ depot.nix.readTree.drvTargets {
     };
   });
 
-  # nix-serve does not work with nix 2.4
-  # https://github.com/edolstra/nix-serve/issues/28
-  nix-serve = super.nix-serve.override { nix = self.nix_2_3; };
-
   # Avoid builds of mkShell derivations in CI.
   mkShell = super.lib.makeOverridable (args: (super.mkShell args).overrideAttrs (_: {
     passthru = {