diff options
Diffstat (limited to 'ops/machines/whitby/default.nix')
| -rw-r--r-- | ops/machines/whitby/default.nix | 597 |
1 files changed, 341 insertions, 256 deletions
diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix index e558d10da9..6a8ee56abc 100644 --- a/ops/machines/whitby/default.nix +++ b/ops/machines/whitby/default.nix @@ -4,33 +4,52 @@ let inherit (builtins) listToAttrs; inherit (lib) range; -in { + + mod = name: depot.path.origSrc + ("/ops/modules/" + name); +in +{ imports = [ - "${depot.path}/ops/modules/atward.nix" - "${depot.path}/ops/modules/automatic-gc.nix" - "${depot.path}/ops/modules/clbot.nix" - "${depot.path}/ops/modules/irccat.nix" - "${depot.path}/ops/modules/monorepo-gerrit.nix" - "${depot.path}/ops/modules/panettone.nix" - "${depot.path}/ops/modules/paroxysm.nix" - "${depot.path}/ops/modules/smtprelay.nix" - "${depot.path}/ops/modules/sourcegraph.nix" - "${depot.path}/ops/modules/tvl-buildkite.nix" - "${depot.path}/ops/modules/tvl-slapd/default.nix" - "${depot.path}/ops/modules/tvl-sso/default.nix" - "${depot.path}/ops/modules/www/atward.tvl.fyi.nix" - "${depot.path}/ops/modules/www/b.tvl.fyi.nix" - "${depot.path}/ops/modules/www/cache.tvl.su.nix" - "${depot.path}/ops/modules/www/cl.tvl.fyi.nix" - "${depot.path}/ops/modules/www/code.tvl.fyi.nix" - "${depot.path}/ops/modules/www/cs.tvl.fyi.nix" - "${depot.path}/ops/modules/www/login.tvl.fyi.nix" - "${depot.path}/ops/modules/www/status.tvl.su.nix" - "${depot.path}/ops/modules/www/tazj.in.nix" - "${depot.path}/ops/modules/www/todo.tvl.fyi.nix" - "${depot.path}/ops/modules/www/tvl.fyi.nix" - "${depot.path}/ops/modules/www/wigglydonke.rs.nix" - "${pkgs.path}/nixos/modules/services/web-apps/gerrit.nix" + (mod "atward.nix") + (mod "cgit.nix") + (mod "clbot.nix") + (mod "gerrit-autosubmit.nix") + (mod "irccat.nix") + (mod "josh.nix") + (mod "journaldriver.nix") + (mod "known-hosts.nix") + (mod "livegrep.nix") + (mod "monorepo-gerrit.nix") + (mod "owothia.nix") + (mod "panettone.nix") + (mod "paroxysm.nix") + (mod "restic.nix") + (mod "smtprelay.nix") + (mod "sourcegraph.nix") + (mod "tvl-buildkite.nix") + (mod "tvl-slapd/default.nix") + (mod "tvl-users.nix") + (mod "www/atward.tvl.fyi.nix") + (mod "www/auth.tvl.fyi.nix") + (mod "www/b.tvl.fyi.nix") + (mod "www/cache.tvl.su.nix") + (mod "www/cl.tvl.fyi.nix") + (mod "www/code.tvl.fyi.nix") + (mod "www/cs.tvl.fyi.nix") + (mod "www/deploys.tvl.fyi.nix") + (mod "www/self-redirect.nix") + (mod "www/signup.tvl.fyi.nix") + (mod "www/static.tvl.fyi.nix") + (mod "www/status.tvl.su.nix") + (mod "www/todo.tvl.fyi.nix") + (mod "www/tvix.dev.nix") + (mod "www/tvl.fyi.nix") + (mod "www/tvl.su.nix") + (mod "www/wigglydonke.rs.nix") + + # experimental! + (mod "www/grep.tvl.fyi.nix") + + (depot.third_party.agenix.src + "/modules/age.nix") ]; hardware = { @@ -39,13 +58,19 @@ in { }; boot = { - tmpOnTmpfs = true; + tmp.useTmpfs = true; kernelModules = [ "kvm-amd" ]; supportedFilesystems = [ "zfs" ]; initrd = { availableKernelModules = [ - "igb" "xhci_pci" "nvme" "ahci" "usbhid" "usb_storage" "sr_mod" + "igb" + "xhci_pci" + "nvme" + "ahci" + "usbhid" + "usb_storage" + "sr_mod" ]; # Enable SSH in the initrd so that we can enter disk encryption @@ -58,7 +83,7 @@ in { authorizedKeys = depot.users.tazjin.keys.all ++ depot.users.lukegb.keys.all - ++ [ depot.users.grfn.keys.whitby ]; + ++ [ depot.users.aspen.keys.whitby ]; hostKeys = [ /etc/secrets/initrd_host_ed25519_key @@ -79,7 +104,6 @@ in { loader.grub = { enable = true; - version = 2; efiSupport = true; efiInstallAsRemovable = true; device = "/dev/disk/by-id/nvme-SAMSUNG_MZQLB1T9HAJR-00007_S439NA0N201620"; @@ -129,7 +153,8 @@ in { interface = "enp196s0"; }; - firewall.allowedTCPPorts = [ 22 80 443 4238 29418 ]; + firewall.allowedTCPPorts = [ 22 80 443 4238 8443 29418 ]; + firewall.allowedUDPPorts = [ 8443 ]; interfaces.enp196s0.useDHCP = true; interfaces.enp196s0.ipv6.addresses = [ @@ -159,26 +184,26 @@ in { nix = { nrBuildUsers = 256; - maxJobs = lib.mkDefault 64; - extraOptions = '' - secret-key-files = /etc/secrets/nix-cache-privkey - ''; - - trustedUsers = [ - "grfn" - "lukegb" - "tazjin" - "sterni" - ]; + settings = { + max-jobs = lib.mkDefault 64; + secret-key-files = "/run/agenix/nix-cache-priv"; + + trusted-users = [ + "aspen" + "lukegb" + "tazjin" + "sterni" + ]; + }; sshServe = { enable = true; keys = with depot.users; tazjin.keys.all ++ lukegb.keys.all - ++ [ grfn.keys.whitby ] + ++ [ aspen.keys.whitby ] ++ sterni.keys.all - ; + ; }; }; @@ -186,10 +211,84 @@ in { programs.mosh.enable = true; services.openssh = { enable = true; - passwordAuthentication = false; - challengeResponseAuthentication = false; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + }; }; + # Configure secrets for services that need them. + age.secrets = + let + secretFile = name: depot.ops.secrets."${name}.age"; + in + { + clbot.file = secretFile "clbot"; + gerrit-autosubmit.file = secretFile "gerrit-autosubmit"; + grafana.file = secretFile "grafana"; + irccat.file = secretFile "irccat"; + keycloak-db.file = secretFile "keycloak-db"; + nix-cache-priv.file = secretFile "nix-cache-priv"; + owothia.file = secretFile "owothia"; + panettone.file = secretFile "panettone"; + smtprelay.file = secretFile "smtprelay"; + + buildkite-agent-token = { + file = secretFile "buildkite-agent-token"; + mode = "0440"; + group = "buildkite-agents"; + }; + + buildkite-graphql-token = { + file = secretFile "buildkite-graphql-token"; + mode = "0440"; + group = "buildkite-agents"; + }; + + buildkite-besadii-config = { + file = secretFile "besadii"; + mode = "0440"; + group = "buildkite-agents"; + }; + + buildkite-private-key = { + file = secretFile "buildkite-ssh-private-key"; + mode = "0440"; + group = "buildkite-agents"; + }; + + gerrit-besadii-config = { + file = secretFile "besadii"; + owner = "git"; + }; + + gerrit-secrets = { + file = secretFile "gerrit-secrets"; + path = "/var/lib/gerrit/etc/secure.config"; + owner = "git"; + mode = "0400"; + }; + + clbot-ssh = { + file = secretFile "clbot-ssh"; + owner = "clbot"; + }; + + # Not actually a secret + nix-cache-pub = { + file = secretFile "nix-cache-pub"; + mode = "0444"; + }; + + depot-replica-key = { + file = secretFile "depot-replica-key"; + mode = "0500"; + owner = "git"; + group = "git"; + path = "/var/lib/git/.ssh/id_ed25519"; + }; + }; + # Automatically collect garbage from the Nix store. services.depot.automatic-gc = { enable = true; @@ -216,61 +315,96 @@ in { }; }; + # Start a ZNC instance which bounces for tvlbot and owothia. + services.znc = { + enable = true; + useLegacyConfig = false; + config = { + LoadModule = [ + "webadmin" + "adminlog" + ]; + + User.admin = { + Admin = true; + Pass.password = { + Method = "sha256"; + Hash = "bb00aa8239de484c2925b1c3f6a196fb7612633f001daa9b674f83abe7e1103f"; + Salt = "TiB0Ochb1CrtpMTl;2;j"; + }; + }; + + Listener.l = { + Host = "localhost"; + Port = 2627; # bncr + SSL = false; + }; + }; + }; + # Start the Gerrit->IRC bot services.depot.clbot = { enable = true; + channels = [ "#tvix-dev" "#tvl" ]; - # Almost all configuration values are already correct (well, duh), - # see //fun/clbot for details. + # See //fun/clbot for details. flags = { gerrit_host = "cl.tvl.fyi:29418"; gerrit_ssh_auth_username = "clbot"; - gerrit_ssh_auth_key = "/etc/secrets/clbot-key"; - irc_server = "znc.lukegb.com:6697"; + gerrit_ssh_auth_key = config.age.secretsDir + "/clbot-ssh"; + + irc_server = "localhost:${toString config.services.znc.config.Listener.l.Port}"; + irc_user = "tvlbot"; + irc_nick = "tvlbot"; notify_branches = "canon,refs/meta/config"; notify_repo = "depot"; # This secret is read from an environment variable, which is - # populated from /etc/secrets/clbot + # populated by a systemd EnvironmentFile. irc_pass = "$CLBOT_PASS"; }; - - channels = [ - "##tvl" - "##tvl-dev" - ]; }; services.depot = { # Run a SourceGraph code search instance sourcegraph.enable = true; + # Run a livegrep code search instance + livegrep.enable = true; + # Run the Panettone issue tracker panettone = { enable = true; dbUser = "panettone"; dbName = "panettone"; - secretsFile = "/etc/secrets/panettone"; - irccatChannel = "##tvl,##tvl-dev"; + irccatChannel = "#tvl"; }; # Run the first cursed bot (quote bot) paroxysm.enable = true; + # Run the second cursed bot + owothia = { + enable = true; + ircServer = "localhost"; + ircPort = config.services.znc.config.Listener.l.Port; + }; + # Run irccat to forward messages to IRC irccat = { enable = true; config = { tcp.listen = ":4722"; # "ircc" irc = { - server = "chat.freenode.net:6697"; - tls = true; + server = "localhost:${toString config.services.znc.config.Listener.l.Port}"; + tls = false; nick = "tvlbot"; - realname = "TVL Bot"; + # Note: irccat means 'ident' where it says 'realname', so + # this is critical for connecting to ZNC. + realname = "tvlbot"; channels = [ - "##tvl" - "##tvl-dev" + "#tvl" ]; }; }; @@ -278,11 +412,33 @@ in { # Run atward, the search engine redirection thing. atward.enable = true; + + # Run cgit & josh to serve git + cgit = { + enable = true; + user = "git"; # run as the same user as gerrit + }; + + josh.enable = true; + + # Configure backups to GleSYS + restic = { + enable = true; + paths = [ + "/var/backup/postgresql" + "/var/lib/grafana" + "/var/lib/znc" + ]; + }; + + # Run autosubmit bot for Gerrit + gerrit-autosubmit.enable = true; }; services.postgresql = { enable = true; enableTCPIP = true; + package = pkgs.postgresql_16; authentication = lib.mkForce '' local all all trust @@ -298,270 +454,199 @@ in { ensureUsers = [{ name = "panettone"; - ensurePermissions = { - "DATABASE panettone" = "ALL PRIVILEGES"; - }; + ensureDBOwnership = true; }]; }; services.postgresqlBackup = { enable = true; databases = [ - "tvldb" + "keycloak" "panettone" + "tvldb" ]; }; services.nix-serve = { enable = true; port = 6443; - secretKeyFile = "/etc/secrets/nix-cache-key.sec"; + secretKeyFile = config.age.secretsDir + "/nix-cache-priv"; bindAddress = "localhost"; }; - environment.systemPackages = with pkgs; [ + services.fail2ban.enable = true; + + environment.systemPackages = (with pkgs; [ + bat bb curl + direnv emacs-nox + fd git htop + hyperfine + jq nano - rxvt_unicode.terminfo + nvd + ripgrep + tree + unzip vim zfs zfstools - ]; + ]) ++ (with depot; [ + ops.deploy-whitby + ]); - # Run cgit for the depot. The onion here is nginx(thttpd(cgit)). - systemd.services.cgit = { - wantedBy = [ "multi-user.target" ]; - script = "${depot.web.cgit-taz}/bin/cgit-launch"; - - serviceConfig = { - Restart = "on-failure"; - User = "git"; - Group = "git"; - }; - }; - - # Regularly back up whitby to Google Cloud Storage. - systemd.services.restic = { - description = "Backups to Google Cloud Storage"; - script = "${pkgs.restic}/bin/restic backup /var/lib/gerrit /var/backup/postgresql /var/lib/grafana"; - - environment = { - GOOGLE_PROJECT_ID = "tazjins-infrastructure"; - GOOGLE_APPLICATION_CREDENTIALS = "/var/backup/restic/gcp-key.json"; - RESTIC_REPOSITORY = "gs:tvl-fyi-backups:/whitby"; - RESTIC_PASSWORD_FILE = "/var/backup/restic/secret"; - RESTIC_CACHE_DIR = "/var/backup/restic/cache"; - RESTIC_EXCLUDE_FILE = builtins.toFile "exclude-files" '' - /var/lib/gerrit/tmp - ''; - }; - }; - - systemd.timers.restic = { - wantedBy = [ "multi-user.target" ]; - timerConfig.OnCalendar = "hourly"; - }; - - services.journaldriver = { - enable = true; - googleCloudProject = "tvl-fyi"; - logStream = "whitby"; - applicationCredentials = "/var/lib/journaldriver/key.json"; - }; + # Required for prometheus to be able to scrape stats + services.nginx.statusPage = true; # Configure Prometheus & Grafana. Exporter configuration for # Prometheus is inside the respective service modules. services.prometheus = { enable = true; - exporters.node = { - enable = true; + retentionTime = "90d"; - enabledCollectors = [ - "logind" - "processes" - "systemd" - ]; + exporters = { + node = { + enable = true; + + enabledCollectors = [ + "logind" + "processes" + "systemd" + ]; + }; + + nginx = { + enable = true; + sslVerify = false; + constLabels = [ "host=whitby" ]; + }; }; scrapeConfigs = [{ job_name = "node"; scrape_interval = "5s"; static_configs = [{ - targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; + targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; + }]; + } + { + job_name = "nginx"; + scrape_interval = "5s"; + static_configs = [{ + targets = [ "localhost:${toString config.services.prometheus.exporters.nginx.port}" ]; + }]; }]; - }]; }; services.grafana = { enable = true; - port = 4723; # "graf" on phone keyboard - domain = "status.tvl.su"; - rootUrl = "https://status.tvl.su"; - analytics.reporting.enable = false; - extraOptions = let - options = { - auth = { - generic_oauth = { - enabled = true; - client_id = "OAUTH-TVL-grafana-f1A1EmHLDT"; - scopes = "openid profile email"; - name = "TVL"; - email_attribute_path = "mail"; - login_attribute_path = "sub"; - name_attribute_path = "displayName"; - auth_url = "https://login.tvl.fyi/oidc/authorize"; - token_url = "https://login.tvl.fyi/oidc/accessToken"; - api_url = "https://login.tvl.fyi/oidc/profile"; - - # Give lukegb, grfn, tazjin "Admin" rights. - role_attribute_path = "((sub == 'lukegb' || sub == 'grfn' || sub == 'tazjin') && 'Admin') || 'Editor'"; - - # Allow creating new Grafana accounts from OAuth accounts. - allow_sign_up = true; - }; - anonymous = { - enabled = true; - org_name = "The Virus Lounge"; - org_role = "Viewer"; - }; - basic.enabled = false; - oauth_auto_login = true; - disable_login_form = true; - }; + + settings = { + server = { + http_port = 4723; # "graf" on phone keyboard + domain = "status.tvl.su"; + root_url = "https://status.tvl.su"; + }; + + analytics.reporting_enabled = false; + + "auth.generic_oauth" = { + enabled = true; + client_id = "grafana"; + scopes = "openid profile email"; + name = "TVL"; + email_attribute_path = "mail"; + login_attribute_path = "sub"; + name_attribute_path = "displayName"; + auth_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/auth"; + token_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/token"; + api_url = "https://auth.tvl.fyi/auth/realms/TVL/protocol/openid-connect/userinfo"; + + # Give lukegb, aspen, tazjin "Admin" rights. + role_attribute_path = "((sub == 'lukegb' || sub == 'aspen' || sub == 'tazjin') && 'Admin') || 'Editor'"; + + # Allow creating new Grafana accounts from OAuth accounts. + allow_sign_up = true; + }; + + "auth.anonymous" = { + enabled = true; + org_name = "The Virus Lounge"; + org_role = "Viewer"; }; - inherit (builtins) typeOf replaceStrings listToAttrs concatLists; - inherit (lib) toUpper mapAttrsToList nameValuePair concatStringsSep; - - # Take ["auth" "generic_oauth" "enabled"] and turn it into OPTIONS_GENERIC_OAUTH_ENABLED. - encodeName = raw: replaceStrings ["."] ["_"] (toUpper (concatStringsSep "_" raw)); - - # Turn an option value into a string, but we want bools to be sensible strings and not "1" or "". - optionToString = value: - if (typeOf value) == "bool" then - if value then "true" else "false" - else builtins.toString value; - - # Turn an nested options attrset into a flat listToAttrs-compatible list. - encodeOptions = prefix: inp: concatLists (mapAttrsToList (name: value: - if (typeOf value) == "set" - then encodeOptions (prefix ++ [name]) value - else [ (nameValuePair (encodeName (prefix ++ [name])) (optionToString value)) ] - ) inp); - in listToAttrs (encodeOptions [] options); + + "auth.basic".enabled = false; + + auth = { + oauth_auto_login = true; + disable_login_form = true; + }; + }; provision = { enable = true; - datasources = [{ + datasources.settings.datasources = [{ name = "Prometheus"; type = "prometheus"; url = "http://localhost:9090"; }]; }; }; - # Contains GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET. - systemd.services.grafana.serviceConfig.EnvironmentFile = "/etc/secrets/grafana"; - - security.sudo.extraRules = [ - { - groups = ["wheel"]; - commands = [{ command = "ALL"; options = ["NOPASSWD"]; }]; - } - ]; - - users = { - users.tazjin = { - isNormalUser = true; - extraGroups = [ "git" "wheel" ]; - shell = pkgs.fish; - openssh.authorizedKeys.keys = depot.users.tazjin.keys.all; - }; - - users.lukegb = { - isNormalUser = true; - extraGroups = [ "git" "wheel" ]; - openssh.authorizedKeys.keys = depot.users.lukegb.keys.all; - }; - users.grfn = { - isNormalUser = true; - extraGroups = [ "git" "wheel" ]; - openssh.authorizedKeys.keys = [ - depot.users.grfn.keys.whitby - ]; - }; - - users.isomer = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.isomer.keys.all; - }; - - users.riking = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.riking.keys.u2f ++ depot.users.riking.keys.passworded; - }; - - users.edef = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.edef.keys.all; - }; + # Contains GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET. + systemd.services.grafana.serviceConfig.EnvironmentFile = config.age.secretsDir + "/grafana"; - users.qyliss = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.qyliss.keys.all; - }; + services.keycloak = { + enable = true; - users.eta = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.eta.keys.whitby; + settings = { + http-port = 5925; # kycl + hostname = "auth.tvl.fyi"; + http-relative-path = "/auth"; + proxy = "edge"; }; - users.cynthia = { - isNormalUser = true; # I'm normal OwO :3 - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.cynthia.keys.all; + database = { + type = "postgresql"; + passwordFile = config.age.secretsDir + "/keycloak-db"; + createLocally = false; }; + }; - users.firefly = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.firefly.keys.whitby; - }; + # Join TVL Tailscale network at net.tvl.fyi + services.tailscale = { + enable = true; + useRoutingFeatures = "server"; # for exit-node usage + }; - users.sterni = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.sterni.keys.all; - }; + # Allow Keycloak access to the LDAP module by forcing in the JVM + # configuration + systemd.services.keycloak.environment.PREPEND_JAVA_OPTS = + "--add-exports=java.naming/com.sun.jndi.ldap=ALL-UNNAMED"; - users.flokli = { - isNormalUser = true; - extraGroups = [ "git" ]; - openssh.authorizedKeys.keys = depot.users.flokli.keys.all; - }; + security.sudo.extraRules = [ + { + groups = [ "wheel" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + } + ]; + users = { # Set up a user & group for git shenanigans - groups.git = {}; + groups.git = { }; users.git = { group = "git"; - isNormalUser = false; + isSystemUser = true; createHome = true; home = "/var/lib/git"; }; }; - security.acme = { - acceptTerms = true; - email = "certs@tvl.fyi"; - }; + zramSwap.enable = true; system.stateVersion = "20.03"; } |