feat(wpcarro/diogenes): Nixify diogenes's Terraform configuration r/3531

TL;DR:
- Define googleCloudVM function to provision NixOS VMs on Google Cloud.
- Consume googleCloudVM in diogenes/default.nix
- Define README.md for basic usage instructions (subject to change).
- Delete diogenes's HCL
- Remove `diogenesSystem` from meta.targets

I'm still having trouble with DNS:
- I need to transfer the Google Domains config to Cloud DNS
- `host billandhiscomputer.com` is NXDOMAIN, so I don't trust my tf DNS config
- This is preventing me from getting SSL certs, which blocks my website, quassel

Change-Id: If315876c96298e83a5953f13b62784d2f65a1024
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4747
Tested-by: BuildkiteCI
Reviewed-by: wpcarro <wpcarro@gmail.com>
Autosubmit: wpcarro <wpcarro@gmail.com>

5 files changed, 331 insertions, 178 deletions

diff --git a/users/wpcarro/nixos/default.nix b/users/wpcarro/nixos/default.nix
index 15de8c0e113c..b1a878c95a07 100644
--- a/users/wpcarro/nixos/default.nix
+++ b/users/wpcarro/nixos/default.nix
@@ -1,9 +1,8 @@
-{ depot, lib, ... }:
+{ depot, ... }:
 
 let systemFor = sys: (depot.ops.nixos.nixosFor sys).system;
 in {
-  diogenesSystem = systemFor depot.users.wpcarro.nixos.diogenes;
   marcusSystem = systemFor depot.users.wpcarro.nixos.marcus;
 
-  meta.targets = [ "diogenesSystem" "marcusSystem" ];
+  meta.targets = [ "marcusSystem" ];
 }
diff --git a/users/wpcarro/nixos/diogenes/README.md b/users/wpcarro/nixos/diogenes/README.md
new file mode 100644
index 000000000000..75741f0244de
--- /dev/null
+++ b/users/wpcarro/nixos/diogenes/README.md
@@ -0,0 +1,17 @@
+# diogenes
+
+diogenes is a NixOS machine deployed on a Google VM. It hosts
+https://wpcarro.dev.
+
+## Deployment
+
+I manage diogenes's deployment with Terraform. My current workflow looks like
+this (highly subject to change):
+
+```shell
+cd /tmp/terraform # or any directory that hosts terraform state
+outpath=$(nix-build /depot -A users.wpcarro.nixos.diogenes)
+cp <out-path> .
+nix-shell -p terraform google-cloud-sdk # gcloud to authenticate if necessary
+terraform init/apply
+```
diff --git a/users/wpcarro/nixos/diogenes/default.nix b/users/wpcarro/nixos/diogenes/default.nix
index decf632c1b64..7cc6443a568d 100644
--- a/users/wpcarro/nixos/diogenes/default.nix
+++ b/users/wpcarro/nixos/diogenes/default.nix
@@ -1,119 +1,150 @@
 { depot, pkgs, ... }:
-{ ... }:
 
 let
   inherit (depot.users) wpcarro;
-in {
-  imports = [
-    "${depot.path}/ops/modules/quassel.nix"
-    (pkgs.path + "/nixos/modules/virtualisation/google-compute-image.nix")
-  ];
+  name = "diogenes";
+  domainName = "billandhiscomputer.com";
+in wpcarro.terraform.googleCloudVM {
+  project = "wpcarros-infrastructure";
+  name = "diogenes";
+  region = "us-central1";
+  zone = "us-central1-a";
+
+  # DNS configuration
+  extraConfig = {
+    resource.google_dns_managed_zone."${name}" = {
+      inherit name;
+      dns_name = "${domainName}.";
+    };
 
-  networking = {
-    hostName = "diogenes";
-    firewall.enable = false;
+    resource.google_dns_record_set."${name}" = {
+      name = "${name}.${domainName}.";
+      type = "A";
+      ttl = 300; # 5m
+      managed_zone = "\${google_dns_managed_zone.${name}.name}";
+      rrdatas = ["\${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip}"];
+    };
   };
 
-  # Use the TVL binary cache
-  tvl.cache.enable = true;
+  configuration = {
+    imports = [
+      "${depot.path}/ops/modules/quassel.nix"
+    ];
+
+    networking = {
+      firewall.allowedTCPPorts = [
+        22   # ssh
+        80   # http
+        443  # https
+        6698 # quassel
+      ];
+      firewall.allowedUDPPortRanges = [
+        { from = 60000; to = 61000; } # mosh
+      ];
+    };
 
-  # Use 100G volume for /nix
-  fileSystems."/nix" = {
-    device = "/dev/disk/by-uuid/62396bde-9002-4025-83eb-2a6c731b7adc";
-    fsType = "ext4";
-  };
+    # Use the TVL binary cache
+    tvl.cache.enable = true;
 
-  users = {
-    mutableUsers = true;
     users = {
-      wpcarro = {
-        isNormalUser = true;
-        extraGroups = [ "wheel" "quassel" ];
-        openssh.authorizedKeys.keys = wpcarro.keys.all;
-        shell = pkgs.fish;
+      mutableUsers = true;
+      users = {
+        root = {
+          openssh.authorizedKeys.keys = wpcarro.keys.all;
+        };
+        wpcarro = {
+          isNormalUser = true;
+          extraGroups = [ "wheel" "quassel" ];
+          openssh.authorizedKeys.keys = wpcarro.keys.all;
+          shell = pkgs.fish;
+        };
       };
     };
-  };
-
-  security = {
-    acme = {
-      acceptTerms = true;
-      email = "wpcarro@gmail.com";
-    };
 
-    sudo.wheelNeedsPassword = false;
-  };
-
-  programs = wpcarro.common.programs // {
-    mosh.enable = true;
-  };
-
-  # I won't have an Emacs server running on diogenes, and I'll likely be in an
-  # SSH session from within vterm. As such, Vim is one of the few editors that I
-  # tolerably navigate this way.
-  environment.variables = {
-    EDITOR = "vim";
-  };
-
-  environment.systemPackages = wpcarro.common.shell-utils;
+    security = {
+      acme = {
+        acceptTerms = true;
+        email = "wpcarro@gmail.com";
+      };
 
-  services = wpcarro.common.services // {
-    depot.quassel = {
-      enable = true;
-      acmeHost = "wpcarro.dev";
-      bindAddresses = [
-        "0.0.0.0"
-      ];
+      sudo.wheelNeedsPassword = false;
     };
 
-    depot.auto-deploy = {
-      enable = true;
-      interval = "1h";
+    programs = wpcarro.common.programs // {
+      mosh.enable = true;
     };
 
-    journaldriver = {
-      enable = true;
-      logStream = "home";
-      googleCloudProject = "wpcarros-infrastructure";
-      applicationCredentials = "/etc/gcp/key.json";
+    # I won't have an Emacs server running on diogenes, and I'll likely be in an
+    # SSH session from within vterm. As such, Vim is one of the few editors that
+    # I tolerably navigate this way.
+    environment.variables = {
+      EDITOR = "vim";
     };
 
-    nginx = {
-      enable = true;
-      enableReload = true;
-
-      recommendedTlsSettings = true;
-      recommendedGzipSettings = true;
-      recommendedProxySettings = true;
-
-      # for journaldriver
-      commonHttpConfig = ''
-        log_format json_combined escape=json
-        '{'
-            '"remote_addr":"$remote_addr",'
-            '"method":"$request_method",'
-            '"host":"$host",'
-            '"uri":"$request_uri",'
-            '"status":$status,'
-            '"request_size":$request_length,'
-            '"response_size":$body_bytes_sent,'
-            '"response_time":$request_time,'
-            '"referrer":"$http_referer",'
-            '"user_agent":"$http_user_agent"'
-        '}';
-
-        access_log syslog:server=unix:/dev/log,nohostname json_combined;
-      '';
-
-      virtualHosts = {
-        "wpcarro.dev" = {
-          addSSL = true;
-          enableACME = true;
-          root = wpcarro.website.root;
-        };
-      };
+    environment.systemPackages = wpcarro.common.shell-utils;
+
+    services = wpcarro.common.services // {
+      # TODO(wpcarro): Re-enable this when rebuild-system better supports
+      # terraform deployments.
+      # depot.auto-deploy = {
+      #   enable = true;
+      #   interval = "1h";
+      # };
+
+      # TODO(wpcarro): Re-enable this after debugging ACME and NXDOMAIN.
+      # depot.quassel = {
+      #   enable = true;
+      #   acmeHost = domainName;
+      #   bindAddresses = [
+      #     "0.0.0.0"
+      #   ];
+      # };
+      #
+      # journaldriver = {
+      #   enable = true;
+      #   logStream = "home";
+      #   googleCloudProject = "wpcarros-infrastructure";
+      #   applicationCredentials = "/etc/gcp/key.json";
+      # };
+      #
+      #
+      # nginx = {
+      #   enable = true;
+      #   enableReload = true;
+      #
+      #   recommendedTlsSettings = true;
+      #   recommendedGzipSettings = true;
+      #   recommendedProxySettings = true;
+      #
+      #   # for journaldriver
+      #   commonHttpConfig = ''
+      #     log_format json_combined escape=json
+      #     '{'
+      #         '"remote_addr":"$remote_addr",'
+      #         '"method":"$request_method",'
+      #         '"host":"$host",'
+      #         '"uri":"$request_uri",'
+      #         '"status":$status,'
+      #         '"request_size":$request_length,'
+      #         '"response_size":$body_bytes_sent,'
+      #         '"response_time":$request_time,'
+      #         '"referrer":"$http_referer",'
+      #         '"user_agent":"$http_user_agent"'
+      #     '}';
+      #
+      #     access_log syslog:server=unix:/dev/log,nohostname json_combined;
+      #   '';
+      #
+      #   virtualHosts = {
+      #     "${domainName}" = {
+      #       addSSL = true;
+      #       enableACME = true;
+      #       root = wpcarro.website.root;
+      #     };
+      #   };
+      # };
     };
-  };
 
-  system.stateVersion = "21.11";
+    system.stateVersion = "21.11";
+  };
 }
diff --git a/users/wpcarro/terraform/default.nix b/users/wpcarro/terraform/default.nix
new file mode 100644
index 000000000000..b3c16144a209
--- /dev/null
+++ b/users/wpcarro/terraform/default.nix
@@ -0,0 +1,185 @@
+{ depot, pkgs, lib, ... }:
+
+let
+  inherit (builtins) concatLists concatStringsSep toJSON unsafeDiscardStringContext;
+  inherit (depot.users) wpcarro;
+  inherit (pkgs) writeText;
+
+  images = import "${pkgs.path}/nixos/modules/virtualisation/gce-images.nix";
+  nixosImage = images."20.09";
+in {
+  googleCloudVM = {
+    project,
+    name,
+    region,
+    zone,
+    configuration,
+    extraConfig ? {},
+  }: let
+    inherit (configuration.users.users) root;
+    inherit (configuration.networking) firewall;
+
+    # Convert NixOS-style port numbers to Terraform-style.
+    asStrings = xs: map toString xs;
+    asRanges = xs: map (x: "${toString x.from}-${toString x.to}") xs;
+
+    sshKeys = concatStringsSep "\n"
+      (map (key: "root:${key}") root.openssh.authorizedKeys.keys);
+
+    os = depot.ops.nixos.nixosFor (_: {
+      imports = [
+        "${pkgs.path}/nixos/modules/virtualisation/google-compute-image.nix"
+        configuration
+      ];
+
+      networking.hostName = name;
+
+      fileSystems."/nix" = {
+        device = "/dev/disk/by-label/google-${name}-disk";
+        fsType = "ext4";
+      };
+    });
+
+    osRoot = os.config.system.build.toplevel;
+    osPath = unsafeDiscardStringContext (toString osRoot.outPath);
+    drvPath = unsafeDiscardStringContext (toString osRoot.drvPath);
+  in writeText "terraform.tf.json" (toJSON (lib.recursiveUpdate extraConfig {
+    provider.google = {
+      inherit project region zone;
+    };
+
+    resource.google_compute_instance."${name}" = {
+      inherit name zone;
+      machine_type = "e2-standard-2";
+  
+      tags = [
+        "http-server"
+        "https-server"
+        "${name}-firewall"
+      ];
+  
+      boot_disk = {
+        device_name = "boot";
+        initialize_params = {
+          size = 10;
+          image = "projects/nixos-cloud/global/images/${nixosImage.name}";
+        };
+      };
+  
+      attached_disk = {
+        source = "\${google_compute_disk.${name}.id}";
+        device_name = "${name}-disk";
+      };
+  
+      network_interface = {
+        network = "default";
+        subnetwork = "default";
+        access_config = {};
+      };
+  
+      # Copy root's SSH keys from the NixOS configuration and expose them to the
+      # metadata server.
+      metadata = {
+        inherit sshKeys;
+        ssh-keys = sshKeys;
+
+        # NixOS's fetch-instance-ssh-keys.bash relies on these fields being
+        # available on the metadata server.
+        ssh_host_ed25519_key = "\${tls_private_key.${name}.private_key_pem}";
+        ssh_host_ed25519_key_pub = "\${tls_private_key.${name}.public_key_pem}";
+
+        # Even though we have SSH access, having oslogin can still be useful for
+        # troubleshooting in the browser if for some reason SSH isn't working as
+        # expected.
+        enable-oslogin = "TRUE";
+      };
+  
+      service_account.scopes = ["cloud-platform"];
+    };
+
+    resource.tls_private_key."${name}" = {
+      algorithm = "ECDSA";
+      ecdsa_curve = "P384";
+    };
+
+    resource.google_compute_firewall."${name}" = {
+      name = "${name}-firewall";
+      network = "default";
+
+      # Read the firewall configuration from the NixOS configuration.
+      allow = [
+        {
+          protocol = "tcp";
+          ports = concatLists [
+            (asStrings (firewall.allowedTCPPorts or []))
+            (asRanges (firewall.allowedTCPPortRanges or []))
+          ];
+        }
+        {
+          protocol = "udp";
+          ports = concatLists [
+            (asStrings (firewall.allowedUDPPorts or []))
+            (asRanges (firewall.allowedUDPPortRanges or []))
+          ];
+        }
+      ];
+      source_tags = ["${name}-firewall"];
+    };
+  
+    resource.google_compute_disk."${name}" = {
+      inherit zone;
+      name = "${name}-disk";
+      size = 100;
+    };
+
+    resource.null_resource.deploy_nixos = {
+      triggers = {
+        # Redeploy when the NixOS configuration changes.
+        os = "${osPath}";
+        # Redeploy when a new machine is provisioned.
+        machine_id = "\${google_compute_instance.${name}.id}";
+      };
+
+      connection = {
+        host = "\${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip}";
+      };
+
+      provisioner = [
+        { remote-exec.inline = ["true"]; }
+        {
+          local-exec.command = ''
+            export PATH="${pkgs.openssh}/bin:$PATH"
+
+            scratch="$(mktemp -d)"
+            function cleanup() {
+              rm -rf $scratch
+            }
+            trap cleanup EXIT
+
+            # write out ssh key
+            echo -n "''${tls_private_key.${name}.private_key_pem}" > $scratch/id_rsa.pem
+            chmod 0600 $scratch/id_rsa.pem
+
+            export NIX_SSHOPTS="\
+              -o StrictHostKeyChecking=no\
+              -o UserKnownHostsFile=/dev/null\
+              -o GlobalKnownHostsFile=/dev/null\
+              -o IdentityFile=$scratch/id_rsa.pem
+            "
+
+            nix-build ${drvPath}
+            nix-copy-closure --to \
+              root@''${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip} \
+              ${osPath} --gzip --use-substitutes
+          '';
+        }
+        {
+          remote-exec.inline = [
+            "nix-env --profile /nix/var/nix/profiles/system --set ${osPath}"
+            "${osPath}/bin/switch-to-configuration switch"
+          ];
+        }
+      ];
+    };
+  }));
+}
diff --git a/users/wpcarro/terraform/gcp.tf b/users/wpcarro/terraform/gcp.tf
deleted file mode 100644
index 03bb721c9d35..000000000000
--- a/users/wpcarro/terraform/gcp.tf
+++ /dev/null
@@ -1,79 +0,0 @@
-provider "google" {
-  project = "wpcarros-infrastructure"
-  region  = "us-central1"
-  zone    = "us-central1-a"
-}
-
-data "google_compute_default_service_account" "default" {}
-
-resource "google_compute_instance" "default" {
-  name         = "diogenes-2"
-  machine_type = "e2-standard-2"
-  zone         = "us-central1-a"
-  hostname     = "diogenes.wpcarro.dev"
-
-  tags = [
-    "http-server",
-    "https-server",
-    "diogenes-firewall"
-  ]
-
-  boot_disk {
-    device_name = "boot"
-
-    initialize_params {
-      size  = 10
-      image = "projects/nixos-cloud/global/images/nixos-image-20-09-3531-3858fbc08e6-x86-64-linux"
-    }
-  }
-
-  attached_disk {
-    source      = "diogenes-2-disk"
-    device_name = "diogenes-2-disk"
-  }
-
-  network_interface {
-    network    = "default"
-    subnetwork = "default"
-
-    access_config {}
-  }
-
-  metadata = {
-    # sshKeys is deprecated, but the GCE NixOS image relies on it, so we need
-    # both values:
-    # - deprecation: https://cloud.google.com/compute/docs/metadata/default-metadata-values
-    # - NixOS bug: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/fetch-instance-ssh-keys.bash#L14
-    ssh-keys = "wpcarro:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJkNQJBXekuSzZJ8+gxT+V1+eXTm3hYsfigllr/ARXkf wpcarro@gmail.com"
-    sshKeys  = "wpcarro:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJkNQJBXekuSzZJ8+gxT+V1+eXTm3hYsfigllr/ARXkf wpcarro@gmail.com"
-  }
-
-  service_account {
-    scopes = ["cloud-platform"]
-  }
-}
-
-resource "google_compute_firewall" "default" {
-  name    = "diogenes-firewall"
-  network = "default"
-
-  allow {
-    protocol = "tcp"
-    ports    = ["6698"]
-  }
-
-  allow {
-    protocol = "udp"
-    ports = [
-      "60000-61000" # mosh
-    ]
-  }
-
-  source_tags = ["diogenes-firewall"]
-}
-
-resource "google_compute_disk" "default" {
-  name = "diogenes-2-disk"
-  zone = "us-central1-a"
-  size = 100
-}
\ No newline at end of file