feat(wpcarro/diogenes): Support rebuild-diogenes r/3807

- deploy-diogenes: terraform updates + NixOS rebuilds
- rebuild-diogenes: NixOS rebuilds

Change-Id: Ibd6db7115d9919fa44ee9d318f88e1bf29e2bdce
Reviewed-on: https://cl.tvl.fyi/c/depot/+/5160
Tested-by: BuildkiteCI
Reviewed-by: wpcarro <wpcarro@gmail.com>
Autosubmit: wpcarro <wpcarro@gmail.com>

4 files changed, 166 insertions, 133 deletions

diff --git a/users/wpcarro/bin/__dispatch.sh b/users/wpcarro/bin/__dispatch.sh
index 17556ad2e020..6da9a1c416cf 100755
--- a/users/wpcarro/bin/__dispatch.sh
+++ b/users/wpcarro/bin/__dispatch.sh
@@ -12,6 +12,9 @@ case "${TARGET_TOOL}" in
   deploy-diogenes)
     attr="users.wpcarro.nixos.deploy-diogenes"
     ;;
+  rebuild-diogenes)
+    attr="users.wpcarro.nixos.rebuild-diogenes"
+    ;;
   import-gpg)
     attr="users.wpcarro.configs.import-gpg"
     ;;
diff --git a/users/wpcarro/bin/rebuild-diogenes b/users/wpcarro/bin/rebuild-diogenes
new file mode 120000
index 000000000000..8390ec9c9652
--- /dev/null
+++ b/users/wpcarro/bin/rebuild-diogenes
@@ -0,0 +1 @@
+__dispatch.sh
\ No newline at end of file
diff --git a/users/wpcarro/nixos/default.nix b/users/wpcarro/nixos/default.nix
index aa1dfea55e92..de8bb028f1a2 100644
--- a/users/wpcarro/nixos/default.nix
+++ b/users/wpcarro/nixos/default.nix
@@ -1,22 +1,48 @@
 { depot, pkgs, ... }:
 
-let systemFor = sys: (depot.ops.nixos.nixosFor sys).system;
-in {
+let
+  inherit (depot.users.wpcarro.nixos) diogenes;
+  systemFor = sys: (depot.ops.nixos.nixosFor sys).system;
+in
+{
   marcusSystem = systemFor depot.users.wpcarro.nixos.marcus;
+
+  # Apply terraform updates and rebuild NixOS for diogenes.
   deploy-diogenes = pkgs.writeShellScriptBin "deploy-diogenes" ''
     set -euo pipefail
     readonly TF_STATE_DIR=/depot/users/wpcarro/terraform
     rm -f $TF_STATE_DIR/*.json
-    readonly STORE_PATH="$(nix-build /depot -A users.wpcarro.nixos.diogenes)"
+    readonly STORE_PATH="${diogenes.json}"
+    # We can't use the result symlink because terraform looks for a *.json file
+    # in the current working directory.
     cp $STORE_PATH $TF_STATE_DIR
 
-    function cleanup() {
-      rm -f "$TF_STATE_DIR/$(basename $STORE_PATH)"
-    }
+    if [ ! -d $TF_STATE_DIR/.terraform ]; then
+      ${pkgs.terraform}/bin/terraform -chdir="$TF_STATE_DIR" init
+    fi
+
+    # function cleanup() {
+    #   rm -f "$TF_STATE_DIR/$(basename $STORE_PATH)"
+    # }
+    # trap cleanup EXIT
 
-    trap cleanup EXIT
     ${pkgs.terraform}/bin/terraform -chdir="$TF_STATE_DIR" apply
   '';
 
+  # Rebuild NixOS for diogenes without applying terraform updates.
+  rebuild-diogenes = pkgs.writeShellScriptBin "rebuild-diogenes" ''
+    set -euo pipefail
+    readonly target="root@''${1}"
+
+    # We need to call nix-build here on the drvPath because it may not be in
+    # /nix/store yet.
+    readonly STORE_PATH="$(nix-build ${diogenes.drvPath} --no-out-link --show-trace)"
+    nix-copy-closure --to $target ${diogenes.osPath} \
+      --gzip --use-substitutes $STORE_PATH
+
+    ssh $target 'nix-env --profile /nix/var/nix/profiles/system --set ${diogenes.osPath}'
+    ssh $target '${diogenes.osPath}/bin/switch-to-configuration switch'
+  '';
+
   meta.ci.targets = [ "marcusSystem" ];
 }
diff --git a/users/wpcarro/terraform/default.nix b/users/wpcarro/terraform/default.nix
index d73d46dbf91e..55b68451b11a 100644
--- a/users/wpcarro/terraform/default.nix
+++ b/users/wpcarro/terraform/default.nix
@@ -47,143 +47,146 @@ in
       osPath = unsafeDiscardStringContext (toString osRoot.outPath);
       drvPath = unsafeDiscardStringContext (toString osRoot.drvPath);
     in
-    writeText "terraform.tf.json" (toJSON (lib.recursiveUpdate extraConfig {
-      provider.google = {
-        inherit project region zone;
-      };
-
-      resource.google_compute_instance."${name}" = {
-        inherit name zone;
-        machine_type = "e2-standard-2";
-
-        tags = [
-          "http-server"
-          "https-server"
-          "${name}-firewall"
-        ];
+    {
+      inherit drvPath osPath;
+      json = writeText "terraform.tf.json" (toJSON (lib.recursiveUpdate extraConfig {
+        provider.google = {
+          inherit project region zone;
+        };
 
-        boot_disk = {
-          device_name = "boot";
-          initialize_params = {
-            size = 10;
-            image = "projects/nixos-cloud/global/images/${nixosImage.name}";
+        resource.google_compute_instance."${name}" = {
+          inherit name zone;
+          machine_type = "e2-standard-2";
+
+          tags = [
+            "http-server"
+            "https-server"
+            "${name}-firewall"
+          ];
+
+          boot_disk = {
+            device_name = "boot";
+            initialize_params = {
+              size = 10;
+              image = "projects/nixos-cloud/global/images/${nixosImage.name}";
+            };
           };
+
+          attached_disk = {
+            source = "\${google_compute_disk.${name}.id}";
+            device_name = "${name}-disk";
+          };
+
+          network_interface = {
+            network = "default";
+            subnetwork = "default";
+            access_config = { };
+          };
+
+          # Copy root's SSH keys from the NixOS configuration and expose them to the
+          # metadata server.
+          metadata = {
+            inherit sshKeys;
+            ssh-keys = sshKeys;
+
+            # NixOS's fetch-instance-ssh-keys.bash relies on these fields being
+            # available on the metadata server.
+            ssh_host_ed25519_key = "\${tls_private_key.${name}.private_key_pem}";
+            ssh_host_ed25519_key_pub = "\${tls_private_key.${name}.public_key_pem}";
+
+            # Even though we have SSH access, having oslogin can still be useful for
+            # troubleshooting in the browser if for some reason SSH isn't working as
+            # expected.
+            enable-oslogin = "TRUE";
+          };
+
+          service_account.scopes = [ "cloud-platform" ];
         };
 
-        attached_disk = {
-          source = "\${google_compute_disk.${name}.id}";
-          device_name = "${name}-disk";
+        resource.tls_private_key."${name}" = {
+          algorithm = "ECDSA";
+          ecdsa_curve = "P384";
         };
 
-        network_interface = {
+        resource.google_compute_firewall."${name}" = {
+          name = "${name}-firewall";
           network = "default";
-          subnetwork = "default";
-          access_config = { };
-        };
 
-        # Copy root's SSH keys from the NixOS configuration and expose them to the
-        # metadata server.
-        metadata = {
-          inherit sshKeys;
-          ssh-keys = sshKeys;
-
-          # NixOS's fetch-instance-ssh-keys.bash relies on these fields being
-          # available on the metadata server.
-          ssh_host_ed25519_key = "\${tls_private_key.${name}.private_key_pem}";
-          ssh_host_ed25519_key_pub = "\${tls_private_key.${name}.public_key_pem}";
-
-          # Even though we have SSH access, having oslogin can still be useful for
-          # troubleshooting in the browser if for some reason SSH isn't working as
-          # expected.
-          enable-oslogin = "TRUE";
+          # Read the firewall configuration from the NixOS configuration.
+          allow = [
+            {
+              protocol = "tcp";
+              ports = concatLists [
+                (asStrings (firewall.allowedTCPPorts or [ ]))
+                (asRanges (firewall.allowedTCPPortRanges or [ ]))
+              ];
+            }
+            {
+              protocol = "udp";
+              ports = concatLists [
+                (asStrings (firewall.allowedUDPPorts or [ ]))
+                (asRanges (firewall.allowedUDPPortRanges or [ ]))
+              ];
+            }
+          ];
+          source_ranges = [ "0.0.0.0/0" ];
         };
 
-        service_account.scopes = [ "cloud-platform" ];
-      };
-
-      resource.tls_private_key."${name}" = {
-        algorithm = "ECDSA";
-        ecdsa_curve = "P384";
-      };
-
-      resource.google_compute_firewall."${name}" = {
-        name = "${name}-firewall";
-        network = "default";
-
-        # Read the firewall configuration from the NixOS configuration.
-        allow = [
-          {
-            protocol = "tcp";
-            ports = concatLists [
-              (asStrings (firewall.allowedTCPPorts or [ ]))
-              (asRanges (firewall.allowedTCPPortRanges or [ ]))
-            ];
-          }
-          {
-            protocol = "udp";
-            ports = concatLists [
-              (asStrings (firewall.allowedUDPPorts or [ ]))
-              (asRanges (firewall.allowedUDPPortRanges or [ ]))
-            ];
-          }
-        ];
-        source_ranges = [ "0.0.0.0/0" ];
-      };
-
-      resource.google_compute_disk."${name}" = {
-        inherit zone;
-        name = "${name}-disk";
-        size = 100;
-      };
-
-      resource.null_resource.deploy_nixos = {
-        triggers = {
-          # Redeploy when the NixOS configuration changes.
-          os = "${osPath}";
-          # Redeploy when a new machine is provisioned.
-          machine_id = "\${google_compute_instance.${name}.id}";
+        resource.google_compute_disk."${name}" = {
+          inherit zone;
+          name = "${name}-disk";
+          size = 100;
         };
 
-        connection = {
-          host = "\${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip}";
-        };
+        resource.null_resource.deploy_nixos = {
+          triggers = {
+            # Redeploy when the NixOS configuration changes.
+            os = "${osPath}";
+            # Redeploy when a new machine is provisioned.
+            machine_id = "\${google_compute_instance.${name}.id}";
+          };
 
-        provisioner = [
-          { remote-exec.inline = [ "true" ]; }
-          {
-            local-exec.command = ''
-              export PATH="${pkgs.openssh}/bin:$PATH"
-
-              scratch="$(mktemp -d)"
-              function cleanup() {
-                rm -rf $scratch
-              }
-              trap cleanup EXIT
-
-              # write out ssh key
-              echo -n "''${tls_private_key.${name}.private_key_pem}" > $scratch/id_rsa.pem
-              chmod 0600 $scratch/id_rsa.pem
-
-              export NIX_SSHOPTS="\
-                -o StrictHostKeyChecking=no\
-                -o UserKnownHostsFile=/dev/null\
-                -o GlobalKnownHostsFile=/dev/null\
-                -o IdentityFile=$scratch/id_rsa.pem
-              "
-
-              nix-build ${drvPath}
-              nix-copy-closure --to \
-                root@''${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip} \
-                ${osPath} --gzip --use-substitutes
-            '';
-          }
-          {
-            remote-exec.inline = [
-              "nix-env --profile /nix/var/nix/profiles/system --set ${osPath}"
-              "${osPath}/bin/switch-to-configuration switch"
-            ];
-          }
-        ];
-      };
-    }));
+          connection = {
+            host = "\${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip}";
+          };
+
+          provisioner = [
+            { remote-exec.inline = [ "true" ]; }
+            {
+              local-exec.command = ''
+                export PATH="${pkgs.openssh}/bin:$PATH"
+
+                scratch="$(mktemp -d)"
+                function cleanup() {
+                  rm -rf $scratch
+                }
+                trap cleanup EXIT
+
+                # write out ssh key
+                echo -n "''${tls_private_key.${name}.private_key_pem}" > $scratch/id_rsa.pem
+                chmod 0600 $scratch/id_rsa.pem
+
+                export NIX_SSHOPTS="\
+                  -o StrictHostKeyChecking=no\
+                  -o UserKnownHostsFile=/dev/null\
+                  -o GlobalKnownHostsFile=/dev/null\
+                  -o IdentityFile=$scratch/id_rsa.pem
+                "
+
+                nix-build ${drvPath}
+                nix-copy-closure --to \
+                  root@''${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip} \
+                  ${osPath} --gzip --use-substitutes
+              '';
+            }
+            {
+              remote-exec.inline = [
+                "nix-env --profile /nix/var/nix/profiles/system --set ${osPath}"
+                "${osPath}/bin/switch-to-configuration switch"
+              ];
+            }
+          ];
+        };
+      }));
+    };
 }