From 8fb1ff3f2549a3ebe8ba7c8e57756392350afe6e Mon Sep 17 00:00:00 2001 From: William Carroll Date: Tue, 1 Feb 2022 13:34:49 -0800 Subject: feat(wpcarro/diogenes): Support rebuild-diogenes - deploy-diogenes: terraform updates + NixOS rebuilds - rebuild-diogenes: NixOS rebuilds Change-Id: Ibd6db7115d9919fa44ee9d318f88e1bf29e2bdce Reviewed-on: https://cl.tvl.fyi/c/depot/+/5160 Tested-by: BuildkiteCI Reviewed-by: wpcarro Autosubmit: wpcarro --- users/wpcarro/bin/__dispatch.sh | 3 + users/wpcarro/bin/rebuild-diogenes | 1 + users/wpcarro/nixos/default.nix | 40 +++++- users/wpcarro/terraform/default.nix | 255 ++++++++++++++++++------------------ 4 files changed, 166 insertions(+), 133 deletions(-) create mode 120000 users/wpcarro/bin/rebuild-diogenes (limited to 'users/wpcarro') diff --git a/users/wpcarro/bin/__dispatch.sh b/users/wpcarro/bin/__dispatch.sh index 17556ad2e020..6da9a1c416cf 100755 --- a/users/wpcarro/bin/__dispatch.sh +++ b/users/wpcarro/bin/__dispatch.sh @@ -12,6 +12,9 @@ case "${TARGET_TOOL}" in deploy-diogenes) attr="users.wpcarro.nixos.deploy-diogenes" ;; + rebuild-diogenes) + attr="users.wpcarro.nixos.rebuild-diogenes" + ;; import-gpg) attr="users.wpcarro.configs.import-gpg" ;; diff --git a/users/wpcarro/bin/rebuild-diogenes b/users/wpcarro/bin/rebuild-diogenes new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/users/wpcarro/bin/rebuild-diogenes @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/users/wpcarro/nixos/default.nix b/users/wpcarro/nixos/default.nix index aa1dfea55e92..de8bb028f1a2 100644 --- a/users/wpcarro/nixos/default.nix +++ b/users/wpcarro/nixos/default.nix @@ -1,22 +1,48 @@ { depot, pkgs, ... }: -let systemFor = sys: (depot.ops.nixos.nixosFor sys).system; -in { +let + inherit (depot.users.wpcarro.nixos) diogenes; + systemFor = sys: (depot.ops.nixos.nixosFor sys).system; +in +{ marcusSystem = systemFor depot.users.wpcarro.nixos.marcus; + + # Apply terraform updates and rebuild NixOS for diogenes. deploy-diogenes = pkgs.writeShellScriptBin "deploy-diogenes" '' set -euo pipefail readonly TF_STATE_DIR=/depot/users/wpcarro/terraform rm -f $TF_STATE_DIR/*.json - readonly STORE_PATH="$(nix-build /depot -A users.wpcarro.nixos.diogenes)" + readonly STORE_PATH="${diogenes.json}" + # We can't use the result symlink because terraform looks for a *.json file + # in the current working directory. cp $STORE_PATH $TF_STATE_DIR - function cleanup() { - rm -f "$TF_STATE_DIR/$(basename $STORE_PATH)" - } + if [ ! -d $TF_STATE_DIR/.terraform ]; then + ${pkgs.terraform}/bin/terraform -chdir="$TF_STATE_DIR" init + fi + + # function cleanup() { + # rm -f "$TF_STATE_DIR/$(basename $STORE_PATH)" + # } + # trap cleanup EXIT - trap cleanup EXIT ${pkgs.terraform}/bin/terraform -chdir="$TF_STATE_DIR" apply ''; + # Rebuild NixOS for diogenes without applying terraform updates. + rebuild-diogenes = pkgs.writeShellScriptBin "rebuild-diogenes" '' + set -euo pipefail + readonly target="root@''${1}" + + # We need to call nix-build here on the drvPath because it may not be in + # /nix/store yet. + readonly STORE_PATH="$(nix-build ${diogenes.drvPath} --no-out-link --show-trace)" + nix-copy-closure --to $target ${diogenes.osPath} \ + --gzip --use-substitutes $STORE_PATH + + ssh $target 'nix-env --profile /nix/var/nix/profiles/system --set ${diogenes.osPath}' + ssh $target '${diogenes.osPath}/bin/switch-to-configuration switch' + ''; + meta.ci.targets = [ "marcusSystem" ]; } diff --git a/users/wpcarro/terraform/default.nix b/users/wpcarro/terraform/default.nix index d73d46dbf91e..55b68451b11a 100644 --- a/users/wpcarro/terraform/default.nix +++ b/users/wpcarro/terraform/default.nix @@ -47,143 +47,146 @@ in osPath = unsafeDiscardStringContext (toString osRoot.outPath); drvPath = unsafeDiscardStringContext (toString osRoot.drvPath); in - writeText "terraform.tf.json" (toJSON (lib.recursiveUpdate extraConfig { - provider.google = { - inherit project region zone; - }; - - resource.google_compute_instance."${name}" = { - inherit name zone; - machine_type = "e2-standard-2"; - - tags = [ - "http-server" - "https-server" - "${name}-firewall" - ]; + { + inherit drvPath osPath; + json = writeText "terraform.tf.json" (toJSON (lib.recursiveUpdate extraConfig { + provider.google = { + inherit project region zone; + }; - boot_disk = { - device_name = "boot"; - initialize_params = { - size = 10; - image = "projects/nixos-cloud/global/images/${nixosImage.name}"; + resource.google_compute_instance."${name}" = { + inherit name zone; + machine_type = "e2-standard-2"; + + tags = [ + "http-server" + "https-server" + "${name}-firewall" + ]; + + boot_disk = { + device_name = "boot"; + initialize_params = { + size = 10; + image = "projects/nixos-cloud/global/images/${nixosImage.name}"; + }; }; + + attached_disk = { + source = "\${google_compute_disk.${name}.id}"; + device_name = "${name}-disk"; + }; + + network_interface = { + network = "default"; + subnetwork = "default"; + access_config = { }; + }; + + # Copy root's SSH keys from the NixOS configuration and expose them to the + # metadata server. + metadata = { + inherit sshKeys; + ssh-keys = sshKeys; + + # NixOS's fetch-instance-ssh-keys.bash relies on these fields being + # available on the metadata server. + ssh_host_ed25519_key = "\${tls_private_key.${name}.private_key_pem}"; + ssh_host_ed25519_key_pub = "\${tls_private_key.${name}.public_key_pem}"; + + # Even though we have SSH access, having oslogin can still be useful for + # troubleshooting in the browser if for some reason SSH isn't working as + # expected. + enable-oslogin = "TRUE"; + }; + + service_account.scopes = [ "cloud-platform" ]; }; - attached_disk = { - source = "\${google_compute_disk.${name}.id}"; - device_name = "${name}-disk"; + resource.tls_private_key."${name}" = { + algorithm = "ECDSA"; + ecdsa_curve = "P384"; }; - network_interface = { + resource.google_compute_firewall."${name}" = { + name = "${name}-firewall"; network = "default"; - subnetwork = "default"; - access_config = { }; - }; - # Copy root's SSH keys from the NixOS configuration and expose them to the - # metadata server. - metadata = { - inherit sshKeys; - ssh-keys = sshKeys; - - # NixOS's fetch-instance-ssh-keys.bash relies on these fields being - # available on the metadata server. - ssh_host_ed25519_key = "\${tls_private_key.${name}.private_key_pem}"; - ssh_host_ed25519_key_pub = "\${tls_private_key.${name}.public_key_pem}"; - - # Even though we have SSH access, having oslogin can still be useful for - # troubleshooting in the browser if for some reason SSH isn't working as - # expected. - enable-oslogin = "TRUE"; + # Read the firewall configuration from the NixOS configuration. + allow = [ + { + protocol = "tcp"; + ports = concatLists [ + (asStrings (firewall.allowedTCPPorts or [ ])) + (asRanges (firewall.allowedTCPPortRanges or [ ])) + ]; + } + { + protocol = "udp"; + ports = concatLists [ + (asStrings (firewall.allowedUDPPorts or [ ])) + (asRanges (firewall.allowedUDPPortRanges or [ ])) + ]; + } + ]; + source_ranges = [ "0.0.0.0/0" ]; }; - service_account.scopes = [ "cloud-platform" ]; - }; - - resource.tls_private_key."${name}" = { - algorithm = "ECDSA"; - ecdsa_curve = "P384"; - }; - - resource.google_compute_firewall."${name}" = { - name = "${name}-firewall"; - network = "default"; - - # Read the firewall configuration from the NixOS configuration. - allow = [ - { - protocol = "tcp"; - ports = concatLists [ - (asStrings (firewall.allowedTCPPorts or [ ])) - (asRanges (firewall.allowedTCPPortRanges or [ ])) - ]; - } - { - protocol = "udp"; - ports = concatLists [ - (asStrings (firewall.allowedUDPPorts or [ ])) - (asRanges (firewall.allowedUDPPortRanges or [ ])) - ]; - } - ]; - source_ranges = [ "0.0.0.0/0" ]; - }; - - resource.google_compute_disk."${name}" = { - inherit zone; - name = "${name}-disk"; - size = 100; - }; - - resource.null_resource.deploy_nixos = { - triggers = { - # Redeploy when the NixOS configuration changes. - os = "${osPath}"; - # Redeploy when a new machine is provisioned. - machine_id = "\${google_compute_instance.${name}.id}"; + resource.google_compute_disk."${name}" = { + inherit zone; + name = "${name}-disk"; + size = 100; }; - connection = { - host = "\${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip}"; - }; + resource.null_resource.deploy_nixos = { + triggers = { + # Redeploy when the NixOS configuration changes. + os = "${osPath}"; + # Redeploy when a new machine is provisioned. + machine_id = "\${google_compute_instance.${name}.id}"; + }; - provisioner = [ - { remote-exec.inline = [ "true" ]; } - { - local-exec.command = '' - export PATH="${pkgs.openssh}/bin:$PATH" - - scratch="$(mktemp -d)" - function cleanup() { - rm -rf $scratch - } - trap cleanup EXIT - - # write out ssh key - echo -n "''${tls_private_key.${name}.private_key_pem}" > $scratch/id_rsa.pem - chmod 0600 $scratch/id_rsa.pem - - export NIX_SSHOPTS="\ - -o StrictHostKeyChecking=no\ - -o UserKnownHostsFile=/dev/null\ - -o GlobalKnownHostsFile=/dev/null\ - -o IdentityFile=$scratch/id_rsa.pem - " - - nix-build ${drvPath} - nix-copy-closure --to \ - root@''${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip} \ - ${osPath} --gzip --use-substitutes - ''; - } - { - remote-exec.inline = [ - "nix-env --profile /nix/var/nix/profiles/system --set ${osPath}" - "${osPath}/bin/switch-to-configuration switch" - ]; - } - ]; - }; - })); + connection = { + host = "\${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip}"; + }; + + provisioner = [ + { remote-exec.inline = [ "true" ]; } + { + local-exec.command = '' + export PATH="${pkgs.openssh}/bin:$PATH" + + scratch="$(mktemp -d)" + function cleanup() { + rm -rf $scratch + } + trap cleanup EXIT + + # write out ssh key + echo -n "''${tls_private_key.${name}.private_key_pem}" > $scratch/id_rsa.pem + chmod 0600 $scratch/id_rsa.pem + + export NIX_SSHOPTS="\ + -o StrictHostKeyChecking=no\ + -o UserKnownHostsFile=/dev/null\ + -o GlobalKnownHostsFile=/dev/null\ + -o IdentityFile=$scratch/id_rsa.pem + " + + nix-build ${drvPath} + nix-copy-closure --to \ + root@''${google_compute_instance.${name}.network_interface[0].access_config[0].nat_ip} \ + ${osPath} --gzip --use-substitutes + ''; + } + { + remote-exec.inline = [ + "nix-env --profile /nix/var/nix/profiles/system --set ${osPath}" + "${osPath}/bin/switch-to-configuration switch" + ]; + } + ]; + }; + })); + }; } -- cgit 1.4.1