feat(ops/terraform): add module for deploying NixOS system closures r/5867

This module makes it fairly easy to deploy NixOS system closures using
Terraform, while properly separating the evaluation of a
derivation (to determine whether a deploy is needed) from the building
and copying of the closure itself.

This has been on my stack for a while. It was originally developed for
Resoptima, who agreed to open-sourcing it in depot back when we
completed our work with them. Their contribution has been acknowledged
in the README.

Co-Authored-By: Florian Klink <flokli@flokli.de>
Change-Id: Ica4c170658cd25f1fb7072c9a45735fcc4351474
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7950
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI

5 files changed, 187 insertions, 0 deletions

diff --git a/ops/terraform/README.md b/ops/terraform/README.md
new file mode 100644
index 0000000000..9ff6c23d47
--- /dev/null
+++ b/ops/terraform/README.md
@@ -0,0 +1,5 @@
+//ops/terraform
+===============
+
+This folder contains Terraform modules and other related
+Terraform-tooling by TVL.
diff --git a/ops/terraform/deploy-nixos/README.md b/ops/terraform/deploy-nixos/README.md
new file mode 100644
index 0000000000..a51e6bdb5f
--- /dev/null
+++ b/ops/terraform/deploy-nixos/README.md
@@ -0,0 +1,45 @@
+deploy-nixos
+============
+
+This is a Terraform module to deploy a NixOS system closure to a
+remote machine.
+
+The system closure must be accessible by Nix-importing the repository
+root and building a specific attribute
+(e.g. `nix-build -A ops.machines.machine-name`).
+
+The target machine must be accessible normally over SSH, and an SSH
+key must be used for access.
+
+Notably this module separates the evaluation of the system closure from building
+and deploying it, and uses the closure's derivation hash to determine whether a
+deploy is necessary.
+
+## Usage example:
+
+```terraform
+module "deploy_somehost" {
+  source              = "git::https://code.tvl.fyi/depot.git:/ops/terraform/deploy-nixos.git"
+  attrpath            = "ops.nixos.somehost"
+  target_name         = "somehost"
+  target_host         = "somehost.tvl.su"
+  target_user         = "someone"
+  target_user_ssh_key = tls_private_key.somehost.private_key_pem
+}
+```
+
+## Future work
+
+Several things can be improved about this module, for example:
+
+* The repository root (relative to which the attribute path is evaluated) could
+  be made configurable.
+
+* The remote system closure could be discovered to restore remote system state
+  after manual deploys on the target (i.e. "stomping" of changes).
+
+More ideas and contributions are, of course, welcome.
+
+## Acknowledgements
+
+Development of this module was sponsored by [Resoptima](https://resoptima.com/).
diff --git a/ops/terraform/deploy-nixos/main.tf b/ops/terraform/deploy-nixos/main.tf
new file mode 100644
index 0000000000..3ff7bfc3f3
--- /dev/null
+++ b/ops/terraform/deploy-nixos/main.tf
@@ -0,0 +1,98 @@
+# This module deploys a NixOS host by building a system closure
+# located at the specified attribute in the current repository.
+#
+# The closure's derivation path is persisted in the Terraform state to
+# determine after Nix evaluation whether the system closure has
+# changed and needs to be built/deployed.
+#
+# The system configuration is then built (or substituted) on the
+# machine that runs `terraform apply`, then copied and activated on
+# the target machine using `nix-copy-closure`.
+
+variable "attrpath" {
+  description = "attribute set path pointing to the NixOS system closure"
+  type        = string
+}
+
+variable "target_name" {
+  description = "unique name of the target machine"
+  type        = string
+}
+
+variable "target_host" {
+  description = "address (IP or hostname) at which the target is reachable"
+  type        = string
+}
+
+variable "target_user" {
+  description = "username on the target machine"
+  type        = string
+}
+
+variable "target_user_ssh_key" {
+  description = "SSH key to use for connecting to the target"
+  type        = string
+  sensitive   = true
+}
+
+# Fetch the derivation hash for the NixOS system.
+data "external" "nixos_system" {
+  program = ["${path.module}/nixos-eval.sh"]
+
+  query = {
+    attrpath = var.attrpath
+  }
+}
+
+# Deploy the NixOS configuration if anything changed.
+resource "null_resource" "nixos_deploy" {
+  connection {
+    type        = "ssh"
+    host        = var.target_host
+    user        = var.target_user
+    private_key = var.target_user_ssh_key
+  }
+
+  # 1. Wait for SSH to become available.
+  provisioner "remote-exec" {
+    inline = ["true"]
+  }
+
+  # 2. Build NixOS system.
+  provisioner "local-exec" {
+    command = "nix-build ${data.external.nixos_system.result.drv} --no-out-link"
+  }
+
+  # 3. Copy closure to the target.
+  provisioner "local-exec" {
+    command = "${path.module}/nixos-copy.sh"
+
+    environment = {
+      SYSTEM_DRV  = data.external.nixos_system.result.drv
+      TARGET_HOST = var.target_host
+      DEPLOY_KEY  = var.target_user_ssh_key
+      TARGET_USER = var.target_user
+    }
+  }
+
+  # 4. Activate closure on the target.
+  provisioner "remote-exec" {
+    inline = [
+      "set -eu",
+      "SYSTEM=$(nix-build ${data.external.nixos_system.result.drv} --no-out-link)",
+      "sudo nix-env --profile /nix/var/nix/profiles/system --set $SYSTEM",
+      "sudo $SYSTEM/bin/switch-to-configuration switch",
+    ]
+  }
+
+  triggers = {
+    nixos_drv   = data.external.nixos_system.result.drv
+    attrpath    = var.attrpath
+    target_host = var.target_host
+    target_name = var.target_name
+  }
+}
+
+output "nixos_drv" {
+  value = data.external.nixos_system.result
+}
diff --git a/ops/terraform/deploy-nixos/nixos-copy.sh b/ops/terraform/deploy-nixos/nixos-copy.sh
new file mode 100755
index 0000000000..1c6b8df024
--- /dev/null
+++ b/ops/terraform/deploy-nixos/nixos-copy.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# Copies a NixOS system to a target host, using the provided key.
+set -ueo pipefail
+
+scratch="$(mktemp -d)"
+trap 'rm -rf -- "${scratch}"' EXIT
+
+echo -n "$DEPLOY_KEY" > $scratch/id_deploy
+chmod 0600 $scratch/id_deploy
+
+export NIX_SSHOPTS="\
+    -o StrictHostKeyChecking=no\
+    -o UserKnownHostsFile=/dev/null\
+    -o GlobalKnownHostsFile=/dev/null\
+    -o IdentityFile=$scratch/id_deploy"
+
+nix-copy-closure \
+  --to ${TARGET_USER}@${TARGET_ADDRESS} \
+  ${SYSTEM_DRV} \
+  --gzip \
+  --include-outputs \
+  --use-substitutes
diff --git a/ops/terraform/deploy-nixos/nixos-eval.sh b/ops/terraform/deploy-nixos/nixos-eval.sh
new file mode 100755
index 0000000000..dd15784b1b
--- /dev/null
+++ b/ops/terraform/deploy-nixos/nixos-eval.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Builds a NixOS system configuration at the given attribute path.
+set -ueo pipefail
+
+# Load input variables from Terraform. jq's @sh format takes care of
+# escaping.
+eval "$(jq -r '@sh "ATTRPATH=\(.attrpath)"')"
+
+# Evaluate the system derivation.
+# TODO: configurable REPO_ROOT
+REPO_ROOT=$(git rev-parse --show-toplevel)
+SYSTEM_DRV=$(nix-instantiate -A "${ATTRPATH}" "${REPO_ROOT}")
+
+# Return system derivation back to Terraform.
+jq -n --arg drv "$SYSTEM_DRV" '{"drv":$drv}'