From dbca46d05205bdaab07d4faf1899a6452f34dd0f Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 29 Jan 2023 19:44:23 +0300 Subject: feat(ops/terraform): add module for deploying NixOS system closures This module makes it fairly easy to deploy NixOS system closures using Terraform, while properly separating the evaluation of a derivation (to determine whether a deploy is needed) from the building and copying of the closure itself. This has been on my stack for a while. It was originally developed for Resoptima, who agreed to open-sourcing it in depot back when we completed our work with them. Their contribution has been acknowledged in the README. Co-Authored-By: Florian Klink Change-Id: Ica4c170658cd25f1fb7072c9a45735fcc4351474 Reviewed-on: https://cl.tvl.fyi/c/depot/+/7950 Reviewed-by: tazjin Tested-by: BuildkiteCI --- ops/terraform/README.md | 5 ++ ops/terraform/deploy-nixos/README.md | 45 +++++++++++++++ ops/terraform/deploy-nixos/main.tf | 98 ++++++++++++++++++++++++++++++++ ops/terraform/deploy-nixos/nixos-copy.sh | 23 ++++++++ ops/terraform/deploy-nixos/nixos-eval.sh | 16 ++++++ 5 files changed, 187 insertions(+) create mode 100644 ops/terraform/README.md create mode 100644 ops/terraform/deploy-nixos/README.md create mode 100644 ops/terraform/deploy-nixos/main.tf create mode 100755 ops/terraform/deploy-nixos/nixos-copy.sh create mode 100755 ops/terraform/deploy-nixos/nixos-eval.sh (limited to 'ops') diff --git a/ops/terraform/README.md b/ops/terraform/README.md new file mode 100644 index 000000000000..9ff6c23d4701 --- /dev/null +++ b/ops/terraform/README.md @@ -0,0 +1,5 @@ +//ops/terraform +=============== + +This folder contains Terraform modules and other related +Terraform-tooling by TVL. diff --git a/ops/terraform/deploy-nixos/README.md b/ops/terraform/deploy-nixos/README.md new file mode 100644 index 000000000000..a51e6bdb5f1e --- /dev/null +++ b/ops/terraform/deploy-nixos/README.md @@ -0,0 +1,45 @@ +deploy-nixos +============ + +This is a Terraform module to deploy a NixOS system closure to a +remote machine. + +The system closure must be accessible by Nix-importing the repository +root and building a specific attribute +(e.g. `nix-build -A ops.machines.machine-name`). + +The target machine must be accessible normally over SSH, and an SSH +key must be used for access. + +Notably this module separates the evaluation of the system closure from building +and deploying it, and uses the closure's derivation hash to determine whether a +deploy is necessary. + +## Usage example: + +```terraform +module "deploy_somehost" { + source = "git::https://code.tvl.fyi/depot.git:/ops/terraform/deploy-nixos.git" + attrpath = "ops.nixos.somehost" + target_name = "somehost" + target_host = "somehost.tvl.su" + target_user = "someone" + target_user_ssh_key = tls_private_key.somehost.private_key_pem +} +``` + +## Future work + +Several things can be improved about this module, for example: + +* The repository root (relative to which the attribute path is evaluated) could + be made configurable. + +* The remote system closure could be discovered to restore remote system state + after manual deploys on the target (i.e. "stomping" of changes). + +More ideas and contributions are, of course, welcome. + +## Acknowledgements + +Development of this module was sponsored by [Resoptima](https://resoptima.com/). diff --git a/ops/terraform/deploy-nixos/main.tf b/ops/terraform/deploy-nixos/main.tf new file mode 100644 index 000000000000..3ff7bfc3f30a --- /dev/null +++ b/ops/terraform/deploy-nixos/main.tf @@ -0,0 +1,98 @@ +# This module deploys a NixOS host by building a system closure +# located at the specified attribute in the current repository. +# +# The closure's derivation path is persisted in the Terraform state to +# determine after Nix evaluation whether the system closure has +# changed and needs to be built/deployed. +# +# The system configuration is then built (or substituted) on the +# machine that runs `terraform apply`, then copied and activated on +# the target machine using `nix-copy-closure`. + +variable "attrpath" { + description = "attribute set path pointing to the NixOS system closure" + type = string +} + +variable "target_name" { + description = "unique name of the target machine" + type = string +} + +variable "target_host" { + description = "address (IP or hostname) at which the target is reachable" + type = string +} + +variable "target_user" { + description = "username on the target machine" + type = string +} + +variable "target_user_ssh_key" { + description = "SSH key to use for connecting to the target" + type = string + sensitive = true +} + +# Fetch the derivation hash for the NixOS system. +data "external" "nixos_system" { + program = ["${path.module}/nixos-eval.sh"] + + query = { + attrpath = var.attrpath + } +} + +# Deploy the NixOS configuration if anything changed. +resource "null_resource" "nixos_deploy" { + connection { + type = "ssh" + host = var.target_host + user = var.target_user + private_key = var.target_user_ssh_key + } + + # 1. Wait for SSH to become available. + provisioner "remote-exec" { + inline = ["true"] + } + + # 2. Build NixOS system. + provisioner "local-exec" { + command = "nix-build ${data.external.nixos_system.result.drv} --no-out-link" + } + + # 3. Copy closure to the target. + provisioner "local-exec" { + command = "${path.module}/nixos-copy.sh" + + environment = { + SYSTEM_DRV = data.external.nixos_system.result.drv + TARGET_HOST = var.target_host + DEPLOY_KEY = var.target_user_ssh_key + TARGET_USER = var.target_user + } + } + + # 4. Activate closure on the target. + provisioner "remote-exec" { + inline = [ + "set -eu", + "SYSTEM=$(nix-build ${data.external.nixos_system.result.drv} --no-out-link)", + "sudo nix-env --profile /nix/var/nix/profiles/system --set $SYSTEM", + "sudo $SYSTEM/bin/switch-to-configuration switch", + ] + } + + triggers = { + nixos_drv = data.external.nixos_system.result.drv + attrpath = var.attrpath + target_host = var.target_host + target_name = var.target_name + } +} + +output "nixos_drv" { + value = data.external.nixos_system.result +} diff --git a/ops/terraform/deploy-nixos/nixos-copy.sh b/ops/terraform/deploy-nixos/nixos-copy.sh new file mode 100755 index 000000000000..1c6b8df024d2 --- /dev/null +++ b/ops/terraform/deploy-nixos/nixos-copy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Copies a NixOS system to a target host, using the provided key. +set -ueo pipefail + +scratch="$(mktemp -d)" +trap 'rm -rf -- "${scratch}"' EXIT + +echo -n "$DEPLOY_KEY" > $scratch/id_deploy +chmod 0600 $scratch/id_deploy + +export NIX_SSHOPTS="\ + -o StrictHostKeyChecking=no\ + -o UserKnownHostsFile=/dev/null\ + -o GlobalKnownHostsFile=/dev/null\ + -o IdentityFile=$scratch/id_deploy" + +nix-copy-closure \ + --to ${TARGET_USER}@${TARGET_ADDRESS} \ + ${SYSTEM_DRV} \ + --gzip \ + --include-outputs \ + --use-substitutes diff --git a/ops/terraform/deploy-nixos/nixos-eval.sh b/ops/terraform/deploy-nixos/nixos-eval.sh new file mode 100755 index 000000000000..dd15784b1b49 --- /dev/null +++ b/ops/terraform/deploy-nixos/nixos-eval.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Builds a NixOS system configuration at the given attribute path. +set -ueo pipefail + +# Load input variables from Terraform. jq's @sh format takes care of +# escaping. +eval "$(jq -r '@sh "ATTRPATH=\(.attrpath)"')" + +# Evaluate the system derivation. +# TODO: configurable REPO_ROOT +REPO_ROOT=$(git rev-parse --show-toplevel) +SYSTEM_DRV=$(nix-instantiate -A "${ATTRPATH}" "${REPO_ROOT}") + +# Return system derivation back to Terraform. +jq -n --arg drv "$SYSTEM_DRV" '{"drv":$drv}' -- cgit 1.4.1