feat(grfn/bbbg): Production deployment r/3456

Start of a production deployment of the app with nixos+terraform, using
provisioners and null-resources to provision nixos machines a'la espes.

Change-Id: I2ddaed76d0037dadbf9fc9e2ee27e9e67a852228
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4695
Reviewed-by: grfn <grfn@gws.fyi>
Autosubmit: grfn <grfn@gws.fyi>
Tested-by: BuildkiteCI

9 files changed, 458 insertions, 19 deletions

diff --git a/users/grfn/bbbg/default.nix b/users/grfn/bbbg/default.nix
index 90f112bf2866..5b5b4badbf4b 100644
--- a/users/grfn/bbbg/default.nix
+++ b/users/grfn/bbbg/default.nix
@@ -1,4 +1,4 @@
-{ depot, pkgs, ... }:
+args@{ depot, pkgs, ... }:
 
 with pkgs.lib;
 
@@ -12,6 +12,7 @@ in rec {
   meta.targets = [
     "db-util"
     "server"
+    "tf"
   ];
 
   depsPaths = deps.makePaths {};
@@ -75,4 +76,6 @@ in rec {
   server = pkgs.writeShellScriptBin "bbbg-server" ''
     exec ${pkgs.openjdk17_headless}/bin/java -jar ${server-jar} "$@"
   '';
+
+  tf = import ./tf.nix args;
 }
diff --git a/users/grfn/bbbg/shell.nix b/users/grfn/bbbg/shell.nix
index 195562519ed4..48bcd73759d0 100644
--- a/users/grfn/bbbg/shell.nix
+++ b/users/grfn/bbbg/shell.nix
@@ -11,6 +11,15 @@ mkShell {
     openjdk11_headless
     postgresql_12
     nix-prefetch-git
+    (writeShellScriptBin "terraform" ''
+      set -e
+      module=$(nix-build ~/code/depot -A users.grfn.bbbg.tf.module)
+      rm -f ~/tfstate/bbbg/*.json
+      cp ''${module}/*.json ~/tfstate/bbbg
+      exec ${depot.users.grfn.bbbg.tf.terraform}/bin/terraform \
+        -chdir=/home/grfn/tfstate/bbbg \
+        "$@"
+    '')
   ];
 
   PGHOST = "localhost";
diff --git a/users/grfn/bbbg/tf.nix b/users/grfn/bbbg/tf.nix
new file mode 100644
index 000000000000..71f07d343501
--- /dev/null
+++ b/users/grfn/bbbg/tf.nix
@@ -0,0 +1,93 @@
+{ depot, ... }:
+
+let
+  inherit (depot.users.grfn)
+    terraform
+  ;
+
+in terraform.workspace "bbbg" {
+  plugins = (p: with p; [
+    aws
+    cloudflare
+  ]);
+} {
+  machine = terraform.nixosMachine {
+    name = "bbbg";
+    instanceType = "t3a.small";
+    rootVolumeSizeGb = 250;
+    extraIngressPorts = [ 80 443 ];
+    configuration = { pkgs, lib, config, depot, ... }: {
+      imports = [
+        ./module.nix
+        "${depot.third_party.agenix.src}/modules/age.nix"
+      ];
+
+      services.openssh.enable = true;
+
+      services.nginx = {
+        enable = true;
+        recommendedTlsSettings = true;
+        recommendedOptimisation = true;
+        recommendedGzipSettings = true;
+        recommendedProxySettings = true;
+      };
+
+      networking.firewall.enable = false;
+
+      programs.zsh.enable = true;
+
+      users.users.grfn = {
+        isNormalUser = true;
+        initialPassword = "password";
+        extraGroups = [
+          "wheel"
+          "networkmanager"
+          "audio"
+          "docker"
+        ];
+        shell = pkgs.zsh;
+        openssh.authorizedKeys.keys = [
+          depot.users.grfn.keys.main
+        ];
+      };
+
+      security.sudo.extraRules = [{
+        groups = ["wheel"];
+        commands = [{ command = "ALL"; options = ["NOPASSWD"]; }];
+      }];
+
+      nix.gc = {
+        automatic = true;
+        dates = "weekly";
+        options = "--delete-older-than 30d";
+      };
+
+      age.secrets = {
+        bbbg.file =
+          depot.users.grfn.secrets."bbbg.age";
+      };
+
+      services.bbbg.enable = true;
+      services.bbbg.database.enable = true;
+      services.bbbg.proxy.enable = true;
+      services.bbbg.domain = "bbbg.gws.fyi";
+
+      security.acme.email = "root@gws.fyi";
+      security.acme.acceptTerms = true;
+    };
+  };
+
+  dns = {
+    data.cloudflare_zone.gws-fyi = {
+      name = "gws.fyi";
+    };
+
+    resource.cloudflare_record.bbbg = {
+      zone_id = "\${data.cloudflare_zone.gws-fyi.id}";
+      name = "bbbg";
+      type = "A";
+      value = "\${aws_instance.bbbg_machine.public_ip}";
+      proxied = false;
+    };
+  };
+}
diff --git a/users/grfn/secrets/bbbg.age b/users/grfn/secrets/bbbg.age
index d2d4c7362597..6c15dcdf7361 100644
--- a/users/grfn/secrets/bbbg.age
+++ b/users/grfn/secrets/bbbg.age
@@ -1,10 +1,12 @@
 age-encryption.org/v1
--> ssh-ed25519 CpJBgQ dHPaZt3ZRV6rBPQrqiEpKXd48OjUC1joVIm/ZHcimVQ
-Q8JwGJ91nsxspJFwZaq2BENdJYHxdHG30Ef0/Cae58M
--> ssh-ed25519 LfBFbQ oN98wLqM69Kv2Ldg31v0eBNtfpNP4nbyqAC+gCOT3yI
-U8weIdIqhGs2eoKXqCxO8zHe2Ddo5fVJ5ZYua/hcBs8
--> \Z^u8-grease ., ,^=lH#0> +P=Z," d
-fwUdQTFyoVYOmMUWN2nQ9JWg+Mj0iF325eJaEYkWTNvDZfUGioravnCEQxAErbAN
-S1v0wgUUM8/ja3uI
---- erMVG5PLHMBECjcKtR+OLq5hYa+6dS4gPsQ5CzQByQ0
-S�8�Y�"g|D��Z��0�X�	�1�gg�����|�.�]&�m=��4O���-���T=Em�8(\�b���D����~��~+��h��a�~��Re�����W�#�-�5���b���fO`�m�4��<'�|U��8"<D����2>��\ө3$@�ϔ�8;�|:u	WKz@�%#���NE?�+�!1�xN������8h>
\ No newline at end of file
+-> ssh-ed25519 CpJBgQ 6vLlq2WEcn6TE0rgahQyl7CYhCF3uiBD3hOnZkHswmM
+BSUiKPdDWMhYbi/+j9Kw5YDEOvjaickYQuhpWkhLktQ
+-> ssh-ed25519 LfBFbQ mQJfyk35Ghd7UWouPlq4kTIFFwlRGh24r0kvJUgUbBw
+eYpBJEG9Cdc2qHI0maFpp9/2o30R0KGLRSQ7DzsVaZ0
+-> ssh-ed25519 lZtaEQ npyXpqTMWITvRVfPwEQ1rXJ0sxnJvurLOfeiE07m92E
+oCXVRGOegBgQUJof8UHJsDdMyNsx6X575Rd4mWZ9LRk
+-> ;^O0_l-grease
+sseb4RnQz93Wlgs5B0PE+j7AzFyMkzHjFbn9sCn0UA
+--- Bqq0uedob5/kJOSoavN7Aq1fH7QVNW134M3uS6u2lFA
+�r3?u�j���F !�_��R�/�#BQ����
+�Nה�"��V1�mw�l��y��'s� ���P���^�6��K{��
��t�3m��%��'zOo���8^SJ���xꨯR=�
����ŕzEz*Ѥ���>�g뻐�%>\)Lj0���5VQ8�/H�XO�G��s7g��DGI�s��N��piuYFj?x;]
\ No newline at end of file
diff --git a/users/grfn/secrets/cloudflare.age b/users/grfn/secrets/cloudflare.age
index 1c9fa3ca6bf6..e2f6e9360385 100644
--- a/users/grfn/secrets/cloudflare.age
+++ b/users/grfn/secrets/cloudflare.age
@@ -1,9 +1,9 @@
 age-encryption.org/v1
--> ssh-ed25519 CpJBgQ w4W+pzmVIEMF0uZN7KZMAppJaLjEeDKoe7i9LGayKDQ
-Rd8k+3csmbZQIrp09ZUfCAOZVwI0BZ6hCBN3nkZQMp4
--> ssh-ed25519 LfBFbQ dyv1splvcftMd1zWDkPBfsgvXxH5neZlO7ZjrhyzNHI
-N/kqc/luOl8lsZcbaxF8/3ULsL78zvZhkiCarohe+G4
--> \w7t-grease lo&b JZpCA
-nN2lH0W9+zulMjZMLPMk61+xsrQ
---- voTpUbu8OiJQyuKB7tIOvlErgY0jg2w7N3MehD5FIdM
-&c�z�l�	�|K��M�~��2�e���
��
�UN8�P�~}�*�h���SYJJ���FɊ�o��c=�L���`z���O7����K��gZ�����.�aXD�HЦ���8�7�������8
\ No newline at end of file
+-> ssh-ed25519 CpJBgQ tWx7wXCFjOOfD0wKRHHvLUdR+SF0i43xvnQG9GKurnk
+NRh7kSn7wqw80Y9EFr9Ccft+zYMadXZhYNPEaQlQXtQ
+-> ssh-ed25519 LfBFbQ SPQMLC3Ehw00IG1CcbcLFZI2tHy89fjRgVgH4Iw2iBM
+oo2gT9472/DFRoZ6TYxhnM9ylRUNzoS8mLQYvn+4OSM
+-> D[7+*-grease `>j ~Jk Dz%o vaKET3
+TkKVm8IpqfiVzETAi9+zuUtCdkReB+lHtthwNw
+--- 3iOmY4TNICMi/Fz7k8pmoZlFym9uQBWNtHNlizoAMaM
+ZPzQ6���5���AT��I���;�;Зy5]��k�^!�`��t��$�R�ւ�t��K)�<��k���_�#XmAS�pU1���@��)��c�ֺ�q��j1z�,H���g��:�
\ No newline at end of file
diff --git a/users/grfn/secrets/secrets.nix b/users/grfn/secrets/secrets.nix
index 03871cc5cd4c..2208ba6893fa 100644
--- a/users/grfn/secrets/secrets.nix
+++ b/users/grfn/secrets/secrets.nix
@@ -1,9 +1,10 @@
 let
   grfn = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMcBGBoWd5pPIIQQP52rcFOQN3wAY0J/+K2fuU6SffjA";
   mugwump = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFE2fxPgWO+zeQoLBTgsgxP7Vg7QNHlrQ+Rb3fHFTomB";
+  bbbg = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL/VzrNEY47KPTce3dgfORkAbweWkr4BI8j54BAIs7bG";
 in
 
 {
-  "bbbg.age".publicKeys = [ grfn mugwump ];
+  "bbbg.age".publicKeys = [ grfn mugwump bbbg ];
   "cloudflare.age".publicKeys = [ grfn mugwump ];
 }
diff --git a/users/grfn/terraform/globals.nix b/users/grfn/terraform/globals.nix
new file mode 100644
index 000000000000..5f373c664604
--- /dev/null
+++ b/users/grfn/terraform/globals.nix
@@ -0,0 +1,24 @@
+{ pkgs, ... }:
+
+{
+  provider.aws = map (region: {
+    inherit region;
+    alias = region;
+    profile = "personal";
+  }) [
+    "us-east-1"
+    "us-east-2"
+    "us-west-2"
+  ];
+
+  data.external.cloudflare_api_key = {
+    program = [(pkgs.writeShellScript "cloudflare_api_key" ''
+      jq -n --arg api_key "$(pass cloudflare-api-key)" '{"api_key":$api_key}'
+    '')];
+  };
+
+  provider.cloudflare = {
+    email = "root@gws.fyi";
+    api_key = "\${data.external.cloudflare_api_key.result.api_key}";
+  };
+}
diff --git a/users/grfn/terraform/nixosMachine.nix b/users/grfn/terraform/nixosMachine.nix
new file mode 100644
index 000000000000..ef8830d66c21
--- /dev/null
+++ b/users/grfn/terraform/nixosMachine.nix
@@ -0,0 +1,203 @@
+{ depot, pkgs, lib, ... }:
+
+# mostly stolen from espes
+
+{ name
+, instanceType
+, configuration
+, prefix ? "${name}_"
+, region ? "us-east-2"
+, rootVolumeSizeGb ? 50
+, securityGroupId ? null
+, extraIngressPorts ? []
+}:
+
+let
+  os = depot.ops.nixos.nixosFor ({ modulesPath, ... }: {
+    imports = [
+      "${pkgs.path}/nixos/modules/virtualisation/amazon-image.nix"
+      configuration
+    ];
+
+    ec2.hvm = true;
+    networking.hostName = name;
+    # TODO: remove this once the terraform tls provider supports ed25519 keys
+    # https://github.com/hashicorp/terraform-provider-tls/issues/26
+    services.openssh.extraConfig = ''
+      PubkeyAcceptedKeyTypes=+ssh-rsa
+      PubkeyAcceptedAlgorithms=+ssh-rsa
+    '';
+  });
+
+  targetUser = "root";
+
+  ec2Amis = import "${pkgs.path}/nixos/modules/virtualisation/ec2-amis.nix";
+
+  osRoot = os.config.system.build.toplevel;
+
+  osRootPath = builtins.unsafeDiscardStringContext (toString osRoot.outPath);
+  drvPath = builtins.unsafeDiscardStringContext (toString osRoot.drvPath);
+
+  machineResource = "aws_instance.${prefix}machine";
+
+  recursiveMerge = builtins.foldl' lib.recursiveUpdate {};
+
+  securityGroupId' =
+    if isNull securityGroupId
+    then "\${aws_security_group.${prefix}group.id}"
+    else securityGroupId;
+in recursiveMerge [
+  (lib.optionalAttrs (isNull securityGroupId) {
+    resource.aws_security_group."${prefix}group" = {
+      provider = "aws.${region}";
+      vpc_id = null;
+
+      # terraform isn't good about knowing what other resources depend on
+      # security groups
+      lifecycle.create_before_destroy = true;
+    };
+
+    resource.aws_security_group_rule.all_egress = {
+      provider = "aws.${region}";
+      security_group_id = securityGroupId';
+      type            = "egress";
+      protocol        = "-1";
+      from_port       = 0;
+      to_port         = 0;
+      cidr_blocks     = ["0.0.0.0/0"];
+      ipv6_cidr_blocks = ["::/0"];
+
+      description = null;
+      prefix_list_ids = null;
+      self = null;
+    };
+  })
+  rec {
+    data.external.my_ip = {
+      program = [(pkgs.writeShellScript "my_ip" ''
+        ${pkgs.jq}/bin/jq \
+          -n \
+          --arg ip "$(curl ifconfig.me)" \
+          '{"ip":$ip}'
+      '')];
+    };
+
+    resource.aws_security_group_rule.provision_ssh_access = {
+      provider = "aws.${region}";
+      security_group_id = securityGroupId';
+      type = "ingress";
+      protocol = "TCP";
+      from_port = 22;
+      to_port = 22;
+      cidr_blocks = ["\${data.external.my_ip.result.ip}/32"];
+      ipv6_cidr_blocks = [];
+      description = null;
+      prefix_list_ids = null;
+      self = null;
+    };
+
+    resource.tls_private_key."${prefix}key" = {
+      algorithm = "RSA";
+    };
+
+    resource.aws_key_pair."${prefix}generated_key" = {
+      provider = "aws.${region}";
+      key_name = "generated-key-\${sha256(tls_private_key.${prefix}key.public_key_openssh)}";
+      public_key = "\${tls_private_key.${prefix}key.public_key_openssh}";
+    };
+
+    resource.aws_instance."${prefix}machine" = {
+      provider = "aws.${region}";
+      ami = ec2Amis."21.05"."${region}".hvm-ebs;
+      instance_type = instanceType;
+      vpc_security_group_ids = [ securityGroupId' ];
+      key_name = "\${aws_key_pair.${prefix}generated_key.key_name}";
+      root_block_device = {
+        volume_size = rootVolumeSizeGb;
+        tags.Name = name;
+      };
+      tags.Name = name;
+    };
+
+    resource.null_resource."${prefix}deploy_nixos" = {
+      triggers = {
+        # deploy if the machine is recreated
+        machine_id = "\${${machineResource}.id}";
+
+        # deploy on os changes
+        os_drv = drvPath;
+      };
+
+      connection = {
+        type = "ssh";
+        host = "\${${machineResource}.public_ip}";
+        user = targetUser;
+        private_key = "\${tls_private_key.${prefix}key.private_key_pem}";
+      };
+
+      # do the actual deployment
+      provisioner = [
+        # wait till ssh is up
+        { remote-exec.inline = [ "true" ]; }
+
+        # copy the nixos closure
+        {
+          local-exec.command = ''
+            export PATH="${pkgs.openssh}/bin:$PATH"
+
+            scratch="$(mktemp -d)"
+            trap 'rm -rf -- "$scratch"' EXIT
+
+            # write out ssh key
+            echo -n "''${tls_private_key.${prefix}key.private_key_pem}" > $scratch/id_rsa.pem
+            chmod 0600 $scratch/id_rsa.pem
+
+            export NIX_SSHOPTS="\
+                -o StrictHostKeyChecking=no\
+                -o UserKnownHostsFile=/dev/null\
+                -o GlobalKnownHostsFile=/dev/null\
+                -o IdentityFile=$scratch/id_rsa.pem"
+
+            nix-build ${drvPath}
+            nix-copy-closure \
+              --to ${targetUser}@''${${machineResource}.public_ip} \
+              ${osRootPath} \
+              --gzip \
+              --use-substitutes
+          '';
+        }
+
+        # activate it
+        {
+          remote-exec.inline = [
+            # semicolons mandatory
+            ''
+              set -e;
+              nix-env --profile /nix/var/nix/profiles/system --set ${osRootPath};
+              ${osRootPath}/bin/switch-to-configuration switch;
+            ''
+          ];
+        }
+      ];
+    };
+  }
+
+  {
+    resource.aws_security_group_rule = builtins.listToAttrs (map (port: {
+      name = "ingress_${toString port}";
+      value = {
+        provider = "aws.${region}";
+        security_group_id = securityGroupId';
+        type = "ingress";
+        protocol = "TCP";
+        from_port = port;
+        to_port = port;
+        cidr_blocks = ["0.0.0.0/0"];
+        ipv6_cidr_blocks = [];
+        description = null;
+        prefix_list_ids = null;
+        self = null;
+      };
+    }) extraIngressPorts);
+  }
+]
diff --git a/users/grfn/terraform/workspace.nix b/users/grfn/terraform/workspace.nix
new file mode 100644
index 000000000000..c2a0fdb97793
--- /dev/null
+++ b/users/grfn/terraform/workspace.nix
@@ -0,0 +1,104 @@
+{ pkgs, depot, ... }:
+name: { plugins }: module_tf:
+
+let
+
+  inherit (pkgs) lib runCommandNoCC writeText writeScript;
+  inherit (lib) filterAttrsRecursive;
+
+  allPlugins = (p: plugins p ++ (with p; [
+    external
+    local
+    tls
+    p.null
+  ]));
+
+  tf = pkgs.terraform.withPlugins allPlugins;
+
+  cleanTerraform = filterAttrsRecursive (k: _: ! (builtins.elem k [
+    "__readTree"
+    "__readTreeChildren"
+  ]));
+
+  plugins_tf = {
+    terraform.required_providers = (builtins.listToAttrs (map (p: {
+      name = lib.last (lib.splitString "/" p.provider-source-address);
+      value = {
+        source = p.provider-source-address;
+        version = p.version;
+      };
+    }) (allPlugins pkgs.terraform.plugins)));
+  };
+
+
+  module_tf' = module_tf // {
+    inherit (depot.users.grfn.terraform) globals;
+    plugins = plugins_tf;
+  };
+
+  module = runCommandNoCC "module" {} ''
+    mkdir $out
+    ${lib.concatStrings (lib.mapAttrsToList (k: config_tf:
+      (let
+        # TODO: filterAttrsRecursive?
+        configJson = writeText "${k}.tf.json"
+          (builtins.toJSON (cleanTerraform config_tf));
+      in ''
+        ${pkgs.jq}/bin/jq . ${configJson} > $out/${lib.escapeShellArg k}.tf.json
+      ''))
+      (cleanTerraform module_tf'))}
+  '';
+
+
+  tfcmd = writeScript "${name}-tfcmd" ''
+    set -e
+    dir="''${TF_STATE_ROOT:-$HOME/tfstate}/${name}"
+    cd "$dir"
+    rm -f *.json
+    cp ${module}/*.json .
+    exec ${tf}/bin/terraform "$(basename "$0")"
+  '';
+
+  init = writeScript "${name}-init" ''
+    set -e
+    dir="''${TF_STATE_ROOT:-$HOME/tfstate}/${name}"
+    [ -d "$dir" ] || mkdir -p "$dir"
+    cd "$dir"
+    rm -f *.json
+    cp ${module}/*.json .
+    exec ${tf}/bin/terraform init
+  '';
+
+  # TODO: import (-config)
+  tfcmds = runCommandNoCC "${name}-tfcmds" {} ''
+    mkdir -p $out/bin
+    ln -s ${init} $out/bin/init
+    ln -s ${tfcmd} $out/bin/validate
+    ln -s ${tfcmd} $out/bin/plan
+    ln -s ${tfcmd} $out/bin/apply
+    ln -s ${tfcmd} $out/bin/destroy
+  '';
+
+in {
+  inherit name module;
+  terraform = tf;
+  cmds = tfcmds;
+
+  # run = {
+  #   init = depot.nix.nixRunWrapper "init" tfcmds;
+  #   validate = depot.nix.nixRunWrapper "validate" tfcmds;
+  #   plan = depot.nix.nixRunWrapper "plan" tfcmds;
+  #   apply = depot.nix.nixRunWrapper "apply" tfcmds;
+  #   destroy = depot.nix.nixRunWrapper "destroy" tfcmds;
+  # };
+
+  test = runCommandNoCC "${name}-test" {} ''
+    set -e
+    export TF_STATE_ROOT=$(pwd)
+    ${tfcmds}/bin/init
+    ${tfcmds}/bin/validate
+    touch $out
+  '';
+
+  meta.targets = [ "module" "test" ];
+}