about summary refs log tree commit diff
path: root/users/grfn/terraform
diff options
context:
space:
mode:
authorGriffin Smith <grfn@gws.fyi>2021-12-27T03·37-0500
committerclbot <clbot@tvl.fyi>2021-12-27T03·46+0000
commit784e35bf553bc7f426aa2f663db6d32121431590 (patch)
treebf9de60f8d49113d6d450c1e868aaf4ae3f55219 /users/grfn/terraform
parent503ac8c78253b8339fd99719a3c02658ddf6e70e (diff)
feat(grfn/bbbg): Production deployment r/3456
Start of a production deployment of the app with nixos+terraform, using
provisioners and null-resources to provision nixos machines a'la espes.

Change-Id: I2ddaed76d0037dadbf9fc9e2ee27e9e67a852228
Reviewed-on: https://cl.tvl.fyi/c/depot/+/4695
Reviewed-by: grfn <grfn@gws.fyi>
Autosubmit: grfn <grfn@gws.fyi>
Tested-by: BuildkiteCI
Diffstat (limited to 'users/grfn/terraform')
-rw-r--r--users/grfn/terraform/globals.nix24
-rw-r--r--users/grfn/terraform/nixosMachine.nix203
-rw-r--r--users/grfn/terraform/workspace.nix104
3 files changed, 331 insertions, 0 deletions
diff --git a/users/grfn/terraform/globals.nix b/users/grfn/terraform/globals.nix
new file mode 100644
index 000000000000..5f373c664604
--- /dev/null
+++ b/users/grfn/terraform/globals.nix
@@ -0,0 +1,24 @@
+{ pkgs, ... }:
+
+{
+  provider.aws = map (region: {
+    inherit region;
+    alias = region;
+    profile = "personal";
+  }) [
+    "us-east-1"
+    "us-east-2"
+    "us-west-2"
+  ];
+
+  data.external.cloudflare_api_key = {
+    program = [(pkgs.writeShellScript "cloudflare_api_key" ''
+      jq -n --arg api_key "$(pass cloudflare-api-key)" '{"api_key":$api_key}'
+    '')];
+  };
+
+  provider.cloudflare = {
+    email = "root@gws.fyi";
+    api_key = "\${data.external.cloudflare_api_key.result.api_key}";
+  };
+}
diff --git a/users/grfn/terraform/nixosMachine.nix b/users/grfn/terraform/nixosMachine.nix
new file mode 100644
index 000000000000..ef8830d66c21
--- /dev/null
+++ b/users/grfn/terraform/nixosMachine.nix
@@ -0,0 +1,203 @@
+{ depot, pkgs, lib, ... }:
+
+# mostly stolen from espes
+
+{ name
+, instanceType
+, configuration
+, prefix ? "${name}_"
+, region ? "us-east-2"
+, rootVolumeSizeGb ? 50
+, securityGroupId ? null
+, extraIngressPorts ? []
+}:
+
+let
+  os = depot.ops.nixos.nixosFor ({ modulesPath, ... }: {
+    imports = [
+      "${pkgs.path}/nixos/modules/virtualisation/amazon-image.nix"
+      configuration
+    ];
+
+    ec2.hvm = true;
+    networking.hostName = name;
+    # TODO: remove this once the terraform tls provider supports ed25519 keys
+    # https://github.com/hashicorp/terraform-provider-tls/issues/26
+    services.openssh.extraConfig = ''
+      PubkeyAcceptedKeyTypes=+ssh-rsa
+      PubkeyAcceptedAlgorithms=+ssh-rsa
+    '';
+  });
+
+  targetUser = "root";
+
+  ec2Amis = import "${pkgs.path}/nixos/modules/virtualisation/ec2-amis.nix";
+
+  osRoot = os.config.system.build.toplevel;
+
+  osRootPath = builtins.unsafeDiscardStringContext (toString osRoot.outPath);
+  drvPath = builtins.unsafeDiscardStringContext (toString osRoot.drvPath);
+
+  machineResource = "aws_instance.${prefix}machine";
+
+  recursiveMerge = builtins.foldl' lib.recursiveUpdate {};
+
+  securityGroupId' =
+    if isNull securityGroupId
+    then "\${aws_security_group.${prefix}group.id}"
+    else securityGroupId;
+in recursiveMerge [
+  (lib.optionalAttrs (isNull securityGroupId) {
+    resource.aws_security_group."${prefix}group" = {
+      provider = "aws.${region}";
+      vpc_id = null;
+
+      # terraform isn't good about knowing what other resources depend on
+      # security groups
+      lifecycle.create_before_destroy = true;
+    };
+
+    resource.aws_security_group_rule.all_egress = {
+      provider = "aws.${region}";
+      security_group_id = securityGroupId';
+      type            = "egress";
+      protocol        = "-1";
+      from_port       = 0;
+      to_port         = 0;
+      cidr_blocks     = ["0.0.0.0/0"];
+      ipv6_cidr_blocks = ["::/0"];
+
+      description = null;
+      prefix_list_ids = null;
+      self = null;
+    };
+  })
+  rec {
+    data.external.my_ip = {
+      program = [(pkgs.writeShellScript "my_ip" ''
+        ${pkgs.jq}/bin/jq \
+          -n \
+          --arg ip "$(curl ifconfig.me)" \
+          '{"ip":$ip}'
+      '')];
+    };
+
+    resource.aws_security_group_rule.provision_ssh_access = {
+      provider = "aws.${region}";
+      security_group_id = securityGroupId';
+      type = "ingress";
+      protocol = "TCP";
+      from_port = 22;
+      to_port = 22;
+      cidr_blocks = ["\${data.external.my_ip.result.ip}/32"];
+      ipv6_cidr_blocks = [];
+      description = null;
+      prefix_list_ids = null;
+      self = null;
+    };
+
+    resource.tls_private_key."${prefix}key" = {
+      algorithm = "RSA";
+    };
+
+    resource.aws_key_pair."${prefix}generated_key" = {
+      provider = "aws.${region}";
+      key_name = "generated-key-\${sha256(tls_private_key.${prefix}key.public_key_openssh)}";
+      public_key = "\${tls_private_key.${prefix}key.public_key_openssh}";
+    };
+
+    resource.aws_instance."${prefix}machine" = {
+      provider = "aws.${region}";
+      ami = ec2Amis."21.05"."${region}".hvm-ebs;
+      instance_type = instanceType;
+      vpc_security_group_ids = [ securityGroupId' ];
+      key_name = "\${aws_key_pair.${prefix}generated_key.key_name}";
+      root_block_device = {
+        volume_size = rootVolumeSizeGb;
+        tags.Name = name;
+      };
+      tags.Name = name;
+    };
+
+    resource.null_resource."${prefix}deploy_nixos" = {
+      triggers = {
+        # deploy if the machine is recreated
+        machine_id = "\${${machineResource}.id}";
+
+        # deploy on os changes
+        os_drv = drvPath;
+      };
+
+      connection = {
+        type = "ssh";
+        host = "\${${machineResource}.public_ip}";
+        user = targetUser;
+        private_key = "\${tls_private_key.${prefix}key.private_key_pem}";
+      };
+
+      # do the actual deployment
+      provisioner = [
+        # wait till ssh is up
+        { remote-exec.inline = [ "true" ]; }
+
+        # copy the nixos closure
+        {
+          local-exec.command = ''
+            export PATH="${pkgs.openssh}/bin:$PATH"
+
+            scratch="$(mktemp -d)"
+            trap 'rm -rf -- "$scratch"' EXIT
+
+            # write out ssh key
+            echo -n "''${tls_private_key.${prefix}key.private_key_pem}" > $scratch/id_rsa.pem
+            chmod 0600 $scratch/id_rsa.pem
+
+            export NIX_SSHOPTS="\
+                -o StrictHostKeyChecking=no\
+                -o UserKnownHostsFile=/dev/null\
+                -o GlobalKnownHostsFile=/dev/null\
+                -o IdentityFile=$scratch/id_rsa.pem"
+
+            nix-build ${drvPath}
+            nix-copy-closure \
+              --to ${targetUser}@''${${machineResource}.public_ip} \
+              ${osRootPath} \
+              --gzip \
+              --use-substitutes
+          '';
+        }
+
+        # activate it
+        {
+          remote-exec.inline = [
+            # semicolons mandatory
+            ''
+              set -e;
+              nix-env --profile /nix/var/nix/profiles/system --set ${osRootPath};
+              ${osRootPath}/bin/switch-to-configuration switch;
+            ''
+          ];
+        }
+      ];
+    };
+  }
+
+  {
+    resource.aws_security_group_rule = builtins.listToAttrs (map (port: {
+      name = "ingress_${toString port}";
+      value = {
+        provider = "aws.${region}";
+        security_group_id = securityGroupId';
+        type = "ingress";
+        protocol = "TCP";
+        from_port = port;
+        to_port = port;
+        cidr_blocks = ["0.0.0.0/0"];
+        ipv6_cidr_blocks = [];
+        description = null;
+        prefix_list_ids = null;
+        self = null;
+      };
+    }) extraIngressPorts);
+  }
+]
diff --git a/users/grfn/terraform/workspace.nix b/users/grfn/terraform/workspace.nix
new file mode 100644
index 000000000000..c2a0fdb97793
--- /dev/null
+++ b/users/grfn/terraform/workspace.nix
@@ -0,0 +1,104 @@
+{ pkgs, depot, ... }:
+name: { plugins }: module_tf:
+
+let
+
+  inherit (pkgs) lib runCommandNoCC writeText writeScript;
+  inherit (lib) filterAttrsRecursive;
+
+  allPlugins = (p: plugins p ++ (with p; [
+    external
+    local
+    tls
+    p.null
+  ]));
+
+  tf = pkgs.terraform.withPlugins allPlugins;
+
+  cleanTerraform = filterAttrsRecursive (k: _: ! (builtins.elem k [
+    "__readTree"
+    "__readTreeChildren"
+  ]));
+
+  plugins_tf = {
+    terraform.required_providers = (builtins.listToAttrs (map (p: {
+      name = lib.last (lib.splitString "/" p.provider-source-address);
+      value = {
+        source = p.provider-source-address;
+        version = p.version;
+      };
+    }) (allPlugins pkgs.terraform.plugins)));
+  };
+
+
+  module_tf' = module_tf // {
+    inherit (depot.users.grfn.terraform) globals;
+    plugins = plugins_tf;
+  };
+
+  module = runCommandNoCC "module" {} ''
+    mkdir $out
+    ${lib.concatStrings (lib.mapAttrsToList (k: config_tf:
+      (let
+        # TODO: filterAttrsRecursive?
+        configJson = writeText "${k}.tf.json"
+          (builtins.toJSON (cleanTerraform config_tf));
+      in ''
+        ${pkgs.jq}/bin/jq . ${configJson} > $out/${lib.escapeShellArg k}.tf.json
+      ''))
+      (cleanTerraform module_tf'))}
+  '';
+
+
+  tfcmd = writeScript "${name}-tfcmd" ''
+    set -e
+    dir="''${TF_STATE_ROOT:-$HOME/tfstate}/${name}"
+    cd "$dir"
+    rm -f *.json
+    cp ${module}/*.json .
+    exec ${tf}/bin/terraform "$(basename "$0")"
+  '';
+
+  init = writeScript "${name}-init" ''
+    set -e
+    dir="''${TF_STATE_ROOT:-$HOME/tfstate}/${name}"
+    [ -d "$dir" ] || mkdir -p "$dir"
+    cd "$dir"
+    rm -f *.json
+    cp ${module}/*.json .
+    exec ${tf}/bin/terraform init
+  '';
+
+  # TODO: import (-config)
+  tfcmds = runCommandNoCC "${name}-tfcmds" {} ''
+    mkdir -p $out/bin
+    ln -s ${init} $out/bin/init
+    ln -s ${tfcmd} $out/bin/validate
+    ln -s ${tfcmd} $out/bin/plan
+    ln -s ${tfcmd} $out/bin/apply
+    ln -s ${tfcmd} $out/bin/destroy
+  '';
+
+in {
+  inherit name module;
+  terraform = tf;
+  cmds = tfcmds;
+
+  # run = {
+  #   init = depot.nix.nixRunWrapper "init" tfcmds;
+  #   validate = depot.nix.nixRunWrapper "validate" tfcmds;
+  #   plan = depot.nix.nixRunWrapper "plan" tfcmds;
+  #   apply = depot.nix.nixRunWrapper "apply" tfcmds;
+  #   destroy = depot.nix.nixRunWrapper "destroy" tfcmds;
+  # };
+
+  test = runCommandNoCC "${name}-test" {} ''
+    set -e
+    export TF_STATE_ROOT=$(pwd)
+    ${tfcmds}/bin/init
+    ${tfcmds}/bin/validate
+    touch $out
+  '';
+
+  meta.targets = [ "module" "test" ];
+}