Merge branch 'main' into ibcdpe-1005-test-run

.gitignore

@@ -1,3 +1,4 @@
 *.tfstate*
 .terraform
-terraform.tfvars
+terraform.tfvars
+settings.json

README.md

@@ -218,6 +218,14 @@ log into the tool. Examples:
 - ArgoCD: Secret is named `argocd-initial-admin-secret` with a default username of `admin`
 - Grafana: Secret is named `victoria-metrics-k8s-stack-grafana` with a default username of `admin`
 
+## Tear down of EKS stacks
+If you need to fully tear down all of the infra start at the smallest point and work
+outwards. Destroy items in this order:
+
+- Go into the argoCD UI and delete all applications
+- Run `tofu destroy --auto-approve` as a task in spacelift for the Kubernetes Deployments stack
+- Run `tofu destroy --auto-approve` as a task in spacelift for the infrastructure deployment stack
+
 ## Spacelift
 Here are some instructions on setting up spacelift.
 

common-resources/aws-integrations/main.tf

@@ -6,3 +6,11 @@ resource "spacelift_aws_integration" "org-sagebase-dnt-dev-aws-integration" {
   duration_seconds               = 3600
   space_id                       = "root"
 }
+
+resource "spacelift_aws_integration" "org-sagebase-dpe-prod-aws-integration" {
+  name                           = "org-sagebase-dpe-prod-aws-integration"
+  role_arn                       = "arn:aws:iam::766808016710:role/spacelift-admin-role"
+  generate_credentials_in_worker = false
+  duration_seconds               = 3600
+  space_id                       = "root"
+}

common-resources/aws-integrations/outputs.tf

@@ -1,3 +1,7 @@
 output "org_sagebase_dnt_dev_aws_integration_id" {
   value = spacelift_aws_integration.org-sagebase-dnt-dev-aws-integration.id
 }
+
+output "org_sagebase_dpe_prod_aws_integration_id" {
+  value = spacelift_aws_integration.org-sagebase-dpe-prod-aws-integration.id
+}

common-resources/outputs.tf

@@ -1,3 +1,7 @@
 output "org_sagebase_dnt_dev_aws_integration_id" {
   value = module.aws-integrations.org_sagebase_dnt_dev_aws_integration_id
 }
+
+output "org_sagebase_dpe_prod_aws_integration_id" {
+  value = module.aws-integrations.org_sagebase_dpe_prod_aws_integration_id
+}

deployments/main.tf

@@ -13,12 +13,13 @@ resource "spacelift_space" "production" {
 }
 
 module "dpe-sandbox-spacelift-development" {
-  source          = "./spacelift/dpe-k8s"
-  parent_space_id = spacelift_space.development.id
-  admin_stack_id  = var.admin_stack_id
+  source           = "./spacelift/dpe-k8s"
+  parent_space_id  = spacelift_space.development.id
+  admin_stack_id   = var.admin_stack_id
+  spotinst_account = "act-45de6f47"
 
   aws_integration_id = var.org_sagebase_dnt_dev_aws_integration_id
-  auto_deploy        = true
+  auto_deploy        = false
   auto_prune         = true
   git_branch         = var.git_branch
 
@@ -36,39 +37,54 @@ module "dpe-sandbox-spacelift-development" {
   cluster_name = "dpe-k8-sandbox"
   vpc_name     = "dpe-sandbox"
 
-  vpc_cidr_block       = "10.51.0.0/16"
-  public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"]
-  private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"]
-  azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
+  vpc_cidr_block = "10.51.0.0/16"
+  # public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"]
+  # private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"]
+  # azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
+  # For now, we are only using one public and one private subnet. This is due to how
+  # EBS can only be mounted to a single AZ. We will need to revisit this if we want to
+  # allow usage of EFS ($$$$), or add some kind of EBS volume replication.
+  # Note: EKS requires at least two subnets in different AZs. However, we are only using
+  # a single subnet for node deployment.
+  public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24"]
+  private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24"]
+  azs                  = ["us-east-1a", "us-east-1b"]
 }
 
-# TODO: Fill this out with production specific values when we are ready
+module "dpe-sandbox-spacelift-production" {
+  source           = "./spacelift/dpe-k8s"
+  parent_space_id  = spacelift_space.production.id
+  admin_stack_id   = var.admin_stack_id
+  spotinst_account = "act-ac6522b4"
 
-# module "dpe-sandbox-spacelift-development" {
-#   source          = "./spacelift/dpe-k8s"
-#   parent_space_id = spacelift_space.development.id
-#   admin_stack_id  = var.admin_stack_id
-
-#   aws_integration_id = var.org_sagebase_dnt_dev_aws_integration_id
-#   auto_deploy        = true
-#   git_branch         = var.git_branch
-
-#   space_name = "dpe-sandbox"
+  aws_integration_id = var.org_sagebase_dpe_prod_aws_integration_id
+  auto_deploy        = false
+  git_branch         = var.git_branch
 
-#   k8s_stack_name         = "DPE DEV Kubernetes Infrastructure"
-#   k8s_stack_project_root = "deployments/stacks/dpe-k8s"
+  space_name = "dpe-k8s"
 
-#   k8s_stack_deployments_name         = "DPE DEV Kubernetes Deployments"
-#   k8s_stack_deployments_project_root = "deployments/stacks/dpe-k8s-deployments"
+  k8s_stack_name         = "DPE Kubernetes Infrastructure"
+  k8s_stack_project_root = "deployments/stacks/dpe-k8s"
 
-#   aws_account_id = "631692904429"
-#   region         = "us-east-1"
+  k8s_stack_deployments_name         = "DPE Kubernetes Deployments"
+  k8s_stack_deployments_project_root = "deployments/stacks/dpe-k8s-deployments"
 
-#   cluster_name = "dpe-k8-sandbox"
-#   vpc_name     = "dpe-sandbox"
+  aws_account_id = "766808016710"
+  region         = "us-east-1"
 
-#   vpc_cidr_block       = "10.51.0.0/16"
-#   public_subnet_cidrs  = ["10.51.1.0/24", "10.51.2.0/24", "10.51.3.0/24"]
-#   private_subnet_cidrs = ["10.51.4.0/24", "10.51.5.0/24", "10.51.6.0/24"]
-#   azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
-# }
+  cluster_name = "dpe-k8"
+  vpc_name     = "dpe-k8"
+
+  vpc_cidr_block = "10.52.0.0/16"
+  # public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24", "10.52.3.0/24"]
+  # private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24", "10.52.6.0/24"]
+  # azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
+  # For now, we are only using one public and one private subnet. This is due to how
+  # EBS can only be mounted to a single AZ. We will need to revisit this if we want to
+  # allow usage of EFS ($$$$), or add some kind of EBS volume replication.
+  # Note: EKS requires at least two subnets in different AZs. However, we are only using
+  # a single subnet for node deployment.
+  public_subnet_cidrs  = ["10.52.1.0/24", "10.52.2.0/24"]
+  private_subnet_cidrs = ["10.52.4.0/24", "10.52.5.0/24"]
+  azs                  = ["us-east-1a", "us-east-1b"]
+}

deployments/spacelift/dpe-k8s/main.tf

@@ -18,6 +18,7 @@ locals {
     auto_deploy      = var.auto_deploy
     auto_prune       = var.auto_prune
     git_revision     = var.git_branch
+    aws_account_id   = var.aws_account_id
   }
 
   # Variables to be passed from the k8s stack to the deployments stack

deployments/spacelift/dpe-k8s/variables.tf

@@ -111,7 +111,6 @@ variable "kube_config_path" {
 variable "spotinst_account" {
   description = "Spot.io account"
   type        = string
-  default     = "act-45de6f47"
 }
 
 variable "public_subnet_cidrs" {

deployments/stacks/dpe-k8s-deployments/main.tf

@@ -1,26 +1,41 @@
 module "sage-aws-eks-autoscaler" {
-  source  = "spacelift.io/sagebionetworks/sage-aws-eks-autoscaler/aws"
-  version = "0.5.0"
-
+  source                 = "spacelift.io/sagebionetworks/sage-aws-eks-autoscaler/aws"
+  version                = "0.9.0"
   cluster_name           = var.cluster_name
   private_vpc_subnet_ids = var.private_subnet_ids
   vpc_id                 = var.vpc_id
   node_security_group_id = var.node_security_group_id
   spotinst_account       = var.spotinst_account
-  # desired_capacity       = 2
+  single_az              = true
+  desired_capacity       = 3
+}
+
+module "sage-aws-eks-addons" {
+  source             = "spacelift.io/sagebionetworks/sage-aws-eks-addons/aws"
+  version            = "0.3.0"
+  cluster_name       = var.cluster_name
+  aws_account_id     = var.aws_account_id
+  vpc_id             = var.vpc_id
+  private_subnet_ids = var.private_subnet_ids
+}
+
+module "argo-cd" {
+  depends_on = [module.sage-aws-eks-autoscaler]
+  source     = "spacelift.io/sagebionetworks/argo-cd/aws"
+  version    = "0.3.1"
 }
 
 module "victoria-metrics" {
-  depends_on   = [module.argo-cd, module.sage-aws-eks-autoscaler]
+  depends_on   = [module.argo-cd]
   source       = "spacelift.io/sagebionetworks/victoria-metrics/aws"
-  version      = "0.4.7"
+  version      = "0.4.8"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
   git_revision = var.git_revision
 }
 
 module "trivy-operator" {
-  depends_on   = [module.victoria-metrics, module.argo-cd, module.sage-aws-eks-autoscaler]
+  depends_on   = [module.victoria-metrics, module.argo-cd]
   source       = "spacelift.io/sagebionetworks/trivy-operator/aws"
   version      = "0.3.2"
   auto_deploy  = var.auto_deploy
@@ -29,18 +44,33 @@ module "trivy-operator" {
 }
 
 module "airflow" {
-  depends_on   = [module.victoria-metrics, module.argo-cd, module.sage-aws-eks-autoscaler]
+  depends_on   = [module.victoria-metrics, module.argo-cd]
   source       = "spacelift.io/sagebionetworks/airflow/aws"
-  version      = "0.3.1"
+  version      = "0.4.0"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
   git_revision = var.git_revision
+  namespace    = "airflow"
 }
 
-module "argo-cd" {
-  depends_on = [module.sage-aws-eks-autoscaler]
-  source     = "spacelift.io/sagebionetworks/argo-cd/aws"
-  version    = "0.3.1"
+module "postgres-cloud-native-operator" {
+  depends_on   = [module.argo-cd]
+  source       = "spacelift.io/sagebionetworks/postgres-cloud-native-operator/aws"
+  version      = "0.4.0"
+  auto_deploy  = var.auto_deploy
+  auto_prune   = var.auto_prune
+  git_revision = var.git_revision
+}
+
+module "postgres-cloud-native-database" {
+  depends_on           = [module.postgres-cloud-native-operator, module.airflow, module.argo-cd]
+  source               = "spacelift.io/sagebionetworks/postgres-cloud-native-database/aws"
+  version              = "0.5.0"
+  auto_deploy          = true
+  auto_prune           = true
+  git_revision         = var.git_revision
+  namespace            = "airflow"
+  argo_deployment_name = "airflow-postgres-cloud-native"
 }
 
 locals {

deployments/stacks/dpe-k8s-deployments/variables.tf

@@ -60,3 +60,8 @@ variable "git_revision" {
   type        = string
   default     = "main"
 }
+
+variable "aws_account_id" {
+  description = "AWS account ID"
+  type        = string
+}

deployments/variables.tf

@@ -13,6 +13,11 @@ variable "org_sagebase_dnt_dev_aws_integration_id" {
   type        = string
 }
 
+variable "org_sagebase_dpe_prod_aws_integration_id" {
+  description = "ID of the AWS integration for the org-sagebase-dpe-prod account"
+  type        = string
+}
+
 variable "git_branch" {
   description = "The branch to deploy"
   type        = string