[IBCDPE-1004] Implement Postgres server

.gitignore

@@ -1,3 +1,4 @@
 *.tfstate*
 .terraform
-terraform.tfvars
+terraform.tfvars
+settings.json

deployments/spacelift/dpe-k8s/main.tf

@@ -196,3 +196,75 @@ resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration
   read           = true
   write          = true
 }
+
+resource "spacelift_stack" "k8s-stack-deployments-testing" {
+  github_enterprise {
+    namespace = "Sage-Bionetworks-Workflows"
+    id        = "sage-bionetworks-workflows-gh"
+  }
+
+  depends_on = [
+    spacelift_space.dpe-space
+  ]
+
+  administrative          = false
+  autodeploy              = var.auto_deploy
+  branch                  = "ibcdpe-1004-airflow-ops"
+  description             = "Deployments internal to an EKS cluster"
+  name                    = "${var.k8s_stack_deployments_name}-testing"
+  project_root            = "deployments/stacks/dpe-k8s-deployments-testing"
+  repository              = "eks-stack"
+  terraform_version       = var.opentofu_version
+  terraform_workflow_tool = "OPEN_TOFU"
+  space_id                = spacelift_space.dpe-space.id
+  additional_project_globs = [
+    "deployments/"
+  ]
+}
+
+resource "spacelift_environment_variable" "k8s-stack-deployments-testing-environment-variables" {
+  for_each = local.k8s_stack_deployments_variables
+
+  stack_id   = spacelift_stack.k8s-stack-deployments-testing.id
+  name       = "TF_VAR_${each.key}"
+  value      = try(tostring(each.value), jsonencode(each.value))
+  write_only = false
+}
+
+resource "spacelift_context_attachment" "k8s-kubeconfig-hooks-testing" {
+  context_id = "kubernetes-deployments-kubeconfig"
+  stack_id   = spacelift_stack.k8s-stack-deployments-testing.id
+}
+
+resource "spacelift_stack_dependency" "k8s-stack-to-deployments-testing" {
+  stack_id            = spacelift_stack.k8s-stack-deployments-testing.id
+  depends_on_stack_id = spacelift_stack.k8s-stack.id
+}
+
+resource "spacelift_stack_dependency_reference" "dependency-references-testing" {
+  for_each = local.k8s_stack_to_deployment_variables
+
+  stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments-testing.id
+  output_name         = each.key
+  input_name          = each.value
+}
+
+resource "spacelift_stack_dependency_reference" "region-name-testing" {
+  stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments-testing.id
+  output_name         = "region"
+  input_name          = "REGION"
+}
+
+resource "spacelift_stack_dependency_reference" "cluster-name-testing" {
+  stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments-testing.id
+  output_name         = "cluster_name"
+  input_name          = "CLUSTER_NAME"
+}
+
+resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration-attachment-testing" {
+
+  integration_id = var.aws_integration_id
+  stack_id       = spacelift_stack.k8s-stack-deployments-testing.id
+  read           = true
+  write          = true
+}

deployments/stacks/dpe-k8s-deployments-testing/data.tf

@@ -0,0 +1,15 @@
+data "aws_eks_cluster" "cluster" {
+  name = var.cluster_name
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  name = var.cluster_name
+}
+
+data "aws_secretsmanager_secret" "spotinst_token" {
+  name = "spotinst_token"
+}
+
+data "aws_secretsmanager_secret_version" "secret_credentials" {
+  secret_id = data.aws_secretsmanager_secret.spotinst_token.id
+}

deployments/stacks/dpe-k8s-deployments-testing/main.tf

@@ -0,0 +1,20 @@
+module "postgres-cloud-native-operator" {
+  # source       = "spacelift.io/sagebionetworks/postgres-cloud-native/aws"
+  source = "../../../modules/postgres-cloud-native-operator/"
+  # version      = "0.2.1"
+  auto_deploy  = true
+  auto_prune   = true
+  git_revision = "ibcdpe-1004-airflow-ops"
+}
+
+
+# module "postgres-cloud-native" {
+#   # source       = "spacelift.io/sagebionetworks/postgres-cloud-native/aws"
+#   source = "../../../modules/postgres-cloud-native/"
+#   # version      = "0.2.1"
+#   auto_deploy          = true
+#   auto_prune           = true
+#   git_revision         = "ibcdpe-1004-airflow-ops"
+#   namespace            = "airflow"
+#   argo_deployment_name = "airflow-postgres-cloud-native"
+# }

deployments/stacks/dpe-k8s-deployments-testing/provider.tf

@@ -0,0 +1,28 @@
+provider "aws" {
+  region = var.region
+}
+
+provider "kubernetes" {
+  config_path            = var.kube_config_path
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kube_config_path
+  }
+}
+
+provider "spotinst" {
+  account = var.spotinst_account
+  token   = data.aws_secretsmanager_secret_version.secret_credentials.secret_string
+}
+
+provider "kubectl" {
+  config_path            = var.kube_config_path
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+}

deployments/stacks/dpe-k8s-deployments-testing/variables.tf

@@ -0,0 +1,62 @@
+variable "vpc_id" {
+  description = "VPC ID"
+  type        = string
+}
+
+variable "private_subnet_ids" {
+  description = "Private subnet IDs"
+  type        = list(string)
+}
+
+variable "node_security_group_id" {
+  description = "Node security group ID"
+  type        = string
+}
+
+variable "pod_to_node_dns_sg_id" {
+  description = "Pod to node DNS security group ID."
+  type        = string
+}
+
+variable "vpc_cidr_block" {
+  description = "VPC CIDR block"
+  type        = string
+}
+
+variable "kube_config_path" {
+  description = "Kube config path"
+  type        = string
+  default     = "~/.kube/config"
+}
+
+variable "region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-1"
+}
+
+variable "cluster_name" {
+  description = "EKS cluster name"
+  type        = string
+}
+
+variable "spotinst_account" {
+  description = "Spot.io account"
+  type        = string
+}
+
+variable "auto_deploy" {
+  description = "Automatically deploy the stack"
+  type        = bool
+}
+
+variable "auto_prune" {
+  description = "Automatically prune kubernetes resources"
+  type        = bool
+}
+
+variable "git_revision" {
+  description = "The git revision to deploy"
+  type        = string
+  default     = "main"
+}

deployments/stacks/dpe-k8s-deployments-testing/versions.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    spotinst = {
+      source = "spotinst/spotinst"
+    }
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "1.14.0"
+    }
+  }
+}

deployments/stacks/dpe-k8s-deployments/main.tf

@@ -13,7 +13,7 @@ module "sage-aws-eks-autoscaler" {
 module "victoria-metrics" {
   depends_on   = [module.argo-cd, module.sage-aws-eks-autoscaler]
   source       = "spacelift.io/sagebionetworks/victoria-metrics/aws"
-  version      = "0.4.7"
+  version      = "0.4.8"
   auto_deploy  = var.auto_deploy
   auto_prune   = var.auto_prune
   git_revision = var.git_revision
@@ -28,14 +28,14 @@ module "trivy-operator" {
   git_revision = var.git_revision
 }
 
-module "airflow" {
-  depends_on   = [module.victoria-metrics, module.argo-cd, module.sage-aws-eks-autoscaler]
-  source       = "spacelift.io/sagebionetworks/airflow/aws"
-  version      = "0.3.1"
-  auto_deploy  = var.auto_deploy
-  auto_prune   = var.auto_prune
-  git_revision = var.git_revision
-}
+# module "airflow" {
+#   depends_on   = [module.victoria-metrics, module.argo-cd, module.sage-aws-eks-autoscaler]
+#   source       = "spacelift.io/sagebionetworks/airflow/aws"
+#   version      = "0.3.1"
+#   auto_deploy  = var.auto_deploy
+#   auto_prune   = var.auto_prune
+#   git_revision = var.git_revision
+# }
 
 module "argo-cd" {
   depends_on = [module.sage-aws-eks-autoscaler]

main.tf

@@ -10,7 +10,7 @@
 # }
 
 locals {
-  git_branch = "main"
+  git_branch = "ibcdpe-1004-airflow-ops"
 }
 
 resource "spacelift_stack" "root_administrative_stack" {

modules/apache-airflow/main.tf

@@ -5,7 +5,7 @@
 
 resource "kubernetes_namespace" "airflow" {
   metadata {
-    name = "airflow"
+    name = var.namespace
   }
 }
 
@@ -18,7 +18,7 @@ resource "random_password" "airflow" {
 resource "kubernetes_secret" "airflow_webserver_secret" {
   metadata {
     name      = "airflow-webserver-secret"
-    namespace = "airflow"
+    namespace = var.namespace
   }
 
   data = {
@@ -28,10 +28,26 @@ resource "kubernetes_secret" "airflow_webserver_secret" {
   depends_on = [kubernetes_namespace.airflow]
 }
 
+resource "random_password" "airflow-admin-user" {
+  length  = 32
+  special = false
+}
+
+resource "kubernetes_secret" "airflow-admin-user-secret" {
+  metadata {
+    name      = "airflow-admin-user-secret"
+    namespace = var.namespace
+  }
+
+  data = {
+    "password" = random_password.airflow-admin-user.result
+    "username" = "admin"
+  }
+
+  depends_on = [kubernetes_namespace.airflow]
+}
 
-# TODO: Should a long-term deployment use a managed RDS instance?
-# https://github.com/apache/airflow/blob/main/chart/values.yaml#L2321-L2329
-resource "kubectl_manifest" "argo-deployment" {
+resource "kubectl_manifest" "airflow-deployment" {
   depends_on = [kubernetes_namespace.airflow]
 
   yaml_body = <<YAML
@@ -60,6 +76,6 @@ spec:
     ref: values
   destination:
     server: 'https://kubernetes.default.svc'
-    namespace: airflow
+    namespace: ${var.namespace}
 YAML
 }

modules/apache-airflow/templates/values.yaml

@@ -368,7 +368,7 @@ data:
   # data:
   #   connection: base64_encoded_connection_string
 
-  metadataSecretName: ~
+  metadataSecretName: pg-user-secret
   # When providing secret names and using the same database for metadata and
   # result backend, for Airflow < 2.4.0 it is necessary to create a separate
   # secret for result backend but with a db+ scheme prefix.
@@ -880,15 +880,15 @@ createUserJob:
     - "-r"
     - "{{ .Values.webserver.defaultUser.role }}"
     - "-u"
-    - "{{ .Values.webserver.defaultUser.username }}"
+    - "${AIRFLOW_USERNAME}"
     - "-e"
     - "{{ .Values.webserver.defaultUser.email }}"
     - "-f"
     - "{{ .Values.webserver.defaultUser.firstName }}"
     - "-l"
     - "{{ .Values.webserver.defaultUser.lastName }}"
     - "-p"
-    - "{{ .Values.webserver.defaultUser.password }}"
+    - "${AIRFLOW_PASSWORD}"
 
   # Annotations on the create user job pod
   annotations: {}
@@ -952,7 +952,17 @@ createUserJob:
   useHelmHooks: false
   applyCustomEnv: true
 
-  env: []
+  env:
+  - name: AIRFLOW_USERNAME
+    valueFrom:
+      secretKeyRef:
+        name: airflow-admin-user-secret
+        key: username
+  - name: AIRFLOW_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: airflow-admin-user-secret
+        key: password
 
   resources: {}
   #  limits:
@@ -1138,6 +1148,7 @@ webserver:
   #     memory: 128Mi
 
   # Create initial user.
+  # TODO: Create the initial user via a random secret
   defaultUser:
     enabled: true
     role: Admin
@@ -2077,7 +2088,7 @@ cleanup:
 # Configuration for postgresql subchart
 # Not recommended for production
 postgresql:
-  enabled: true
+  enabled: false
   image:
     tag: "11"
   auth:

modules/apache-airflow/variables.tf

@@ -15,3 +15,8 @@ variable "git_revision" {
   type        = string
   default     = "main"
 }
+
+variable "namespace" {
+  description = "The namespace to deploy into"
+  type        = string
+}