Enable scraping

modules/apache-airflow/main.tf

@@ -5,7 +5,7 @@
 
 resource "kubernetes_namespace" "airflow" {
   metadata {
-    name = "airflow"
+    name = var.namespace
   }
 }
 
@@ -18,7 +18,7 @@ resource "random_password" "airflow" {
 resource "kubernetes_secret" "airflow_webserver_secret" {
   metadata {
     name      = "airflow-webserver-secret"
-    namespace = "airflow"
+    namespace = var.namespace
   }
 
   data = {
@@ -29,8 +29,6 @@ resource "kubernetes_secret" "airflow_webserver_secret" {
 }
 
 
-# TODO: Should a long-term deployment use a managed RDS instance?
-# https://github.com/apache/airflow/blob/main/chart/values.yaml#L2321-L2329
 resource "kubectl_manifest" "airflow-deployment" {
   depends_on = [kubernetes_namespace.airflow]
 
@@ -60,6 +58,6 @@ spec:
     ref: values
   destination:
     server: 'https://kubernetes.default.svc'
-    namespace: airflow
+    namespace: ${var.namespace}
 YAML
 }

modules/apache-airflow/variables.tf

@@ -15,3 +15,8 @@ variable "git_revision" {
   type        = string
   default     = "main"
 }
+
+variable "namespace" {
+  description = "The namespace to deploy into"
+  type        = string
+}

modules/main.tf

@@ -66,7 +66,7 @@ locals {
       description        = "Helm chart deployment for a single node Victoria Metrics instance"
       project_root       = "modules/victoria-metrics"
       space_id           = "root"
-      version_number     = "0.4.7"
+      version_number     = "0.4.8"
     }
 
     trivy-operator = {
@@ -100,7 +100,7 @@ locals {
       description        = "Helm chart deployment for apache airflow."
       project_root       = "modules/apache-airflow"
       space_id           = "root"
-      version_number     = "0.3.1"
+      version_number     = "0.3.2"
     }
 
     argo-cd = {
@@ -135,7 +135,7 @@ locals {
       description    = "Helm chart deployment for postgres-cloud-native."
       project_root   = "modules/postgres-cloud-native"
       space_id       = "root"
-      version_number = "0.2.1"
+      version_number = "0.2.2"
     }
 
     private-workerpool = {

modules/postgres-cloud-native-operator/README.md

@@ -1,15 +1,25 @@
 # Purpose
 The purpose of this module is to deploy the `Cloudnative PG` helm chart <https://github.com/cloudnative-pg/charts/tree/main>.
-This will deploy both the operator and a database cluster.
+This will deploy both the operator.
 
+The `database` deployment is a part of another module. This allows us to add a single
+operator to a cluster and deploy 1 or more databases to that cluster.
 
-Future work:
 
-- Since each microservice/application is meant to recieve it's own database the deployment model within this module should be changed slightly to install the operator at a cluster level, with each application having its own database.
+## Future work to expand the capabilities
+- Setting up backups to S3: https://cloudnative-pg.io/documentation/current/backup/
+- Moving to database only node groups: https://cloudnative-pg.io/documentation/current/architecture/#postgresql-architecture
+- Assign database persistent volumes to an expandable storage class
 
 
+Reading:
+- https://www.cncf.io/blog/2023/09/29/recommended-architectures-for-postgresql-in-kubernetes/
+  - "The next level is to separate the Kubernetes worker nodes for PostgreSQL workloads from the other workloads’, using Kubernetes’ native scheduling capabilities, such as affinity, anti-affinity, node selectors and taints. You’ll still insist on the same storage, but you can get more predictability in terms of CPU and memory usage."
+- Assign database persistent volumes to an expandable storage class
+  - https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/
 
-# How many databases??
+
+# How many databases should I use?
 
 From their documentation:
 

modules/postgres-cloud-native-operator/templates/operator-values.yaml

@@ -132,7 +132,7 @@ affinity: {}
 monitoring:
 
   # -- Specifies whether the monitoring should be enabled. Requires Prometheus Operator CRDs.
-  podMonitorEnabled: false
+  podMonitorEnabled: true
   # -- Metrics relabel configurations to apply to samples before ingestion.
   podMonitorMetricRelabelings: []
   # -- Relabel configurations to apply to samples before scraping.

modules/postgres-cloud-native/templates/cluster-values.yaml

@@ -165,7 +165,7 @@ cluster:
 
   monitoring:
     # -- Whether to enable monitoring
-    enabled: false
+    enabled: true
     podMonitor:
       # -- Whether to enable the PodMonitor
       enabled: true

modules/victoria-metrics/templates/values.yaml

@@ -798,6 +798,10 @@ grafana:
         gnetId: 17813
         revision: 2
         datasource: VictoriaMetrics
+      cloudnativepg:
+        gnetId: 20417
+        revision: 3
+        datasource: VictoriaMetrics
 
   defaultDashboardsTimezone: utc