From 19f9cbf4eff925b339cd17f0e2da35ce4f731fce Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:00:59 -0700 Subject: [PATCH 01/44] Test postgres-CN --- modules/main.tf | 17 + modules/postgres-cloud-native/README.md | 56 ++ modules/postgres-cloud-native/main.tf | 110 +++ .../templates/cluster-values.yaml | 312 +++++++++ .../templates/operator-values.yaml | 629 ++++++++++++++++++ modules/postgres-cloud-native/variables.tf | 17 + modules/postgres-cloud-native/versions.tf | 16 + 7 files changed, 1157 insertions(+) create mode 100644 modules/postgres-cloud-native/README.md create mode 100644 modules/postgres-cloud-native/main.tf create mode 100644 modules/postgres-cloud-native/templates/cluster-values.yaml create mode 100644 modules/postgres-cloud-native/templates/operator-values.yaml create mode 100644 modules/postgres-cloud-native/variables.tf create mode 100644 modules/postgres-cloud-native/versions.tf diff --git a/modules/main.tf b/modules/main.tf index 5737c690..ba9a0165 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -120,6 +120,23 @@ locals { version_number = "0.3.1" } + postgres-cloud-native = { + github_enterprise = { + namespace = "Sage-Bionetworks-Workflows" + id = "sage-bionetworks-workflows-gh" + } + repository = "eks-stack" + + name = "postgres-cloud-native" + terraform_provider = "aws" + administrative = false + branch = var.git_branch + description = "Helm chart deployment for postgres-cloud-native." + project_root = "modules/postgres-cloud-native" + space_id = "root" + version_number = "0.2.0" + } + private-workerpool = { github_enterprise = { namespace = "Sage-Bionetworks-Workflows" diff --git a/modules/postgres-cloud-native/README.md b/modules/postgres-cloud-native/README.md new file mode 100644 index 00000000..5559aa2a --- /dev/null +++ b/modules/postgres-cloud-native/README.md @@ -0,0 +1,56 @@ +# Purpose +The purpose of this module is to deploy the `Apache Airflow` helm chart . + + +## What resources are being deployed through this module + +First we create a namespace to deploy all of the related resources to: +```terraform +resource "kubernetes_namespace" "airflow" { + metadata { + name = "airflow" + } +} +``` + +A secret key is also created, however, the implementation needs to be reviewed before +production: + +Finally, we are creating the ArgoCD Resource definition for the application: +```terraform +resource "kubectl_manifest" "argo-deployment" { + # Does not deploy the resource until the namespace is created + depends_on = [kubernetes_namespace.airflow] + + yaml_body = <.amazonaws.com" + # Leave empty if using the default S3 endpoint + endpointURL: "" + # -- Specifies a CA bundle to validate a privately signed certificate. + endpointCA: + # -- Creates a secret with the given value if true, otherwise uses an existing secret. + create: false + name: "" + key: "" + value: "" + # -- Overrides the provider specific default path. Defaults to: + # S3: s3:// + # Azure: https://..core.windows.net/ + # Google: gs:// + destinationPath: "" + # -- One of `s3`, `azure` or `google` + provider: s3 + s3: + region: "" + bucket: "" + path: "/" + accessKey: "" + secretKey: "" + azure: + path: "/" + connectionString: "" + storageAccount: "" + storageKey: "" + storageSasToken: "" + containerName: "" + serviceName: blob + inheritFromAzureAD: false + google: + path: "/" + bucket: "" + gkeEnvironment: false + applicationCredentials: "" + secret: + # -- Whether to create a secret for the backup credentials + create: true + # -- Name of the backup credentials secret + name: "" + + +cluster: + # -- Number of instances + instances: 3 + + # -- Name of the container image, supporting both tags (:) and digests for deterministic and repeatable deployments: + # :@sha256: + imageName: "" # Default value depends on type (postgresql/postgis/timescaledb) + + # -- Image pull policy. One of Always, Never or IfNotPresent. If not defined, it defaults to IfNotPresent. Cannot be updated. + # More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + imagePullPolicy: IfNotPresent + + # -- The list of pull secrets to be used to pull the images. + # See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-LocalObjectReference + imagePullSecrets: [] + + storage: + size: 8Gi + storageClass: "" + + # -- The UID of the postgres user inside the image, defaults to 26 + postgresUID: 26 + + # -- The GID of the postgres user inside the image, defaults to 26 + postgresGID: 26 + + # -- Resources requirements of every generated Pod. + # Please refer to https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for more information. + # We strongly advise you use the same setting for limits and requests so that your cluster pods are given a Guaranteed QoS. + # See: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ + resources: {} + # limits: + # cpu: 2000m + # memory: 8Gi + # requests: + # cpu: 2000m + # memory: 8Gi + + priorityClassName: "" + + # -- Method to follow to upgrade the primary server during a rolling update procedure, after all replicas have been + # successfully updated. It can be switchover (default) or in-place (restart). + primaryUpdateMethod: switchover + + # -- Strategy to follow to upgrade the primary server during a rolling update procedure, after all replicas have been + # successfully updated: it can be automated (unsupervised - default) or manual (supervised) + primaryUpdateStrategy: unsupervised + + # -- The instances' log level, one of the following values: error, warning, info (default), debug, trace + logLevel: "info" + + # -- Affinity/Anti-affinity rules for Pods. + # See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-AffinityConfiguration + affinity: + topologyKey: topology.kubernetes.io/zone + + # -- The configuration for the CA and related certificates. + # See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-CertificatesConfiguration + certificates: {} + + # -- When this option is enabled, the operator will use the SuperuserSecret to update the postgres user password. + # If the secret is not present, the operator will automatically create one. + # When this option is disabled, the operator will ignore the SuperuserSecret content, delete it when automatically created, + # and then blank the password of the postgres user by setting it to NULL. + enableSuperuserAccess: true + superuserSecret: "" + + # -- This feature enables declarative management of existing roles, as well as the creation of new roles if they are not + # already present in the database. + # See: https://cloudnative-pg.io/documentation/current/declarative_role_management/ + # TODO: Role management + roles: [] + # - name: airflow-pg + # ensure: present + # comment: Service account for airflow + # login: true + # superuser: false + # inRoles: + # - pg_monitor + # - pg_signal_backend + + monitoring: + # -- Whether to enable monitoring + enabled: false + podMonitor: + # -- Whether to enable the PodMonitor + enabled: true + prometheusRule: + # -- Whether to enable the PrometheusRule automated alerts + enabled: true + # -- Exclude specified rules + excludeRules: [] + # - CNPGClusterZoneSpreadWarning + # -- Custom Prometheus metrics + customQueries: [] + # - name: "pg_cache_hit_ratio" + # query: "SELECT current_database() as datname, sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio FROM pg_statio_user_tables;" + # metrics: + # - datname: + # usage: "LABEL" + # description: "Name of the database" + # - ratio: + # usage: GAUGE + # description: "Cache hit ratio" + + # -- Configuration of the PostgreSQL server. + # See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-PostgresConfiguration + postgresql: {} + # max_connections: 300 + + # -- BootstrapInitDB is the configuration of the bootstrap process when initdb is used. + # See: https://cloudnative-pg.io/documentation/current/bootstrap/ + # See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-bootstrapinitdb + # TODO: Verify user/db setup works properly + initdb: + database: airflow-pg + owner: "" # Defaults to the database name + secret: "airflow-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch + # postInitSQL: + # - CREATE EXTENSION IF NOT EXISTS vector; + + additionalLabels: {} + annotations: {} + + +backups: + # -- You need to configure backups manually, so backups are disabled by default. + enabled: false + + # -- Overrides the provider specific default endpoint. Defaults to: + # S3: https://s3..amazonaws.com" + endpointURL: "" # Leave empty if using the default S3 endpoint + # -- Specifies a CA bundle to validate a privately signed certificate. + endpointCA: + # -- Creates a secret with the given value if true, otherwise uses an existing secret. + create: false + name: "" + key: "" + value: "" + + # -- Overrides the provider specific default path. Defaults to: + # S3: s3:// + # Azure: https://..core.windows.net/ + # Google: gs:// + destinationPath: "" + # -- One of `s3`, `azure` or `google` + provider: s3 + s3: + region: "" + bucket: "" + path: "/" + accessKey: "" + secretKey: "" + azure: + path: "/" + connectionString: "" + storageAccount: "" + storageKey: "" + storageSasToken: "" + containerName: "" + serviceName: blob + inheritFromAzureAD: false + google: + path: "/" + bucket: "" + gkeEnvironment: false + applicationCredentials: "" + secret: + # -- Whether to create a secret for the backup credentials + create: true + # -- Name of the backup credentials secret + name: "" + + wal: + # -- WAL compression method. One of `` (for no compression), `gzip`, `bzip2` or `snappy`. + compression: gzip + # -- Whether to instruct the storage provider to encrypt WAL files. One of `` (use the storage container default), `AES256` or `aws:kms`. + encryption: AES256 + # -- Number of WAL files to be archived or restored in parallel. + maxParallel: 1 + data: + # -- Data compression method. One of `` (for no compression), `gzip`, `bzip2` or `snappy`. + compression: gzip + # -- Whether to instruct the storage provider to encrypt data files. One of `` (use the storage container default), `AES256` or `aws:kms`. + encryption: AES256 + # -- Number of data files to be archived or restored in parallel. + jobs: 2 + + scheduledBackups: + - + # -- Scheduled backup name + name: daily-backup + # -- Schedule in cron format + schedule: "0 0 0 * * *" + # -- Backup owner reference + backupOwnerReference: self + # -- Backup method, can be `barmanObjectStore` (default) or `volumeSnapshot` + method: barmanObjectStore + + # -- Retention policy for backups + retentionPolicy: "30d" + + +pooler: + # -- Whether to enable PgBouncer + enabled: false + # -- PgBouncer type of service to forward traffic to. + type: rw + # -- PgBouncer pooling mode + poolMode: transaction + # -- Number of PgBouncer instances + instances: 3 + # -- PgBouncer configuration parameters + parameters: + max_client_conn: "1000" + default_pool_size: "25" + + monitoring: + # -- Whether to enable monitoring + enabled: false + podMonitor: + # -- Whether to enable the PodMonitor + enabled: true + + # -- Custom PgBouncer deployment template. + # Use to override image, specify resources, etc. + template: {} + diff --git a/modules/postgres-cloud-native/templates/operator-values.yaml b/modules/postgres-cloud-native/templates/operator-values.yaml new file mode 100644 index 00000000..be7d92ee --- /dev/null +++ b/modules/postgres-cloud-native/templates/operator-values.yaml @@ -0,0 +1,629 @@ +# +# Copyright The CloudNativePG Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Default values for CloudNativePG. +# This is a YAML-formatted file. +# Please declare variables to be passed to your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/cloudnative-pg/cloudnative-pg + pullPolicy: IfNotPresent + # -- Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +hostNetwork: false +dnsPolicy: "" + +crds: + # -- Specifies whether the CRDs should be created when installing the chart. + create: true + +# -- The webhook configuration. +webhook: + port: 9443 + mutating: + create: true + failurePolicy: Fail + validating: + create: true + failurePolicy: Fail + livenessProbe: + initialDelaySeconds: 3 + readinessProbe: + initialDelaySeconds: 3 + +# -- Operator configuration. +config: + # -- Specifies whether the secret should be created. + create: true + # -- The name of the configmap/secret to use. + name: cnpg-controller-manager-config + # -- Specifies whether it should be stored in a secret, instead of a configmap. + secret: false + # -- The content of the configmap/secret, see + # https://cloudnative-pg.io/documentation/current/operator_conf/#available-options + # for all the available options. + data: {} + # INHERITED_ANNOTATIONS: categories + # INHERITED_LABELS: environment, workload, app + # WATCH_NAMESPACE: namespace-a,namespace-b + +# -- Additinal arguments to be added to the operator's args list. +additionalArgs: [] + +# -- Array containing extra environment variables which can be templated. +# For example: +# - name: RELEASE_NAME +# value: "{{ .Release.Name }}" +# - name: MY_VAR +# value: "mySpecialKey" +additionalEnv: [] + +serviceAccount: + # -- Specifies whether the service account should be created. + create: true + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + +rbac: + # -- Specifies whether ClusterRole and ClusterRoleBinding should be created. + create: true + # -- Aggregate ClusterRoles to Kubernetes default user-facing roles. + # Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles + aggregateClusterRoles: false + +# -- Annotations to be added to all other resources. +commonAnnotations: {} +# -- Annotations to be added to the pod. +podAnnotations: {} +# -- Labels to be added to the pod. +podLabels: {} + +# -- Container Security Context. +containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" + +# -- Security Context for the whole pod. +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + # fsGroup: 2000 + +# -- Priority indicates the importance of a Pod relative to other Pods. +priorityClassName: "" + +service: + type: ClusterIP + # -- DO NOT CHANGE THE SERVICE NAME as it is currently used to generate the certificate + # and can not be configured + name: cnpg-webhook-service + port: 443 + +resources: {} + # If you want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # + # limits: + # cpu: 100m + # memory: 200Mi + # requests: + # cpu: 100m + # memory: 100Mi + +# -- Nodeselector for the operator to be installed. +nodeSelector: {} + +# -- Tolerations for the operator to be installed. +tolerations: [] + +# -- Affinity for the operator to be installed. +affinity: {} + +monitoring: + + # -- Specifies whether the monitoring should be enabled. Requires Prometheus Operator CRDs. + podMonitorEnabled: false + # -- Metrics relabel configurations to apply to samples before ingestion. + podMonitorMetricRelabelings: [] + # -- Relabel configurations to apply to samples before scraping. + podMonitorRelabelings: [] + # -- Additional labels for the podMonitor + podMonitorAdditionalLabels: {} + + grafanaDashboard: + create: false + # -- Allows overriding the namespace where the ConfigMap will be created, defaulting to the same one as the Release. + namespace: "" + # -- The name of the ConfigMap containing the dashboard. + configMapName: "cnpg-grafana-dashboard" + # -- Label that ConfigMaps should have to be loaded as dashboards. DEPRECATED: Use labels instead. + sidecarLabel: "grafana_dashboard" + # -- Label value that ConfigMaps should have to be loaded as dashboards. DEPRECATED: Use labels instead. + sidecarLabelValue: "1" + # -- Labels that ConfigMaps should have to get configured in Grafana. + labels: {} + # -- Annotations that ConfigMaps can have to get configured in Grafana. + annotations: {} + +# Default monitoring queries +monitoringQueriesConfigMap: + # -- The name of the default monitoring configmap. + name: cnpg-default-monitoring + # -- A string representation of a YAML defining monitoring queries. + queries: | + backends: + query: | + SELECT sa.datname + , sa.usename + , sa.application_name + , states.state + , COALESCE(sa.count, 0) AS total + , COALESCE(sa.max_tx_secs, 0) AS max_tx_duration_seconds + FROM ( VALUES ('active') + , ('idle') + , ('idle in transaction') + , ('idle in transaction (aborted)') + , ('fastpath function call') + , ('disabled') + ) AS states(state) + LEFT JOIN ( + SELECT datname + , state + , usename + , COALESCE(application_name, '') AS application_name + , COUNT(*) + , COALESCE(EXTRACT (EPOCH FROM (max(now() - xact_start))), 0) AS max_tx_secs + FROM pg_catalog.pg_stat_activity + GROUP BY datname, state, usename, application_name + ) sa ON states.state = sa.state + WHERE sa.usename IS NOT NULL + metrics: + - datname: + usage: "LABEL" + description: "Name of the database" + - usename: + usage: "LABEL" + description: "Name of the user" + - application_name: + usage: "LABEL" + description: "Name of the application" + - state: + usage: "LABEL" + description: "State of the backend" + - total: + usage: "GAUGE" + description: "Number of backends" + - max_tx_duration_seconds: + usage: "GAUGE" + description: "Maximum duration of a transaction in seconds" + + backends_waiting: + query: | + SELECT count(*) AS total + FROM pg_catalog.pg_locks blocked_locks + JOIN pg_catalog.pg_locks blocking_locks + ON blocking_locks.locktype = blocked_locks.locktype + AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database + AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation + AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page + AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple + AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid + AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid + AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid + AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid + AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid + AND blocking_locks.pid != blocked_locks.pid + JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid + WHERE NOT blocked_locks.granted + metrics: + - total: + usage: "GAUGE" + description: "Total number of backends that are currently waiting on other queries" + + pg_database: + query: | + SELECT datname + , pg_catalog.pg_database_size(datname) AS size_bytes + , pg_catalog.age(datfrozenxid) AS xid_age + , pg_catalog.mxid_age(datminmxid) AS mxid_age + FROM pg_catalog.pg_database + WHERE datallowconn + metrics: + - datname: + usage: "LABEL" + description: "Name of the database" + - size_bytes: + usage: "GAUGE" + description: "Disk space used by the database" + - xid_age: + usage: "GAUGE" + description: "Number of transactions from the frozen XID to the current one" + - mxid_age: + usage: "GAUGE" + description: "Number of multiple transactions (Multixact) from the frozen XID to the current one" + + pg_postmaster: + query: | + SELECT EXTRACT(EPOCH FROM pg_postmaster_start_time) AS start_time + FROM pg_catalog.pg_postmaster_start_time() + metrics: + - start_time: + usage: "GAUGE" + description: "Time at which postgres started (based on epoch)" + + pg_replication: + query: "SELECT CASE WHEN ( + NOT pg_catalog.pg_is_in_recovery() + OR pg_catalog.pg_last_wal_receive_lsn() = pg_catalog.pg_last_wal_replay_lsn()) + THEN 0 + ELSE GREATEST (0, + EXTRACT(EPOCH FROM (now() - pg_catalog.pg_last_xact_replay_timestamp()))) + END AS lag, + pg_catalog.pg_is_in_recovery() AS in_recovery, + EXISTS (TABLE pg_stat_wal_receiver) AS is_wal_receiver_up, + (SELECT count(*) FROM pg_catalog.pg_stat_replication) AS streaming_replicas" + metrics: + - lag: + usage: "GAUGE" + description: "Replication lag behind primary in seconds" + - in_recovery: + usage: "GAUGE" + description: "Whether the instance is in recovery" + - is_wal_receiver_up: + usage: "GAUGE" + description: "Whether the instance wal_receiver is up" + - streaming_replicas: + usage: "GAUGE" + description: "Number of streaming replicas connected to the instance" + + pg_replication_slots: + query: | + SELECT slot_name, + slot_type, + database, + active, + (CASE pg_catalog.pg_is_in_recovery() + WHEN TRUE THEN pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_last_wal_receive_lsn(), restart_lsn) + ELSE pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), restart_lsn) + END) as pg_wal_lsn_diff + FROM pg_catalog.pg_replication_slots + WHERE NOT temporary + metrics: + - slot_name: + usage: "LABEL" + description: "Name of the replication slot" + - slot_type: + usage: "LABEL" + description: "Type of the replication slot" + - database: + usage: "LABEL" + description: "Name of the database" + - active: + usage: "GAUGE" + description: "Flag indicating whether the slot is active" + - pg_wal_lsn_diff: + usage: "GAUGE" + description: "Replication lag in bytes" + + pg_stat_archiver: + query: | + SELECT archived_count + , failed_count + , COALESCE(EXTRACT(EPOCH FROM (now() - last_archived_time)), -1) AS seconds_since_last_archival + , COALESCE(EXTRACT(EPOCH FROM (now() - last_failed_time)), -1) AS seconds_since_last_failure + , COALESCE(EXTRACT(EPOCH FROM last_archived_time), -1) AS last_archived_time + , COALESCE(EXTRACT(EPOCH FROM last_failed_time), -1) AS last_failed_time + , COALESCE(CAST(CAST('x'||pg_catalog.right(pg_catalog.split_part(last_archived_wal, '.', 1), 16) AS pg_catalog.bit(64)) AS pg_catalog.int8), -1) AS last_archived_wal_start_lsn + , COALESCE(CAST(CAST('x'||pg_catalog.right(pg_catalog.split_part(last_failed_wal, '.', 1), 16) AS pg_catalog.bit(64)) AS pg_catalog.int8), -1) AS last_failed_wal_start_lsn + , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time + FROM pg_catalog.pg_stat_archiver + metrics: + - archived_count: + usage: "COUNTER" + description: "Number of WAL files that have been successfully archived" + - failed_count: + usage: "COUNTER" + description: "Number of failed attempts for archiving WAL files" + - seconds_since_last_archival: + usage: "GAUGE" + description: "Seconds since the last successful archival operation" + - seconds_since_last_failure: + usage: "GAUGE" + description: "Seconds since the last failed archival operation" + - last_archived_time: + usage: "GAUGE" + description: "Epoch of the last time WAL archiving succeeded" + - last_failed_time: + usage: "GAUGE" + description: "Epoch of the last time WAL archiving failed" + - last_archived_wal_start_lsn: + usage: "GAUGE" + description: "Archived WAL start LSN" + - last_failed_wal_start_lsn: + usage: "GAUGE" + description: "Last failed WAL LSN" + - stats_reset_time: + usage: "GAUGE" + description: "Time at which these statistics were last reset" + + pg_stat_bgwriter: + runonserver: "<17.0.0" + query: | + SELECT checkpoints_timed + , checkpoints_req + , checkpoint_write_time + , checkpoint_sync_time + , buffers_checkpoint + , buffers_clean + , maxwritten_clean + , buffers_backend + , buffers_backend_fsync + , buffers_alloc + FROM pg_catalog.pg_stat_bgwriter + metrics: + - checkpoints_timed: + usage: "COUNTER" + description: "Number of scheduled checkpoints that have been performed" + - checkpoints_req: + usage: "COUNTER" + description: "Number of requested checkpoints that have been performed" + - checkpoint_write_time: + usage: "COUNTER" + description: "Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds" + - checkpoint_sync_time: + usage: "COUNTER" + description: "Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk, in milliseconds" + - buffers_checkpoint: + usage: "COUNTER" + description: "Number of buffers written during checkpoints" + - buffers_clean: + usage: "COUNTER" + description: "Number of buffers written by the background writer" + - maxwritten_clean: + usage: "COUNTER" + description: "Number of times the background writer stopped a cleaning scan because it had written too many buffers" + - buffers_backend: + usage: "COUNTER" + description: "Number of buffers written directly by a backend" + - buffers_backend_fsync: + usage: "COUNTER" + description: "Number of times a backend had to execute its own fsync call (normally the background writer handles those even when the backend does its own write)" + - buffers_alloc: + usage: "COUNTER" + description: "Number of buffers allocated" + + pg_stat_bgwriter_17: + runonserver: ">=17.0.0" + name: pg_stat_bgwriter + query: | + SELECT buffers_clean + , maxwritten_clean + , buffers_alloc + , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time + FROM pg_catalog.pg_stat_bgwriter + metrics: + - buffers_clean: + usage: "COUNTER" + description: "Number of buffers written by the background writer" + - maxwritten_clean: + usage: "COUNTER" + description: "Number of times the background writer stopped a cleaning scan because it had written too many buffers" + - buffers_alloc: + usage: "COUNTER" + description: "Number of buffers allocated" + - stats_reset_time: + usage: "GAUGE" + description: "Time at which these statistics were last reset" + + pg_stat_checkpointer: + runonserver: ">=17.0.0" + query: | + SELECT num_timed AS checkpoints_timed + , num_requested AS checkpoints_req + , restartpoints_timed + , restartpoints_req + , restartpoints_done + , write_time + , sync_time + , buffers_written + , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time + FROM pg_catalog.pg_stat_checkpointer + metrics: + - checkpoints_timed: + usage: "COUNTER" + description: "Number of scheduled checkpoints that have been performed" + - checkpoints_req: + usage: "COUNTER" + description: "Number of requested checkpoints that have been performed" + - restartpoints_timed: + usage: "COUNTER" + description: "Number of scheduled restartpoints due to timeout or after a failed attempt to perform it" + - restartpoints_req: + usage: "COUNTER" + description: "Number of requested restartpoints that have been performed" + - restartpoints_done: + usage: "COUNTER" + description: "Number of restartpoints that have been performed" + - write_time: + usage: "COUNTER" + description: "Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are written to disk, in milliseconds" + - sync_time: + usage: "COUNTER" + description: "Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are synchronized to disk, in milliseconds" + - buffers_written: + usage: "COUNTER" + description: "Number of buffers written during checkpoints and restartpoints" + - stats_reset_time: + usage: "GAUGE" + description: "Time at which these statistics were last reset" + + pg_stat_database: + query: | + SELECT datname + , xact_commit + , xact_rollback + , blks_read + , blks_hit + , tup_returned + , tup_fetched + , tup_inserted + , tup_updated + , tup_deleted + , conflicts + , temp_files + , temp_bytes + , deadlocks + , blk_read_time + , blk_write_time + FROM pg_catalog.pg_stat_database + metrics: + - datname: + usage: "LABEL" + description: "Name of this database" + - xact_commit: + usage: "COUNTER" + description: "Number of transactions in this database that have been committed" + - xact_rollback: + usage: "COUNTER" + description: "Number of transactions in this database that have been rolled back" + - blks_read: + usage: "COUNTER" + description: "Number of disk blocks read in this database" + - blks_hit: + usage: "COUNTER" + description: "Number of times disk blocks were found already in the buffer cache, so that a read was not necessary (this only includes hits in the PostgreSQL buffer cache, not the operating system's file system cache)" + - tup_returned: + usage: "COUNTER" + description: "Number of rows returned by queries in this database" + - tup_fetched: + usage: "COUNTER" + description: "Number of rows fetched by queries in this database" + - tup_inserted: + usage: "COUNTER" + description: "Number of rows inserted by queries in this database" + - tup_updated: + usage: "COUNTER" + description: "Number of rows updated by queries in this database" + - tup_deleted: + usage: "COUNTER" + description: "Number of rows deleted by queries in this database" + - conflicts: + usage: "COUNTER" + description: "Number of queries canceled due to conflicts with recovery in this database" + - temp_files: + usage: "COUNTER" + description: "Number of temporary files created by queries in this database" + - temp_bytes: + usage: "COUNTER" + description: "Total amount of data written to temporary files by queries in this database" + - deadlocks: + usage: "COUNTER" + description: "Number of deadlocks detected in this database" + - blk_read_time: + usage: "COUNTER" + description: "Time spent reading data file blocks by backends in this database, in milliseconds" + - blk_write_time: + usage: "COUNTER" + description: "Time spent writing data file blocks by backends in this database, in milliseconds" + + pg_stat_replication: + primary: true + query: | + SELECT usename + , COALESCE(application_name, '') AS application_name + , COALESCE(client_addr::text, '') AS client_addr + , COALESCE(client_port::text, '') AS client_port + , EXTRACT(EPOCH FROM backend_start) AS backend_start + , COALESCE(pg_catalog.age(backend_xmin), 0) AS backend_xmin_age + , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), sent_lsn) AS sent_diff_bytes + , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), write_lsn) AS write_diff_bytes + , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), flush_lsn) AS flush_diff_bytes + , COALESCE(pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), replay_lsn),0) AS replay_diff_bytes + , COALESCE((EXTRACT(EPOCH FROM write_lag)),0)::float AS write_lag_seconds + , COALESCE((EXTRACT(EPOCH FROM flush_lag)),0)::float AS flush_lag_seconds + , COALESCE((EXTRACT(EPOCH FROM replay_lag)),0)::float AS replay_lag_seconds + FROM pg_catalog.pg_stat_replication + metrics: + - usename: + usage: "LABEL" + description: "Name of the replication user" + - application_name: + usage: "LABEL" + description: "Name of the application" + - client_addr: + usage: "LABEL" + description: "Client IP address" + - client_port: + usage: "LABEL" + description: "Client TCP port" + - backend_start: + usage: "COUNTER" + description: "Time when this process was started" + - backend_xmin_age: + usage: "COUNTER" + description: "The age of this standby's xmin horizon" + - sent_diff_bytes: + usage: "GAUGE" + description: "Difference in bytes from the last write-ahead log location sent on this connection" + - write_diff_bytes: + usage: "GAUGE" + description: "Difference in bytes from the last write-ahead log location written to disk by this standby server" + - flush_diff_bytes: + usage: "GAUGE" + description: "Difference in bytes from the last write-ahead log location flushed to disk by this standby server" + - replay_diff_bytes: + usage: "GAUGE" + description: "Difference in bytes from the last write-ahead log location replayed into the database on this standby server" + - write_lag_seconds: + usage: "GAUGE" + description: "Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it" + - flush_lag_seconds: + usage: "GAUGE" + description: "Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written and flushed it" + - replay_lag_seconds: + usage: "GAUGE" + description: "Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written, flushed and applied it" + + pg_settings: + query: | + SELECT name, + CASE setting WHEN 'on' THEN '1' WHEN 'off' THEN '0' ELSE setting END AS setting + FROM pg_catalog.pg_settings + WHERE vartype IN ('integer', 'real', 'bool') + ORDER BY 1 + metrics: + - name: + usage: "LABEL" + description: "Name of the setting" + - setting: + usage: "GAUGE" + description: "Setting value" + diff --git a/modules/postgres-cloud-native/variables.tf b/modules/postgres-cloud-native/variables.tf new file mode 100644 index 00000000..07ea3c7a --- /dev/null +++ b/modules/postgres-cloud-native/variables.tf @@ -0,0 +1,17 @@ +variable "auto_deploy" { + description = "Auto deploy through ArgoCD" + type = bool + default = false +} + +variable "auto_prune" { + description = "Auto prune through ArgoCD" + type = bool + default = false +} + +variable "git_revision" { + description = "The git revision to deploy" + type = string + default = "main" +} diff --git a/modules/postgres-cloud-native/versions.tf b/modules/postgres-cloud-native/versions.tf new file mode 100644 index 00000000..c35c044f --- /dev/null +++ b/modules/postgres-cloud-native/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + kubectl = { + source = "gavinbunney/kubectl" + version = "1.14.0" + } + } +} From f9ddb886de39ea32c143e28a179e969fbd6056a1 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:03:24 -0700 Subject: [PATCH 02/44] Set branch --- modules/main.tf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/main.tf b/modules/main.tf index ba9a0165..432ce505 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -130,11 +130,12 @@ locals { name = "postgres-cloud-native" terraform_provider = "aws" administrative = false - branch = var.git_branch - description = "Helm chart deployment for postgres-cloud-native." - project_root = "modules/postgres-cloud-native" - space_id = "root" - version_number = "0.2.0" + # branch = var.git_branch + branch = "ibcdpe-1004-airflow-ops" + description = "Helm chart deployment for postgres-cloud-native." + project_root = "modules/postgres-cloud-native" + space_id = "root" + version_number = "0.2.0" } private-workerpool = { From e3f2c7f69fc7c005185eb0453f067243f2b26e31 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:15:05 -0700 Subject: [PATCH 03/44] Create testing stack --- deployments/spacelift/dpe-k8s/main.tf | 72 +++++++++++++++++++ .../dpe-k8s-deployments-testing/data.tf | 15 ++++ .../dpe-k8s-deployments-testing/main.tf | 7 ++ .../dpe-k8s-deployments-testing/provider.tf | 28 ++++++++ .../dpe-k8s-deployments-testing/variables.tf | 62 ++++++++++++++++ .../dpe-k8s-deployments-testing/versions.tf | 11 +++ 6 files changed, 195 insertions(+) create mode 100644 deployments/stacks/dpe-k8s-deployments-testing/data.tf create mode 100644 deployments/stacks/dpe-k8s-deployments-testing/main.tf create mode 100644 deployments/stacks/dpe-k8s-deployments-testing/provider.tf create mode 100644 deployments/stacks/dpe-k8s-deployments-testing/variables.tf create mode 100644 deployments/stacks/dpe-k8s-deployments-testing/versions.tf diff --git a/deployments/spacelift/dpe-k8s/main.tf b/deployments/spacelift/dpe-k8s/main.tf index 7f9a8429..0b2e9b5d 100644 --- a/deployments/spacelift/dpe-k8s/main.tf +++ b/deployments/spacelift/dpe-k8s/main.tf @@ -196,3 +196,75 @@ resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration read = true write = true } + +resource "spacelift_stack" "k8s-stack-deployments-testing" { + github_enterprise { + namespace = "Sage-Bionetworks-Workflows" + id = "sage-bionetworks-workflows-gh" + } + + depends_on = [ + spacelift_space.dpe-space + ] + + administrative = false + autodeploy = var.auto_deploy + branch = "ibcdpe-1004-airflow-ops" + description = "Deployments internal to an EKS cluster" + name = "${var.k8s_stack_deployments_name}-testing" + project_root = "deployments/stacks/dpe-k8s-deployments-testing" + repository = "eks-stack" + terraform_version = var.opentofu_version + terraform_workflow_tool = "OPEN_TOFU" + space_id = spacelift_space.dpe-space.id + additional_project_globs = [ + "deployments/" + ] +} + +resource "spacelift_environment_variable" "k8s-stack-deployments-testing-environment-variables" { + for_each = local.k8s_stack_deployments_variables + + stack_id = spacelift_stack.k8s-stack-deployments-testing.id + name = "TF_VAR_${each.key}" + value = try(tostring(each.value), jsonencode(each.value)) + write_only = false +} + +resource "spacelift_context_attachment" "k8s-kubeconfig-hooks-testing" { + context_id = "kubernetes-deployments-kubeconfig" + stack_id = spacelift_stack.k8s-stack-deployments-testing.id +} + +resource "spacelift_stack_dependency" "k8s-stack-to-deployments-testing" { + stack_id = spacelift_stack.k8s-stack-deployments-testing.id + depends_on_stack_id = spacelift_stack.k8s-stack.id +} + +resource "spacelift_stack_dependency_reference" "dependency-references-testing" { + for_each = local.k8s_stack_to_deployment_variables + + stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments-testing.id + output_name = each.key + input_name = each.value +} + +resource "spacelift_stack_dependency_reference" "region-name-testing" { + stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments-testing.id + output_name = "region" + input_name = "REGION" +} + +resource "spacelift_stack_dependency_reference" "cluster-name-testing" { + stack_dependency_id = spacelift_stack_dependency.k8s-stack-to-deployments-testing.id + output_name = "cluster_name" + input_name = "CLUSTER_NAME" +} + +resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration-attachment-testing" { + + integration_id = var.aws_integration_id + stack_id = spacelift_stack.k8s-stack-deployments-testing.id + read = true + write = true +} diff --git a/deployments/stacks/dpe-k8s-deployments-testing/data.tf b/deployments/stacks/dpe-k8s-deployments-testing/data.tf new file mode 100644 index 00000000..c1724ceb --- /dev/null +++ b/deployments/stacks/dpe-k8s-deployments-testing/data.tf @@ -0,0 +1,15 @@ +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} + +data "aws_eks_cluster_auth" "cluster" { + name = var.cluster_name +} + +data "aws_secretsmanager_secret" "spotinst_token" { + name = "spotinst_token" +} + +data "aws_secretsmanager_secret_version" "secret_credentials" { + secret_id = data.aws_secretsmanager_secret.spotinst_token.id +} diff --git a/deployments/stacks/dpe-k8s-deployments-testing/main.tf b/deployments/stacks/dpe-k8s-deployments-testing/main.tf new file mode 100644 index 00000000..8094d2d1 --- /dev/null +++ b/deployments/stacks/dpe-k8s-deployments-testing/main.tf @@ -0,0 +1,7 @@ +module "postgres-cloud-native" { + source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" + version = "0.2.0" + auto_deploy = true + auto_prune = true + git_revision = "ibcdpe-1004-airflow-ops" +} diff --git a/deployments/stacks/dpe-k8s-deployments-testing/provider.tf b/deployments/stacks/dpe-k8s-deployments-testing/provider.tf new file mode 100644 index 00000000..32049e25 --- /dev/null +++ b/deployments/stacks/dpe-k8s-deployments-testing/provider.tf @@ -0,0 +1,28 @@ +provider "aws" { + region = var.region +} + +provider "kubernetes" { + config_path = var.kube_config_path + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + config_path = var.kube_config_path + } +} + +provider "spotinst" { + account = var.spotinst_account + token = data.aws_secretsmanager_secret_version.secret_credentials.secret_string +} + +provider "kubectl" { + config_path = var.kube_config_path + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.cluster.token +} diff --git a/deployments/stacks/dpe-k8s-deployments-testing/variables.tf b/deployments/stacks/dpe-k8s-deployments-testing/variables.tf new file mode 100644 index 00000000..2828bba3 --- /dev/null +++ b/deployments/stacks/dpe-k8s-deployments-testing/variables.tf @@ -0,0 +1,62 @@ +variable "vpc_id" { + description = "VPC ID" + type = string +} + +variable "private_subnet_ids" { + description = "Private subnet IDs" + type = list(string) +} + +variable "node_security_group_id" { + description = "Node security group ID" + type = string +} + +variable "pod_to_node_dns_sg_id" { + description = "Pod to node DNS security group ID." + type = string +} + +variable "vpc_cidr_block" { + description = "VPC CIDR block" + type = string +} + +variable "kube_config_path" { + description = "Kube config path" + type = string + default = "~/.kube/config" +} + +variable "region" { + description = "AWS region" + type = string + default = "us-east-1" +} + +variable "cluster_name" { + description = "EKS cluster name" + type = string +} + +variable "spotinst_account" { + description = "Spot.io account" + type = string +} + +variable "auto_deploy" { + description = "Automatically deploy the stack" + type = bool +} + +variable "auto_prune" { + description = "Automatically prune kubernetes resources" + type = bool +} + +variable "git_revision" { + description = "The git revision to deploy" + type = string + default = "main" +} diff --git a/deployments/stacks/dpe-k8s-deployments-testing/versions.tf b/deployments/stacks/dpe-k8s-deployments-testing/versions.tf new file mode 100644 index 00000000..e3f8c566 --- /dev/null +++ b/deployments/stacks/dpe-k8s-deployments-testing/versions.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + spotinst = { + source = "spotinst/spotinst" + } + kubectl = { + source = "gavinbunney/kubectl" + version = "1.14.0" + } + } +} From 87227786f2ece8c4bb701573664c85a0efa03f01 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:20:55 -0700 Subject: [PATCH 04/44] Bump version and secret --- modules/main.tf | 2 +- modules/postgres-cloud-native/main.tf | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/main.tf b/modules/main.tf index 432ce505..cf7bf2ab 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -135,7 +135,7 @@ locals { description = "Helm chart deployment for postgres-cloud-native." project_root = "modules/postgres-cloud-native" space_id = "root" - version_number = "0.2.0" + version_number = "0.2.1" } private-workerpool = { diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index 97396675..2ae03f82 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -8,7 +8,7 @@ resource "kubernetes_namespace" "cnpg-system" { } } -resource "kubectl_manifest" "argo-deployment" { +resource "kubectl_manifest" "argo-deployment-operator" { depends_on = [kubernetes_namespace.cnpg-system] yaml_body = < Date: Fri, 16 Aug 2024 11:23:13 -0700 Subject: [PATCH 05/44] Bump version --- deployments/stacks/dpe-k8s-deployments-testing/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stacks/dpe-k8s-deployments-testing/main.tf b/deployments/stacks/dpe-k8s-deployments-testing/main.tf index 8094d2d1..10316ec4 100644 --- a/deployments/stacks/dpe-k8s-deployments-testing/main.tf +++ b/deployments/stacks/dpe-k8s-deployments-testing/main.tf @@ -1,6 +1,6 @@ module "postgres-cloud-native" { source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" - version = "0.2.0" + version = "0.2.1" auto_deploy = true auto_prune = true git_revision = "ibcdpe-1004-airflow-ops" From 8d9058d92a13c3f2c5cb1f45f6451b499d231ba0 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:26:26 -0700 Subject: [PATCH 06/44] Move source --- deployments/stacks/dpe-k8s-deployments-testing/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployments/stacks/dpe-k8s-deployments-testing/main.tf b/deployments/stacks/dpe-k8s-deployments-testing/main.tf index 10316ec4..f33588d8 100644 --- a/deployments/stacks/dpe-k8s-deployments-testing/main.tf +++ b/deployments/stacks/dpe-k8s-deployments-testing/main.tf @@ -1,5 +1,6 @@ module "postgres-cloud-native" { - source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" + # source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" + source = "../../../modules/postgres-cloud-native/" version = "0.2.1" auto_deploy = true auto_prune = true From 1df7dda7037c75f9eea0e5d2ff91a8162c507262 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:27:14 -0700 Subject: [PATCH 07/44] Take out version --- deployments/stacks/dpe-k8s-deployments-testing/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/stacks/dpe-k8s-deployments-testing/main.tf b/deployments/stacks/dpe-k8s-deployments-testing/main.tf index f33588d8..eb481444 100644 --- a/deployments/stacks/dpe-k8s-deployments-testing/main.tf +++ b/deployments/stacks/dpe-k8s-deployments-testing/main.tf @@ -1,7 +1,7 @@ module "postgres-cloud-native" { # source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" - source = "../../../modules/postgres-cloud-native/" - version = "0.2.1" + source = "../../../modules/postgres-cloud-native/" + # version = "0.2.1" auto_deploy = true auto_prune = true git_revision = "ibcdpe-1004-airflow-ops" From f76c64f3484a3a94446e2bfa1d69aa6466dbdb84 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:28:56 -0700 Subject: [PATCH 08/44] commit fix --- modules/postgres-cloud-native/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index 2ae03f82..ede60b95 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -104,8 +104,8 @@ resource "kubernetes_secret" "airflow-user-secret" { data = { "username" = "apache-airflow" - "password" = random_password.airflow.result + "password" = random_password.airflow-pg-password.result } - depends_on = [kubernetes_namespace.airflow] + depends_on = [kubernetes_namespace.cnpg-database] } From 4bf77c215b1bc74a973543cbfadba3919b9d0c78 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 11:36:34 -0700 Subject: [PATCH 09/44] fix --- modules/postgres-cloud-native/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index ede60b95..d63ca190 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -50,7 +50,7 @@ resource "kubernetes_namespace" "cnpg-database" { } resource "kubectl_manifest" "argo-deployment-database" { - depends_on = [kubernetes_namespace.cnpg-database, resource.kubectl_manifest.argo-deployment] + depends_on = [kubernetes_namespace.cnpg-database, resource.kubectl_manifest.argo-deployment-operator] yaml_body = < Date: Fri, 16 Aug 2024 14:49:09 -0700 Subject: [PATCH 10/44] [IBCDPE-1005] Create workshop for deploying hello-world (#19) * Create workshop for deploying hello-world --- docs/workshop-hello-world.md | 276 +++++++++++++++++++++++ docs/workshop-resources/github-check.png | Bin 0 -> 45274 bytes docs/workshop-resources/k9s-header.png | Bin 0 -> 24152 bytes 3 files changed, 276 insertions(+) create mode 100644 docs/workshop-hello-world.md create mode 100644 docs/workshop-resources/github-check.png create mode 100644 docs/workshop-resources/k9s-header.png diff --git a/docs/workshop-hello-world.md b/docs/workshop-hello-world.md new file mode 100644 index 00000000..ffc78c99 --- /dev/null +++ b/docs/workshop-hello-world.md @@ -0,0 +1,276 @@ +# Purpose +This is meant to be a small workshop that explores deploying a hello world application +to the AWS EKS Kubernetes cluster, as well as exploring a few items available within +the cluster. + +## Pre-Requisites +- Access to the AWS account that an EKS cluster is deployed to. In this workshop we will assume you are using the `org-sagebase-dnt-dev` account. +- You will need either `Administrator` or `Developer` access in the related AWS account. Both will have a level of access to the EKS cluster. In this workshop we are assuming the use of `Administrator` access. +- Install a tool to access the cluster in a user friendly way such as: https://k9scli.io/ +- Set up a SSO login session to access the cluster and login, for example: https://sagebionetworks.jira.com/wiki/spaces/IT/pages/2632286259/AWS+SSM+Session+Manager#%5CuD83D%5CuDCD8-Log-in-to-an-AWS-account + - `aws sso login --profile dnt-dev-admin` + - `aws sso login --profile dnt-dev-developer` +- Update your [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) to connect to the cluster. The kubeconfig in this case is referring to the authentication mechanism used to access the cluster.: + - `aws eks update-kubeconfig --region us-east-1 --name dpe-k8-sandbox --profile dnt-dev-admin` + - `aws eks update-kubeconfig --region us-east-1 --name dpe-k8-sandbox --profile dnt-dev-developer` +- In your cli start k9s + - `k9s` + +If everything is correct you should see a screen like: +![k9s header](./workshop-resources/k9s-header.png) + + +### Deploying the resources through ArgoCD (Recommended) +Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes. What this +means is that it will take care of all the deployment of the resources for you. For our +use case we will be creating an +[Application Specification](https://argo-cd.readthedocs.io/en/stable/user-guide/application-specification/). +This Application Specification defines a number of pointers and configuration elements +for ArgoCD to deploy to the Kubernetes cluster. Once ArgoCD has deployed the resources +you can view the resources in the ArgoCD UI, as well as in the k9s UI. + + +- Clone the `eks-stack` repository +- Create a new branch in git to push your changes to, for example `ibcdpe-1-branch-name` +- Open up `deployments/stacks/dpe-k8s-deployments/main.tf` + - You'll notice a bunch of modules. These are the resources we are already deploying to the kubernetes cluster, wrapped in something called a "module" + - For this tutorial we will create the resource files directly here for simplicity + +- Create 3 sections in the file: + +1) A locals block: to organize our inputs + +```terraform +locals { + # Rename this to the name of your branch + my_branch_name = "ibcdpe-1-branch-name" + # Set this to a unique name + my_namespace_name = "my-cool-namespace" + # Set this to a unique name + my_application_name_in_argocd = "my-cool-application" +} +``` + +2) A namespace: + +Update `my-cool-namespace-resource` to whatever value you'd like. In this +case `my-cool-namespace-resource` is the identifier in which terraform is referencing this specific +resource. + +```terraform +resource "kubernetes_namespace" "my-cool-namespace-resource" { + metadata { + name = local.my_namespace_name + } +} +``` + +3) An Argo CD Application: + +Make sure that `depends_on = [kubernetes_namespace.my-cool-namespace-resource]` is +referencing the name of the `kubernetes_namespace` resource you just created. + +```terraform +resource "kubectl_manifest" "my-argocd-application" { + depends_on = [kubernetes_namespace.my-cool-namespace-resource] + + yaml_body = < +- Add the following to the body of the yaml file: +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flask-helloworld + labels: + app: flask-helloworld +spec: + replicas: 1 + selector: + matchLabels: + app: flask-helloworld + template: + metadata: + labels: + app: flask-helloworld + spec: + containers: + - name: flask + image: digitalocean/flask-helloworld:latest + ports: + - containerPort: 5000 +``` + +- Commit the changes and push the changes to github +- Create a pull request to main +- In the git checks that run you'll see a section that says something like: `spacelift/dpe-dev-kubernetes-deployments — 2 to add, 6 to change, 0 to destroy` + +![Github check](./workshop-resources/github-check.png) + +- Click on `Details` and on the next screen `Deploy` +- A job will be kicked off and run within Spacelift to deploy the resources. This takes a few minutes. + +#### Explanation of the above + +You are deploying a CR (Custom Resource) to the kubernetes cluster. This tells ArgoCD +that you are deploying a set of kubernetes resources. By setting `syncPolicy` +to `automated`, ArgoCD will take care of the deployment of the resources to the +kubernetes cluster. [sources](https://argo-cd.readthedocs.io/en/stable/user-guide/multiple_sources/) +is an ArgoCD concept that allows you to point at one or more locations that ArgoCD will +look at in order to create the resources. In our case we are telling ArgoCD to look +at the `deployments/stacks/dpe-k8s-deployments` folder in the `eks-stack` repo for +the `targetRevision` that we specify. In our case we are setting it to the name of our +branch that we created earlier. + +### Deploying the resources directly to kubernetes (Not recommended, but possible) +If you've already completed the steps above, you can skip to the next section. + +- Create a new branch in git to push your changes to +- Open up `deployments/stacks/dpe-k8s-deployments/main.tf` + - You'll notice a bunch of modules. These are the resources we are already deploying to the kubernetes cluster, wrapped in something called a "module" + - For this tutorial we will create the resource files directly here for simplicity + +- Create 2 sections in the file: + +1) A namespace: + +Update `my-cool-namespace-resource` and `my-cool-namespace` to whatever values you'd like. In this +case `my-cool-namespace-resource` is the identifier in which terraform is referencing this specific +resource. `my-cool-namespace` on the otherhand is the name of the namespace you'll be +creating within the kubernetes cluster. + +```terraform +resource "kubernetes_namespace" "my-cool-namespace-resource" { + metadata { + name = "my-cool-namespace" + } +} +``` + +2) A Deployment: + +You'll want to update the reference to your terraform namespace resource here: + +- If you updated the terraform identifier `my-cool-namespace-resource` to something else, make sure they match +```terraform +resource "kubectl_manifest" "my-deployment" { + depends_on = [kubernetes_namespace.my-cool-namespace-resource] + + yaml_body = <>4AGhyH5B#}Q7e1w32K>jKvrU(H69R>jbnTi1S{>Hpw_T>E!(os?J z3j}bS=G%QH3FxE6-QdVZ@;#5sLLgeS7A475tr}$Xpgd zFtCx)Vk7$E;jH{^Xm@9Cdn1-TFc2o&Cp`rLwLCi!?O$(HWKBWo1DBDX2Zx44t+wd` zVc<)sqp^=|G-IK}z9xz%4&fU8q@z4okU+*K`!@oGIx<62eCQ!{seZAu1(>kv2y^j| zXvozv)mHiBEq{h_#N<=e!$SX-5)6FAp+8dn62SL*SNOf*CAG$JAhKcR0S9Rv*N;$!QVjSoZx8K|FvIhnG3e)wGd`QH zJjrv(Q%xd)_pW2F?}vY1y3Q&XN6_ww&Q2 zF%g67g@{@ZONT~)QXxoxAK3O_Elr`RxAH}r_J5U%-z%9Eyk7M^-5y(HD#|EBOM8gN zdGL#RQ~s+#q4o`n%I3`xRW6M`Nb;-$-gV6blZ4Qdl-Yb1>bU*wYq^5`AO!2_1HIUM zMXut1!x&7)fDLPMcmH#{79#acFqDzyIFPr_^2g-wii_j(4;E43T|{IOvlZov|LOl< z{oh_Vmu;{SoBQxa>0|wSfG~@%eVq`o|0E82Qlnsu9nF_UOiU11oOQQ+hcr-7KzBKn z?Y^2_-&}Qn>x!6O?=}Rr21=XWh{xQ&g}rI9p;y?Zghavnh{is|{HME|OH98gYEAc@ zu>DysP}}3x=cWCEOs?(ge8#%-ai^9&2W{Wm-O$L;PQ9=27l%nB_WcoEHj&;<4&+e1 zU@#Q;P7-R)<^i4dvs|%qX})v2%6gfBC0Dh%{OrL^5YuSZVcz|t`kVUo*E452Z zHhiH8Hvnq8D!{qm{fL&%<938VQ|<-3y4Hf3NyDsG{GUnTCd&Kwz6^!by3)8b)*@%5 zv!l3zw0|qavvWMG*W~>55gwTlyl+?|T zjZ8DuETN0d2WmknvFJ;MEO>xWi3kG?dOEsZEl1ofkifI@w?pqXOX9Pat*?TmQvc-Q z#)JID=wB8-c-1?>nzVl=#mvko5ZE@8S#~{>7}=ApyQiN6s$jn1&B!nBRpc3+POA-5 zeh075m)c#NQ(BC>whLVln!G&3ru?(c5|r_c&5+mb9<5m`R)$FCU_z6RZEIv znRrdars4QS6R$qb1Bo z{nNg5{6_d+{pK+IbsQ202~1uk)IhqR}qyEe^cHUbtyl5*oMwaq448AJ&Tc_ahK(NnAOljc6*?O?jOLHtz1@8-s3?}OHE z;VjZB#4#Crks>U1dOEqU+OK#yuwPYq_gy_ibcw-MtS_7UV;1A8?{VwXNpvbY*1_X3 zKZSqK#oUH(?hi461cT$#Od$!~AMGjEotN$lkNf6idpzLn*>(GEQN66pG?Mi&-<1`G zaxo*umn2Xu_BF&7Qvj0bMsK^2-5r>RLvNKxc<{U5=vx zK!Zb2;!VQm^5r>RacT^d}q6TkJ>Ufra2S?C<`?Mq1aWoGfczGV0<3wxsJ zP5T>hs$Rro3s8_KVAt|KYhzRsZ1zp-Y3!VH6$?=gh&7U`-z8ezVu>K=ap#iSN<7(M z9vwrXs$)cI9gQ4jk^Q^P5F=3#nB=~;y;}A7(>lZ57u0uTYI(T(d*Eh?iYik9kzJ*a`qSWq=W z@1uk7b$?xd^SZ3HlIfeUKRxK46c|K(K0ff({jB~K6LE)gdcTvGq|#m&s>-VE6&`io z+1i`LCM3dvo#=PqbA~tElKf!yS9P#3|LcrTS<*JV26GZ~7iaU^2v>Q8SfGRpfMz(A6pMKY8(r+edasUJ?3Bp)HPTSYsPiz!9xeKP(Lp zNJxyw;L|C@1l<>oOLnX9p05J4r@W^_Svz@L{YNr*@kf(7ZE3lx!BUK{p`CF z(UTyrOA{Wfq+mCw;uX5(grdMu52b6JH70kuPK`3}6Siar-cL~BdZZ{i>Sp*W7o|*I zzVBxQp%WFt&<%@QMCeLB4jcenM@*?r6KbrfAg=qmpTPLp3aUT|O9uW=Aygv@@YKMh z@c~k;IY}VDqp0ef=Y!8CgP8U;8?*JQ*oi46##(yT`Y0kSJk6#r%W6!IyZ3|$(X@*9 zGX%VDU-O|vQWw&Rdjox<_Y)7BD05HW%PwyNW$Q-#*jN+In=6?6g@kCR2_`9eIS{xt zLp9a`C1kr6k`)cwVuxNdk5-(4wDc_Q}=KC zx@k7$7(M6+r^$8Ib7-AgxfPe$%aY7$M()Fcb@m1a&$Mqx__{>n)BeS#FMmGb*Z6kx z+=$=Env*t4jfbe%i%RBkB#1A!)2l<;d+Or<^0dIHHxq}%fC zOMTHcmfk*`rDtNOfC$iEbvb zRN_WFjsLYv^t%$cC$TxNhI`JKuF)vm2U&Y7WZOV*9Z!dUTDN;D8PonojLyQ@n?&vA zB*?Ke_(w(QNZzeZqtxr`dIwAkV4o7&v%%_1wy&%9lr(sHbB}-E2FqN{X`;T&KFr74 z!}me>$#kfsp+<_UWKZq zvcTbUdASFVZv_+vE&1I6=E3aOmVBm!*rZip`{K6?i}-2%S|VqJfU`#AA3R|TZ|0_6 zU^B$B=iAELV6XG*hi%aGh3zGrjrX3Y_S&hgwx9)}%@Lyw8weoT&9IDv@P=~!?ny1`yNy7=1dcH5<+3NhCS$Ec`s+E7s6{pX0sW z@JYYZnc}>n@|afAbp&5Z`wB{KfBwt1?`zmHi3eO zAZ(o|sMKylZI;dLpo5zVg}m}K-HhZwRwpF;Fm&~R@y_sl^OuM3*QViyTYKgbfl$9A zqtrXtd@`qmnsU-qrsOR5(a`P^uMK7C-dnuv{zFPI*2ZYCuVKX_DjtvZA#h*2ttS{?VGhd3N{1qm zI}7->|%@+mDaG z!av_*lYO!wS_<=Tb-3X=2E&0rzs=!*1srfQXNBJBF{ey}3a8Pf}t2V&-E{vP(6gdH}tM zzu;dtgJ<9{i&>>>QK@e|;vnP;yTy{kSc3L`|LF{uRW!XC|BXY!)5WS7Nm!fzD-oDY_&l^a!6IknyHZPY;fbNJeywARlKfWi3;;y~8UNG|79 z{7LK65qs6N&jR^*+VjP{w9aQ@S|ja2omvyhccpYIL2;6|RI+Wylf4fC8gAAK-^jOU zYCmPfA)WlxWIG$;L0*>Y-{Zx}fg{rdCS4H{y_~TF-;@|x>Sq?8-xP0so@@vF@@ow_ zqOrp}IW|fE*jn9pA}=ISCmIu@@9)YJ$^CA+>FwkC1KduT&-P1oE0?PXL8JCTBwc3k zS7!BnEu_xj-!yYt6lMW^e)o?m3YY=SJ6o67bH2&*qn{ffaP-cK^RV%$&A!&I>!Hk9 z0R$%^0cyrhae7YPUaq8eH9e3PUnXd1a9fmSt?(J_f6={oQ^BrVKjCX(zKQAjRJBK` z3^&P4UmMY&J4>(4&P?Q~);^gWt0#uqeWS?ZNx8%9+NxbN+wdijYrZ^Q9Ax|u&dv(%7V$YwpdO5y_bNz zFO!y|RN>74Cj8fl*O5)CO_Z~TPe_ZddI+=5QV!iNC787aU$Ehv?J(gzN>>P|`rMMH z6}!^%t~NrvWMo206ETUuV9FYb=~o|P5)G>9$O528Y{;%A2>fE-gKGe*wE{miqZ%kw$8HX%``k9ba)tFzWeQ;5$_=h0Wu>-Cm)eCOnO_4 z*g{9{bERh~0)*vSz@6rW=jLC1PXS%3WEyrDuMdX=RA2?{(>fm$EgFRVYzfb+F3AgjahO6FsWzlFz;5NKZSs zXBrjzi&d|=B}%mW(5lPCK7b5E3JkYPh_~vesLEKs7~dqgs!CU^=R|Nej#UZaEFv}9 z-+|7_3B@f25>1?zqC=w-AbTpfpG@g2W@GA-DSBi8f;jWJB8Hc-mmgeCtF0-o(qacR ziN+)8>(@VBex9$KO<1xod|{j4Ydj5ddL(_iYlM09U5f&l;ycGPLM>{;a2)uJ=lTp( z&I|R4SC8z(e2y&xu!QYOEiwC-dEDo=oTMyRqZw2yoes}o5LdLL3~EF}m)-yi>bBpC zQ9mi=G~WT{ojtIRM$WMCr7x4*e`bU0xETmLzA%d_QwkJ2)sxLStzw-#5t#FAC22a* zmXDMO4eTXKj1gV>ArWFL1TQF73fudDU^uy_Trjm(ugzG->Qq zk+$gB2JA|kSUkU2$8x{k3(l&c53IJ^LwvlC>7$iL?u;o-PhiBG6m1`vO`7|1mU&+U zsp_)XkW}_)^LXMcfEMOJ6zW#M>AwSV?dJ#IEy6EUJ^k>yedTWf#tTQ4o_~oi0?i;J z{xU`~%5l@1&#(x$J5|&R0@!x^1f0uvkpNbsM|czVZ{N_iY}W6NGDOThHATH7Q8+MJ@Z;y$Ez3xqg@r5F;;?BE zCcFClHthbQCnx#Ae9|dz-WP(HH}o41s}fOwPubo4VV~MMIWVJ!o4td&J#nDrhJ?x8 z8O4Dg|3ZVXs0d*R9#j*v*K>|3F>J$pr zF3|j{nKQB-)>vZ64;9VQUEg>AV|mVKrg4Yo#_LYBmgzLga~k(D=5l(nNQ6Yr)(1`~ z0^IO=13%G-IBLkA4D0lSYUBC9GtJ3D@u*BWDxN&m{aMLvTpFM9O!fg%RJ z((%|{OxKE^WeGzkbA>*-1<}H&yem^L+~MVd_#*)~gPLe<+dhCD1<;v%+gABB@MR|A zL*M1Oar^ydjGhZ+ZFx~e2raJw&Am5TMhEg^zpWT4>Rd?!Qf)aXq>Foz_3cV}98TM>6hOolb@OfYdMy7KgkyW&>vV1H+1 zK2~=E5^(mdcjGnx*)UK=$ccazO+zTUi}YBHg0GB_+53V^+tvqVN#b$*A751F?iY;{H!SWHHLj&t;*^7DNH>l)sYW+H(Cf1aG}Pxa5Pp zSBD+e9atHxFkghDW;n4eRMPaj>9WVOs;A2?&egRUpXDYwYcEuac0G=6eV-d`e*6(m zWpk)CGT;8eYuAUp?Vjo>1%8|SoylW>J~N<9P3)B;CXT7rK-WH~)|qMVcrszMev{{} z_DoN?UUPl+wZ|j-CcxaT=}nnaVoCT4w&p|Q#ls!=Ec~7b%0UFtNS(ZAdNd<+#3@Ip zMqJ1_PVFe6M2Up-Yu`KdE7kkknERQV3mlEtXO*$V5<*SHd5+_fz0jf~jVK`;+3x0j zPlNj~yWSPHnj4+=cY_c0%6iX^+^;xm1%1Ra)-XzDuKk%sVCLMnYB4^ys1$N}d%XI~ zaqjecFbz}ExqXuT2grNZ?R`28<& z>0op%le5@-?%O?Q&1cWfK@1BUtvC5yb%%`OJ!xg3BjY{mufAl8LCy2~@V6DqlaFlSt!u zWQ4UNTxY*4_eJ21{8w%8WbMJ3l~G&6`e$#fM_4*XS(gk)+z*`1D-$YIfw-J?E(77H{82B$q!b~JVMU91q z=dYWaL{>E4fkvP#n(rZR^L(xwvoQ-P-zg84{4YsB&ML)}PoA_IeSWSN$+rbZ(FwX% zQyp~P<#br~Y+fl*94Zcc+ghuNvpcV~wO6L*sV6>4#v(GN$U_%# zt*#Kiv*4ves?y~1*Adp@Q9TIwzm0Hph&2-xUQ*%0G)3rCefr0J-s*aeAhelGyI;M9t2( zH6{OJcbuad4xawI3KeGi$tFXLI`1ovB{t)-Ybq_f3EnyXHIpSCAC6bgeHng_F@#LG zkY;wGD3V}Qu}_qR@P>0+$*{|_)x*rhl1e!fbedyNgOdrL#f004qu|7fu}YZS&i<-# z7)8%dL1pMHS&MHY0G(LbpSa0ujgw;<9d*1NQKuH?cbyr#JCw~76x*|VTiw+v;OSlh znr7qi^+~pQ`-a_1z#1;GU~q;GiF0?`?d?m|4a$S}35As^{Q(oyCc2%%_?EGYBpi|B zh5Ser);`&rg8c%&ZTXxgqZT)D0~?Os{GR%0(!fpsbxHn=cIKyiLaXBzE`kavCvCxz z`gU$yi6))O#eQwBv*WQx)Zwm7!p`oT{ytgskri>9U)arqI(= znT_|n(Ld>oX$;-WMowmNP-5Wh->SiY`vOa_R?C}SxZ-vBji4rxcwbUxhOSeX-mz__J6e{(A zV|8-z+Ap9}8ZVW}&-eb7TYHz*>V+la>#6npc4wT!d-hpDrJt;dA*^HMN@mMg zO9xq(65il2Dp%?D=hU?VUv4@-?g5hkvh=Lr8L_2PP+Zz+N8WCU3_*@U^X$vo>)^M5a8tu%Hfe8rpNH6HWe5bWEItOo#-8E`5oX;@wYkPQ9jmi+kef06p*W&PJC4u>#|GUklu$F z%ZG$f8a`utS$A0RDyatS+xru2w``VQXO&-^D)sWbSN_BZTgzW-s|c?>$Ac`bMnG9m zuo(l>lu>v%{yyh>OgauK!F)V3VffK-bL&>%yi=Sb<^D{XK1-GaMsgpn^^ulzSa{ zW)gR!id1-x6`#wf*-~JA#x5ezfftkt=hQoi?ZMT=6Ix^ba=>N)1y{&aZFS$GzV6)Y zl{hYi8<+oFTJypu-LT)BhC^ZJv*+Kz!uByBx1rQ9CD2vLW82-7a-2a&qDbuz_hNs2 z`|fq&nO3ov3$ymDxtBaraNUX=C<2=;sbu8tn9$Xm77Wz zY=zR_Xv6?;twPtrC?0+?v}9;@DLLwrA!yI&z>(B(`Ei?mq)_-b1k4g@VT29KEY?z` z=tTl2bq~&(5gGr+rr0hWt6sAIa`#09KV1Kbq@cLM3lRG&oPwfS|JQ#RVKE#ejK6z< z(9l7D5sH6t4PE2Y|9$iX5iaa+J7Qu+UH_97%7jPt@7x7?EbRXe!ocA$w{73vH(fuU zB>pv|+~#aGt?kVS)PxNyPvuf@{P$LDc6tpuL8 zI5>R&m5J(FIR1W62GY+D;ll?Jb+`8uJ5Vzi|MXM>Txp|?6)~WWO-KkD{akoCwDq<% z!qU-uce)&#lvD`o^H}gx0TD?&B@Gn|(jWwd*lq;SurZ$PN?vUOc<;N}@jsm^-1d?w z6pxA+sG88cK3SrY;B%pLyuG&IAUr)kTLL2B7B%UHR3!qCa1ap@>waEVk1+S7)|h0g zluDza;}`#TrX=SmSR_YLBR~uvp`d`MZ-Vc90!j>@XjqA+u-*|Bc$wKxnc|&Yh4iXlw?-hNHAwkvgzcVrl9J|a?JDnWZ+9rGW zUQdU#i8x~TCqoeDLAbgT{#~XS?r?1$x82l9+WvFpTDg)jB!BqbtLsk^)O05`@!mHY zTFD`!(t8?ggqKz=+_l)#6YlUT1S+ii6<`HJq2PBvbWs82+zjd}}~TVyzE1@O6Z<_I`6rF}L@ zYN*D})>av%e-pyn@WUrO&HQzF~w{fYz5+cv;N@>#EkP+lKY!126w$A zp$f%omLcfVdcC3C9LLD1yw6x47?R%s>u3z%!)E_329G-rxQmw)T zwtpf>&eyW{(&7T9sc2{{ATzAcy>4gK?61eTwx7>C1|nG6L*wJItTjwb3OK(G#E=fu znon2Stn%KRuj^^seFl_=v9To6#!|xXx0SV+9=n|OWB4{T1 zM$s;x^-E*oFT^he)8n-{q3!tyLV>4$@m1lyK3XAZ@Ye4D^4J#Kt>JlvXrP$UKMfrg z2b*K_AMgav%mjx7?Nm|I$(OAjISj{UtaUzbyVt!rSyUS7F*A1(e>HAhk6hhaW{b3;Z9#v{dvW%< zQH-s%qF53I66PBMUjb!OHF>{wG*RbsRZVt(T&O_~` zCYQj}4-sX6UrO_NrOKv<6pbl1dCSg_9L+HnamB_!^=76X11og*ayQvL^YVM7XfoS4sO*AgswaC0Ys#NDZcv7$L&rY~x z->PyG^=D!11ENI!lsZ$k67&ek_iEd3L(kS-@W`NZNp{AK+^VMQ*xt|AV>ifd!MQLc z;sJPVQNMIYrp`nqYrA0T`e%g9Eln#f4;BaFe?7=AFFp4i!dp(2J-))7l`TuId08wR zY2B60sA)Ad3`LO8C2+(1O>&pVbT;RzBH0{>Zwo!WTVE|MWA&W4MxhE`=?y>6*x4`1 zh(|u(&eyS?=qMbWRuyli;m?I{o!jWosCii`RJ*q{q-@vuh;%qDJ9b>Q%q^&^R}IK4 z-A0F8>r9sVlz2)6&6EI{R;azx)^%BT+Boia{Dg$@tgBVc8-@XiGH0myL9<(NZnV2~ zhQ#fgr5{m<-z0Dv6KAYD{2<%oc-io@YS+*(lR{VvA#!RiX_|PBtWLf8go0N&zKI-e za1Qk2w3fqL6eh#+XRL!>51;yT>{Al}FBMS!n_>Yxp)*12 zKY-fTQP`KS!vYe@@Rb+@WyZF05sTcRu1(nzJ}-X zjW0K!o0!o@i@=FWaE%rKcB3|Ri9XdCxVN^K@+xl8wie76(q%8>u%cc$k}zo;0wfG) ze58lf<^w%b_riY)ucY0xxkl;ms_H;9Af)b5v6wys5_QKaqIO<#jI1Z7&Qk229S^1V zu9|o=kS(jk;%BRoRzE2sZt(Ba1BtYk$Js44!iDq5pX9yJ<@-)M*%z;N?JFBGOT~ma zHf*+&JA*@GEDzYcE&UNrJW8|V5anFSx~9O1!B`;x-|?vtf4WQsn?>FOr}gUbesYf} z3xM`Cg#02*>8^$eMTt-K#h5BO!kxl7MGVToSS8%AdjHGL3tqGAy;^x7Rq+$j$i zw#uKLucyD1#gPf& zS~}O=;`f-yTbgA|lfyuUR;~;}!qfRyca|l|x+pH(;CqX%G_SBoJWI@zB?~xFcubnF z`Sn+Cwy9^Du8NLX+=koFZpQSbT}MHs$rnp>eWcJQ$-c~~b!gcQMH))dyV+E>&0fu_ zCENB&oUZPC&GjRqGMRj=ohu1b)V5l0%D4*BYNPfCwjXG2Vx$F ztcOOsHKO-aMn#I`n%C%K{EV(*^Q@NH-z~RUV3X+*usmuN3C{$XH&oB=B#Thv!Zq8R z>+yy3Q5hJ6ItG%po+VcvDcPRj z>J~IKBBC;H0zlYrbiDUXo|@-oy9@{flJsbHMuP@%cZ3OY+lD)T$8el4IcvK$1@5K1 zPPPB=YqzN~hOe^AIPMsv9d%oODx1?9v1&eS+O_i4M4KL-r z`g6Dv&nvVKJ%{DGS@8jMpo5HQ!oSx@;%4n8fmjxfin0UVuTGvz=lrqjq2lX-SO;=* z23YrcS%1e5NUd%fTt2L`{&DY^4LeS4RV)XdG3mr#juqXlaXnn9metFsHX*JzThDVj zz{-rqwv3uL&Z8t(0PdSNe|{2FihxxepQ2XVy=nUsed@=tp8OhGMDJW(phu0HFY+^Q&4NumYBBY<|LR0?7vFQv3KwQ9>_VF$vROuo6Muj& z=2?w>u`?96(BxP+m#*ASs?biSH#{~r>o43Cownn;jg9;Edf%Z?C`Syi24x6HNneA* z!-+X0I(`tCad~9*=~^~0ai|9hnu&7pH+b=H$*w{zw^J+DrBW|aJJoaYx90Py{g!cM z_W5)XU3CT&(M9PwxAL|4n&T8|&Ezl0CfooYCjNt{dk7=7`BqoOr;ybEM8O>st zEh{cKx_Kk5d#qH;wW&qM%2R%-CoRiObtwxd=8ghdw1uGUiAd)>>O@Iis(9r(xF_Xg zB9N`1x@y2$EqgPWO*DEf!gqM|wH16dqRBOT%IBJ^ zQFGO_3*g~F8&f|?zo^?ai5w30Qg|{Oy|cy@E8*po5m{#a4)Rq6W=g+cW0yw=wP+UH z@OCe+3Fyhb!~2Xo67I-o>y*-_t%cv}!A5rKDG&<3%LDqaL$3G6Fhva6c%6^(+E^~H zMpy>GkJoh~=15m{xNNd;q1B!xNNBs9?>pd(dTl`v-8Iz|6-y^yK5=DmN&OkiQ3d4X z#Y@6k4RP|qpA10hq~geo$cQC7-uq<>1><)b%0CW-TW`Cr>)96Mb=#Njx>nrwQM8>% zuRzazH{n-qIspm^e<(COiWy`MQPJK?lOrCFL*5+&QgX;jzne)c!kC2qrChz+FUxt^ z8kCJK&s8f48qR`YciG{yVPg%ocPP6Xuqw;gs8q=I=`G-%)p!_do$|a5wa#*Wa_-;lhn%CJ zcY#fVgDH)-cYOAX#!2QP8Ax}WRts5UEPCzQo^X>y`y+nVzkmP!Rp&;*91Mc#5Lz35<_(Wa6Obr1l^#Z0U_G;vF_VMvSKtK|cl0xY2rbI(S%lhvkkb?8>nLyY(G!H8) zV=~I%i8?3>MnFcMg%Xw@oc26h;mIq=r~HrlD5CyP2gH}gPuRjDcOp9(OOdT*ZFEe7 zb5pHhV>N7_V^bYT#!~wzn3zUsgySFo(0V%PG)1kb?vY>rqhBZ-f&pRes>-1`U>U5P zJaVCD+cdr_p`Ema(4Pdz8>AS}Z$A+N*-wH96a+R&mDQ{^n#I4nu*%BvgeL-0a1l|` z>u7kjXr|-tNQIqbC|v~pT@EQJcwe03AFpeEIa)*@Ao&WfC0o;t^7-YgVA72W1g*rS z+A|x|%bll1DlOBJu{g#o4L6OBehv)_n`=+qcBe$(zquzT;&(!LzB}!?+#aZ9f7YKO zoBqo@{MT#A_XJ8fLVvh}JjMQk$Br?WW)Bfvy9U}q_X#S;gE~ibMMkVXQk%8tdJ?nrnrVr@X1lRSu0cq3s>M4-p0WA~`Ac=Cn96H%_Zp zd*+MU<@=A98poq?)(j49=qi#LctV2s+YR{ZG5T^P^&9^BzYZwH#hV6s9wY8nvVFhH z!pt`zb=9$m>boI6z;g);N6{d395AEPy&`ffcQ8#6Sd>R;z^{pRHT5tgZCzbm{qknk zvIOw|N{dYQyeFqOXK_Gz@zd-IYAmwH1YE>HPtDj)KsX=I_9U0doIE(f-%+3TbOpfI z`B_r=`ubY`c2{*t%^--NwjMB?9T#IF!9L!T_};O(gY322nfi34RNY+v%CC~-xa(f| z4agcnD{iQIS+Kto&m zxGxV5jRf3p9i(t>IZ{>sUK8$mq(q1y3A|W{MJSs@*jGv8tJ$GIlunuGL%J?+q6 z7wSy8e&5kWiD~+s5qQy^g+Q z_KWjp6wwOJ{Z0h~?En}GdNb2e^Eo3SE%X|HpqJBk+b-&R9z1+@5t$l{PY%(Q(B&@p zkti?1muYHxT72%Iek^gS>hz1H?T~Xc;@9nlA9D>tMhqMK2P=!Kg^5C3C@R+Pt6N= z@KyG^Hof=fe(F4>SCjV~FxPRl30U_7`wALk+Gp|(@tGxs%Q&I7s+y}y`8~GbfAYxVmS+6S_E3e~m`>uDqOMlGM80cMDD=Z$#`mV0lpW>vZ!{-ks(6yvbXE zGgjdG5!J9D>n{>JR-v}V021tHPOT(fgL`T zS(Dhh{%a3!f_CEHtg^x@zO_Dq&>H~)0uZ$$=S!NrR#YoyYOSA@(E|-OPMYM|CFw8k z9VK)rPHev|eHe*)Y!Usu`W1X%^n>&-RBYqe-5oy|VK|O{Zoi5b5n_g zejgg7@ZqVm^?N{jW$jvd6V@^P=zlIq-A=v0xM9v|aq)?w)gm}FtTL)xIy)6HNKMsV z@ft9A*!NibzA9@fqvNlFxfB{G@`rKH_+BaE|L{vPm|Tg_rW9QZ^W&_YitcfiBeR$8 zYO1c!j)|Gv{UPvSWNy@)nl=+$TUzLNTK+m}y|P!BTYCaN-ShR%n-blT4St8zd^VQp zZK20Z<`O^f_s)aKt^c#CZOs5 zTyy3kp8u+w`G%~N$HJtK0DaB&^+qLdb++(uf9SGlYEc)J`ms%4@x;T$c{JlC(o=lR zZJ0_ju^XxctoW#F)<^0hGQ%9 zp=Gz4ueQ}#pAA*C&iNg@hL3-U0O)A%B0sTLfeLGl*;uk1wfU@*Y27(AuhS7KT@1Rq z*^%62K!)#}ImI8lc-7z&&1P+4$1hthmmPg5$Dj;tuI3-mmt!^C_Rb{fiS!H~qiQ** z(A~<>-9xj~VsecT=|5#P_{fcx89UbK8MZa@ee!gJzeE&a>DduPw!XHr2bevU-j*%1 zi@k-F2<}{lK0?MsiTc{3@B>myy)jg@ht#_M#W>j z-m-nkzyX%IWcM%*VxQ;&zIZMj6b|_M>j@+HTl8m3DEq<$a$p~hOA;lN2fSZ29*hc= zkkBViW8jbZHE5Ow5{O6x6Tv)bK@0K7M4fmrG{i#YvTj6WA`tC%a%3z=QEk)I|0UN7 z+7bU!XE0FnQ6wxj-x4>qPT^!%AYzUR@}=i*(YbmSBB0qwogE){ZqSb;)ZYPIjOwy; zhkQ%yYO0ZqIbD;iWg=w2z`^c%C62WI5^nR4JePN~JiKgTDzzR7Ch2Tk%q#^r-_Q96 zBpfRXn7Jc&i!RAl2Y73gHLOs1Vv>Ek#MG@xGX=$H;jb(!n!;K7yffo^ME5`{qTe-Z z8IBFde9fxfUV>Qr^X*}=F}s=#TJyV2vl{o)NC0vC9a6E#vvgUlVYcNL?Ixz0xa*Dh z>rC&|OjA1HTe5137` zO`#uRCZWIF{>gr&_y2PGMm_T)ULK@Bgy$T0n)RHdL|CA1GrAm&&J{o0mDjfQRzA6M zwx=lnPXAe?x^RUN3SY+L?K0P~-{7Q>dyjIZQ|~vItzM*A(8tEng5;VY$g zElo@>U6UlC=ZrUg>=u$%aGSoJlR(nC`|nY%wWpe}y*UbZ!_$XH-o^52Xt=v7=D6dd z8|I`vLN|b$VtImz=2}6bO!~^2o9@_G-Qr``u>+jO0?3Rhn~UWiy%%(4ZeueqGe1(2 z1)uU|xkMY{u5GfbQ6e`6c3E4>ggp12>|FVOUtb?I%fPi^+Y6uHxm?eI%SMeIonQXU z*Pvd=jI8J)5>I?ZEEzj&4ivAcq#h@5S_W?CSTWy!RbGY7Nq9}<6>+AYZ4MymGs1;j zeVvt*(UCNERd(I^$yrw^xvQrO`obo41G(sNP~nUZm{|*>(=$icK4*A@-lslq97!n5JC&0V4GQU zB8L(yUoHmAOaBtdMPlvq4GN;@yy3Ts*BPoMJZXT3lpa_T5HPY)*)?lc*E61bj3Vu$fU!)99Gx|C zO3a+u0JVIuR`Ye?ZR5&hB#^Dg;uw1;J=)4<@!q=IN+S{FL({&$3slr%M#~jnyUOzK zd(Pq6cJYPIkaT^IYw^~Z?OI2l{@+(kF=^8#p*|U3etUGS?YYy38#v3Xpp~e+yQm4p z$DyN!fKM+|ZI63b2i8C{31pvjbat$)2fi!Jl-BGV(^l|rX1Oq6#x)J)9Ss&b2|V@O zwVvJT9slv_zbh*V>hodXxdEr2c|Mq0SygkUzO0c#UP`HSm)b|dsH-1HRqo!0qdui4+P9@mXH|qm_pSKV)L2OAb;rHQ$tq5 zG!6*2nrKY|oLq`F(vlMLUipj)>fr{crSw zUmU5+B?}lE%1OO2#^v{FMG!R3&dkFq zOr9|sH*I1D2OixWH7=h~3QRURR<*ZSOp8@{w(^qrPjWL)HVcG#O8E`vH?hQw0h%oy zo$;`HR)?C&c8|jI`)-QeUhwS5;bhlnT+junEcN&)9OCQQni2zNm&mrfWr-5fU+tnF zq+wsnfR(l)>RP+mn*qAZTVq3+^!-H@?Ia}RPU*NwNb4u)@P2_AJ;W3y*Hr7z1Uk{I z_UcMNYnWnJB(k_!d){lD8Q+;{rbS1|x(Wdgl!MOgvqsLS^1Cl?PsKS(3IzKJdu7x%w zSX&O~!R{?Kb6tD})2{sclVn{@jPzi>$Y+Ed13)V$XS@X622Y85tHdH;-n?&IVGFTj zBSxnq7LhCCybTgD>(X?7GsXrb$Vk&?gJgvmpT1>04tm3L@LYtR6A^;as*rt*Ckp zZn5WbZKrz?dh=3eNq;?Q`eaZ#Pg&d#S1nv?pwFodocPo*8SblD;4oQD3B|N)F2RB) zAeRXeP7_bRlIP?NE>nf8Y|6HU`5JOj)eO)c6w1i%+h?-#0(Rrmh%#@CoEijN?ux8K zTeh-@W=z(XVk;1tu1l8H2m{>RtJM?!aw(rX;lr{4BNXifTrKWCnH^2>3 zE`(yIL{ZA@hX=bNJPjF0Vp~f4BhqKh&I*|*J{g2oI+_FC3`v+R8Cpa&Ee=OS4wd?= z+1aaxJl68#&0-&ERhAj9ti5|{v?1p)8z*D%#xXhSgI2YEir-T28wnU+6+i6t37XhZ z*f{!Jk~6eD6d=`pb5ke2L(0ivjSoh}BMXwLaVwX$wfn`)Xs6->5efB`D{tAOh>Gmn zc5`$hCe=WJ1XG!Q!zd8q@7t|ip*33Eckjr*s)e#wa?$hZB%EY!u@G|ZtJ^Y*3%-Az z(Ud>k0JFLXus6eafZ1_oN#w8ePQIj+f}yQ6-g=-^M6GK?Y0MPegfX3)!S-GUw6_s1 z*BTH>LChSL5BUNaQNb9)r{=t26&nQcLoJw3VTO~6Y?W3wP_@=8BExsudfp$;nO)xB z%!%E?eI?8Fc7YAl%8hVL$#g&xLdUTv`zq>Y676$BYA^Gci>&_+kcIpSE`5CjLk0(C zx&nf2&1k638keSfo`!M3Bwo4!YPK%@LD8 zhJ}OsA@l^lV8Mvga`W*$dwo&L#aBWkL)vS1F7urr(%>bMhr8T#O-OJ5oL#H*~ze0vYuxQI*Ileop=e~4BlRzOaz)A}4z9NYF9CC4Cj$Gn`g z9#|u1;%kl5X3W*T`(ShJTSU8@<7{ODP4SNFuy3uF=!Ag0xAbil!krAVZH0Y*RX(NN zdHN0M+;B*@#&~$F){J(c!77-2Si@pjnMEc|1Z+-i?_h6x^2n9O>xNMzu84+??dPoD zHdD6ci#ALLmHp`9dt^<0Q92RJ=kqmc@2A|dG9Mn*<)_WjxWvf#`A%X;ivUJEiMEf( zG@=I3?uxXf*71?IrC0Z$`|9bE?a3ylr=33&yN84J&3BFaH>ySC+Je@UFULqacf4Q! zZ`;72V6#zTe3%K`*tod5+ND;_sbXaUWP&+ozSx4rJk7WP^dt~)7HjR*Qdr4RjyF&bi+ba}N`N%srxYP}TEry~F}AKO3c`h>1*b^^x0 zBTs<3;|}*vW5qN)p7saiItMiQoEhY40djn(AJS;J7lJk==acs8XhY-oYJQWAxJm)x;39zdui zhaNfX0`mtqRZzZKz-t6r9FO!FaPaB<_z@D~xa|J=FU}g^ryp2j|7)0G@iYzNU%eZ_Q#QNj z=3ms|8Q;H%#4Wi02Ain=;T`T;RP>r);8$2T)&YQ+|7U1KGS-GJY2FA32=uEl(6GFl zpWls6OnmK9AY=Uw6Em=~QcgrfEISL$y5Yst0px29FArVfk*i=daqULiF~cAeunU@+ z7Gl40|CF)fusiae?+N!{sw`kFTCVOd++R%QA_RJPuL8YWJqmiTWY;%H{g?PHl;u`U zF&n?wZ)HTx%sXO946OKp!qZO zi#GsJ;ul;Tew)As20!x$1_Yz%t+sn{dOs#57}LfbHN?0L@X`cr!Qn2@qr5nTzPXn2wT zJLrybyF+D432DqG*s7+2m1-4o1W7weV({iu2tsZ5SMoI~bX%BPdaHiqM+-FspC96E zkLH+C=?h2p=rG-Hy6ca>>grWXu{Gz>RXR$4k829Do>ZK1b|dl2>O{{~Cfe|Is` z;@Aw{3DMCdeRPnqSgnbDXUD&ZYcKm8(arWLhL|ug+g}s?Hq=+)nW8;SAT!{ymNmDa zAg8cW&cdivs|EX0cem}qB^4b>B0B)jWdiWryDE!^%%)@*jy(IFn0Vov}8^Y`rL>Oa5TNc+u6yS{Nx?A7Aw=#8U`xU5oRR7uH2}OD^Q=&Dlgb``+qgiZR`v@WyhrCc~BU{6#`RZXIHtK}b8Y zauGkBX1p}4;ETLT+5IAK;{BSdCcd#m>auQS{r!=g&vJMaj0*Yi%atIBeio~5S*7$L z!AdaSPT{nCB>Aas{Rmep(?z}j6%|A1=8NUjGcKuBPXh>T_K({7HrM$9=NJ3oq!7WI zH|VatvbPHZZVa*@T?tFyg8W0e$eLi6&V-od2TEf_S0eQP|q6r(2tI ziNi$g&X#A4&nXRsd&ZjhbQ*}M3Z>D#hZw;7=g!*ieWQTk4 zvBt_YTruMe4i4!e;(q>~76&!wG({e&yUXQ zO?l$<&B|EpM*&y?kiu1C~avUE~*o4iqD+|%i$A#}6 ze^`=DHOIrlhS-2iX=y*H7-P$1aE2b$n4o9ndFXS*ej`{qA!BS!zNhbzZIoRSJS->R z(|$B)Ls~WGXx!LwRpOx2YA%mzl<(ChjJ_+>Js#L6>jB4tXC+9=>JPm$ohLgu1~|%a zQ)g&l;G?Rj1mJJ#zjwGc9Yqc-pX*y{gk;o}DkdLJjx#k;FM?_Pijelc#()_OhTNBi z#}vNH!xxDqt^uUpJ+;8=7eMcT5qtSQ+{xv`dUSnH#)Z2}zo%6rx0nkcEJCjW!eX0+ z6EgzpK<5x}r=Rbul>IGw_1iu)GK{`rG8*Q#47HBBrHt1dM|%ps68x4Ttm%QCl=mJo z(M;}N+cJHwKlIC4KOO;0dl+`SvALzId%eHWjQ`QUBakTHw{Oy`k)|ZMgg4#FU{swr zgD@_+rg^-BXGQ$;sN`9-Sy}%FNyk?4ZwK_u4QCabB9L$+pllvR)+k?>pGn-{)M1BxFqj%H4xl%au}ac! z>ayYygEjpR;+%?XVwuEWaZa7I%U)a)-~Dhnntz8t&u(&RLMvAe|3#mJ#Kc{Dg7Ye5 zU-yx7QKjDKkGIy{7x&cHN>YYmOnFpUFuHRf~9XJWw7e<{}_Vfm3EyU6QY6vXDagEyeULx0K;f92>(tq$W2TO_3HMo-i+w z7MIFPeft#$Urg};V90aS@vbKTH{V| zY*`c_gN zC+3&(+*c>w88RcyaO|~6nwyU4J+e%6Rc&<|IWxe(q``fToM=N$@l;jGi42BDJkA)n zNNVWYGLq9QHgIZSxNv%30I+;i4GafgL;5-XYXVdP`uzFsw-Tcf*w?r3>$PD=b~13f z^!T&80gq!fI`k8aJS9l@SQ-Z;IMtj!_@@@O4NKvnqEZevDI+N2^{%u$ObNm;7`w%< z#;tcBkdmzZ-6ezATqnS?NW~A!k`56nXjR9&XKiZDq}IiB4>w_V@N>Mn5r|+_fklE% zX$KR~4=W$XxW0Akrl5<|(_;C|`c|4nv0ttb7G(EB&odGHi?qHF<8}0l$DQ!ApmsddL!Kk~+-*(~r2_^ivSKvguf&J@CD=?pOl~n( z+})8;GNLuzH6gq~5FW;7i_uA{fb#=;QbPBD1m?5V0zchlBE1#OqRbTf_ANvN*f7b} z7}Ej1puM}M+SvuHkuH`mX90dLw98xYdMCU0EHYI&+LJKUb;k`Gy>nljb5bMTOZF9) z+v;mr^+>*GGD&|mnYuTfOW`k^9~Ab(3|M{YLUjkqf-oHNdsMV(sz1a2PtHwbAGMRz zNg~gj3zKpdkQqUjYJN5`REbST#2cq}oi^L@;IMOo%QqnGgUzCsNQlsTzf^Ry+kdf1 zInQPQ(%IBYYe6KU>|OSSQj0Px*4XsyH<|kK@h_Zbx!>D}b?@ME);-lpZ%(q1MZF-% zdQvMaaCd-W;Ha4SeR?U~1-rf{f@Zu0tL>jBT8S!h+F3`jl5M0OOTou`qIrp%xx@yT z#D#{xv_8ivOJ-R~>W!{nF?k-^4RK#-Gl}s>bk2EKMp`daq2zC{HWyzJe@c2y7rNuZ zMhsCsix}M23iAdP$`N6e0L&q3mea*Y^YU{E8UB#0S8JLlTvJ7huc%B$-xn}q)dhg$ zd2&%eB4ch_r`-!{Y~xJh{hTp=W0i7&fq|rgu<-@5=|7u}92RuhyM(&H(e961W~>&* z$O^m^ucvXt+pzSvglD%bUWpmhpSGyqD{U765Kd-X#hmLgF1&rM0$@gOxarh$1YBjq zSbYgSZ)NJ?dwY6FnQ1YCnz#^#hsD%nkORp+A;Ja(_}5-NK0Ts?$uR58XEB8I*cB8N zY(Y=Bk72%Z;VIe2AFRfzMG*lCyK@4oZtzRJ7yP%75b6}i(nnW51+THTM8H!Im=ob} z$o)IV#^hKnW{{*)d$YY3olPfO*|re!@hD_KbZl5nTwI(k&xA4=yrEM$R3QlAf;?5) zhbs2u-8%~&7$4?MX9bcjGxHZ2)JuGPhjo&*MZ5 z6SKaKfQULNPFrUICnX~@Ghu&WFJGc!#cDYwNH;z(X|eK==8%?oG(+@w!Aa@uKj@n> z#gR35KvZm|XFaEAp`Ud}cQoYK2+_dOk`}DBRIKJpncNT)FyG&5nDMtNj%1W5UEA8r zy*ilbS`}BY)c$G%>%*+ZDm(JGSWTw*=Y3~N&?MA zwA(*_h0fJ&fBQ&VUqKac%94p#wEzzX8%QEhd0?^0&xU zCj0*(I(KpC4DxznPrEy$e0y;|t%& zLZuy2CP0@+4pxZf^}=IzCQ>L`DVnUON|7cE?9XQc@J6M-*~CiAjffa`Qu=TdElC;t zUT;5xfbh%R_n0BkE6&uP{4#I99}b>E5)R}3VeS4GDWPWdcm_s<$%bX%m0a4-)->p? zZEVxkE7rcaU%~)>c{$VNCOlTZjA^NGkr&(gN_Y{nK9B6YO7lHkjnNKUC4b)MCa~c! zF}RvG?w7|u20x6!>~@+BCsANVwZ{aVZN@Qqr=C3_?Btpw(j&l!V!c3bgzA};{=6Of zia$CK&@Rr=$3gJxRmgRxaNdwv7TFiKoB9XZ6#uGl=VGpN?f+2p14+3;s7nX)QY)}F zE3m+(k*fyJ*jcFt2REPx(eVCj)85$$k@-kdYb$TsoX6CJ6|0T~)f7zo(h{_xQJUyw z-SODkIXkBd`|BX@XK-`c+nyVLP*wSYipE;teN|MHJ8GXbY}kZa8O z9-7k%$RCT-VFEuE)5C}2_#&b0K0Y$RYcD9~aZ>bQO*F@vcV&>g6}3&$kqcQ$^Azb} z7wHj2mM6{bcx;8ngo6b!biC4wICxK+Pj4?WR>)UC8Stg9?blFFRhSaTSp5Pkz$Pg* z8NX#HxObJ?|Hw<*aOm(too<&Kd`G|( zuCi5zzb+vs#c>c>Go74BO7ZjFj52tCUYk|D@{`tU+R7|n<+DTOe)jaZaVl%6;($@8 zkUp1(RksJ&*VQG_XH92WtVC(H*3azQ=N^}8Hm2tvejy|daY@!97v5UYUZ8hYU!BD> zUHY=0+PucELKnxa4Q;l<*5nZ}q}>`R{5$xp5wn+?5V zasf*hQrJt~YkPg1b{aUH@HeZ8Epxyen+_LP+J?%~T)o_0aJNA+Y+5!%Z*|b}QCYiNQG@ViQBQ zlJ4ClH**<1uVN5La}ITo7;-RPV$K&ctffTOGYBhkns*`EWnr|*z|Nauw4bNHM;*&7a-m*+4T|G(F zFK>4j8n0Bs|VN7005I4?IL zM>oFU!rNY$?!&C0lu3W-txbl8QCj!vNE;@J%PRiUfVvG-!| zDKdEm5DPJNW4)jul5_Zwdb9KkV+|Iagn?gL?Xfkje=D?vCVXV&`t1G1A+xI2bfzEp zh=o579u?9P!hJs;qw?mdAGCVsqATg?p@>*qV(+0a3p~G$da&8?+ZodTC+~3c{y~+5 z$%!a_S$;VNW?||lalDTkAL0P%c;9{fez_9`+v_kE_V%a-??*XAG>R)%i9B~13%DsT zw2>SoKcc83AgcfLePtD>2pcw<4Z{3XibfdnQ_u$!E=<-+>yB8ou57SCikw0nrplOZ z4D5L%&0RQ(7#thsqUuEMJcThm?TA+CNc(l9X6MJG_{3Dpvl}!JAtoLN)EPb!}Z4Pv~kHTo*z`*2zG5KE5`!H;OlSlFKZY zR%g~e1G!Ak#5Onk@CNNe>J1V0fgcGpbX|;e&1jnV$h;;xIkYzj+55SlJ2|4Jpc)-l zMcBe^s-+lwVO!#1^N)fyYdWwPa>qi*7kAt44u3|9-Cl=e%0o=nGIVEbIj)1*xs z!@jYYw}6v~GLouKRG6pwAf*Hp7!JD7zcZ(_o{3M`)Z^sH zR7+-7&s!gpTU{G#!@L8MIG-WR_6jvX;w9zjg9daXdlAT~hy zYMG_0k(?XVV{m=d=YR1{i6LekX&;c_`){p2e`=eFu!RIXjc1D!(l=*g^?(4&^`_Qr z&hfD7+91=De1)>X(H}dUN25y0mVEc6*g53Uq6A*A@8f`zX)n2_tCMHpJoWcPxlq!> z9YT+}B%p2TRXvp_@9pY8Y+P^T`4yKG;d7dtQSyoFu%Hs_=Lr~4Xn3}w3rN8x7KPK% z#cFI?YP4UPOqUK>iFa9sKq0%RC78j)Bh}QiHm%GwO~emyEqa>hl~8u@?YgD5ydC|S z&}bLAT88Ta1Fy@MU*xna%wMt#{v6dO7t7p-su(a`xP84ll0bZ!C$e7J?-@TBd zsDBUET*Q9%gF5HKlXyHg(PS^>@&6QHw;_oC$s%~O!f7$aZVwmQ0#?YvTnXNb31v7X z{^Mg2^G=vuLI~yWe#q8=(9pDD%pMf%?r-q| zMW@AVk0q?qJq_0Ifi05c&fj+C7D7v^ifw_6k?u{RMy1|0b^Jz>CBgRs1)1mPXS7wN z;<6Kihabaq>GCbkMDd@1CmhI4c>}!0J|>0@eKH3to1sm@qjYSZT*vPHYLMC%NtO@8 zIZb`!KRexj8;xEr6AjJR-RW>GrSJZIv6tf3&`G|`#~jI`9HW1QVEl*c!B9_5<^&po zJ{5luTV|d@zqg6YM^gxY+J*LqMw@I#U$}0?#JMufg9|~Ai823sT8*;!NpmWY<0SI- z?c3;NH+Z1=_>wTf=d%|^2vI^&c!#Ybe5el`oAb{W^raXG`gg)hEvGH=-{Q*c+kY{{ zMYPTx4*u_J{Z2oBzs>mj_Mmo(zv(d{t)2fxr_%owBlDXtG+shT2&#J2_&o*1Yq+~R zTbF~!$44|Y+>q#Kw7?*7Qc_Yuy1%{DgTZAFpxjfzP*+#)=XJmzmq$j&$1jHS&4@0z z0qklL(vsr;E{LdU+?ucpi{zy4^4$!W3z8X5O_}mSKeJ~I`Kr(SIMebTblYrs!Yuqq z&2#RVffHvpa^%A{U*J)*G|fD)xE;~sDDubreFcd@Ea5Og=Jjg&u;jH_WP9pvgzN+B z%kUBfh~zQ94h8OF|M0L-PST!!*N>-t2W;NKtH}IZ5GN-raNnq}KE)XEf6RA8$LZhy zCV`>2PQ{jvd4ac~SW1?NLxioEuO=XC?WuRCMV{XbvM#0Lw&y%v>dT zz)(hQ5lz{W1Vq4y4q*>lK?}MqXu<_XCJbEWKD|aGnk+% z&eQPYkobo)BP-%32Cyg950%J2;!WX%WbyyWy5%MPXjksmUPAKHgI5|)FC}|6xxX2# zXm1=fr_RS^8aTDbxG{M~?kW2o^8V3?@O^^WjaEDfFsjw?p}@16Wo!uh^V1dL1ph4$ z;H5%;gmPYGXRh%U=gZU@!rcwLchR(nD zS|0;S#kcXa9yV@c1$@meY{fM2aD>+690X+Gd`H!ghfDa!$7hsf0_y=heYU@C-##yN zkhsC!ae8lX${41Js9dXCF18~V6%~PPDU)G({%yV5pBT_(5)J#Tc}tNR_W6-9?T9j; z#_t<}$QJi2P6vwUtB?=^9cX(2=<|*XFJH=yu^+QgqaPQ%R5P$HtRSqHlJRoTuOao! zQPSq^rSaR(hF*;oh>Am**WvOqZu*O@3eyHg+cg*KYxlW>mVh@aEhzpR-8guOu0n!Z zQJsNx1`FLS%eJD?={~bD$X#nmh7Zbc1u?B%bcxIb=Vs9EU`2gN+(ab&(xT`5y?$HO zM#deS-6*g45{u8rb2WyORG)XrR_Jl9ZqaS!*Lsh)hTm2{_VG$!y_mcX%+9o~mtyHM zwinBR7(@S|p&{VZB+D27+w0i4Du+{k@(;(NBR>S>Xd2OQ*sQbRyzK>3?ve4ctlQi< z#sbGVDSHjc;txq4lO;UOpOn6~^&9UfAvR z$+SA8(8A4tAMT}{>n2Y9Fzn&PDW+3gdIfQiP{NA9snX(WF^i;-RmMz*eamj}$fBbs zv*Y0&oy3O1c_F)|B!DiWUBc0s)+!}c&9tNN%d?rySbIv-6UXnPAx)-bWf5II0#i`P zXZEcr%tiKzfH9<|*2oNJW=%R57Pk2RHxsbkTxjqt` zcqS+0asD+ixR>VHUgnQJ`>bzA@t~QOa^v5Mo&dGok6|e(VnksyO#3ap5oX*AzxaD` z0XJa|Lsff7At;6W4d+>H$_doR4v>Ez;&tG(8#*cBY`_?3e!}z;<-&>#4G=j5;X@p} zpK)Orj*ApDFA$LeeIi>Cd`q`P(ndMIF2((icB>8Um_MNmyQ90Le00eB`}>sCv{PWb z?TfG<>w26{_umtuc*GC`nCN(O3JA3DBg!-EnH$Ciu=y{XueES-0Jjg$A^xvBuA^(y zgAt0qTmbZaXziPd@KL;b^Bf|}4!aL%t0W=uEYu_hI)J%KnRbaNbF048*!r4+Yf!d? zf{AF#gzbNDSEbjHCXYrH^jYbcFw2q+FFqPJrKL-nJh(WQaV!12c#qd-sBP^dWz0U? zob;{P*~OaZSe#hVY$PJbDt)?6^-BEKm7RxdsUrDoyKOzn@v<^jSWs%TPMitO2IwkePYtld@5mt4WOzCWYGgpj z;JB!JR--Kz5WMr*VWq*s3{z3y%PR(r;zDdfO! zU@|zH1z*3@00`u&bOu*j+NC4{<_~EcGktulHsVRCE`K_^n#{S*>XV~&qDrG5ja90W z#~1jaL<@=_c$Xo`VTwM}j%QA!?-gICLpqHBi6=kYKvwG_o!fc#O1~qW!ztlkQqYCh zixPCyEm$CRFqCS5)|J_KNRU@Q=?|tbsxImd<*T(r*NSK1mr2WXybGh$0^Cg^eRZ?- zWZ7>;V8gl<{n)6pG$^1u2R2?STbQN6{-fgrKssvly&Kia>qJ%xD|k!@D(9!Dn|=%5 z2(7wSjbIoMFuBeAQsM>@nqrX9Rv+AAuu1tBOPF|OzFz3cMYSBGiJ;OUPv~$4Y zy=6zRJD<8a6Y>apZ$G~Ws9XW=T1)6 zA1=JXVa)vkA-KzVsl@{2bB~%$R5jffYyLHvPhkcWDxsOjP%eiY%LkNHUbA=n2k*p^{Y9ciT&I|JP9Jul)(t4rcc`qT=+>L?Jd^-zv` zpzKRI@c+%+LvD>^gcx91HfCK787}YZkuI?_SW&UG;ViTUr!Fqr;_p}I^~}kvTz*Wl zwwDyHA46jDHkBOEQqPQ|V=zD8n*Z!_cZK1n`twWMbRU8Hk=mJ| z(h2<}_5I#_{jpc8enoAFtOYKGYjEh-@_F^fGE+is8jziej9R30`d@FJ z2S-%Xq&NrAwWNj$4n3yDMHt!%7q^>3Wj8hvg6gX*OK>^r!H2=2G1mJl#!~ODE+?W# zRpQ+{@h8vjnoCd4C&Bs<(Cq$a zhfOe;eNw={Bh2B$XxXhB!b}J>$b2ajDP$ z1`+aR=*E5lrivyFZIxC|v53TNM-7JZ#zj@(w#B>;r*?a%%%m4^Ilok5lLI*+|T|`yYnZ{MIM0Ugc&MVM#cvvx3XX@sj2ooB?DklJGZU2aJh+4;MOz z#Mv?A{=7958GxBiDQ8}O$;+o@HWbr%+@4dM&}WNzGqoVCXNXy(^zW|B-8i1^qZK!hL4OE(T<%7FTiM<96th~r-NU&t7VD2AV$vb4*Q0^I zBrPYKQW2&^`QKGQx?UyUu+6cmS}tBwJC=|Qfd-WdXaML+Mds++2P+;u!&c6Z$8$Zo z+hmc8I%%(b?zFz%mi4&UMLZ0kP@lh+($3syKAlth-c&_dQ%ECV7fj_tjByywJd3qz z5$XiuOF>B+7eMvKqpDZl9|nnffoIP=&plm_W8=!`_SN*w0`MMMZJ)9^7v1e358N~!d1V!`#KJ(FT>;tjCh2&9 z{UCio5l1C}*rLuTCYO&jsVi3SJf6D(?IH0ru3>Tw9|H` z?3N4q^b{9uK6y`M3Ym36cu)JN-rnvXsCsR(!5R1`4|<%NG7x^mdcLZ zx&Sn21&gU9WbkJO%qJ3OpQ9!bo#+^OpHC zhA2^Lro4lsWrtcFG~O~?M3=&rX{EgYov8P9TDt0h7FQbxI(;``RjHLY^@-}>Hkvch z;!4*_mk#M^gT_v~KIEz|{K{ec8lP)BG~XxmSE^$C`81^&zMeVS8?DthetNLE(Eg7DDbq%zR0Tynp9I2% zM?PZ3zVW~yWNCF)A#%J*H%m$y<&l~sa#{M6l0n%X9;aZ^a7%ylVbP-pRW2PS``2g) z^X#WZi^Gtja>@%O`5mRIBZn#(1E$c$m1c^|&xpp5x}qHe@h4eS=GW}KV+`BJcCyDF zi-$er{O{oZ$W9e3%uE*5^D?y0pgi>m^zOvaZUz@Y2`H(4wjxCv(tYo|Vcq#kZ8jbC z&NEs0vC?PMCe6RhuAa`EK5Pl5UsQGAPw#erYP4=HyW77lm4) zhS*+d<<&NXNiY&zJR!mdgBev@r6L;{3J}P18fr?YMefP3WHSzTP`Q!F9nQp-@=Pwn3(2U=9NbV)r#eb>*j zD66l8WuP|9uWiWrw=wBH&R2-7bh4r;7HUowu=8OBk0(WFO*R){%3Ai+9QTH8K7f~S z3D)J+qsy9hkMb?EBs{Ux;ofe$oT-xw)F^@T@970bL#V{1nAH+?KYI?k9VK1g-3Rl-5j;Z)Lhkuj(mOr%6 zvLA|Qn`UQ={G{xyY$%nmR3D@a%pBbz>v@8CagJK695OQpI@v`f6zgL@4|Hkg_BML|%$JOY5p1)oc?L^d#4YoQ{@!N*KuI*xnV7=?K`8rT zihRLjp}Q~IzCU10vac3HUCUd(Tu8XVg(}_M-Y;+hK>u=JBG}_07^&EF%K}5eJ+be? z87tW{Gs|5oFg$@L^lBSQ7RDMxO>vI;n`%Q0EEPlnar^`u1&XHS1vE~kflFcidr6(P zY=dailD{e*DKjsj-n!mGO>nO->({LgpyjRQ|vI z?V%zK69REpy*Xu~*fOHSeQG>+!?Hz1p9 z?ALm?_}Kl%4trVXbyp^ZAO&7` zMVm%&7k-NE(rjyX`h63FnVPsx0F{jdKuj$EX#C+&|D`SGw+?#nB>TE6`#-Vs0Mr&njFv15%P1+&122$$q*+h+h1v9bPU_d7 zX+c9n>3@+==$W5c|AV@JEA_9ium7&PZy3E-4Ek%TDu!{XLT^d^OIQ7GiE{tnkPE-A zN?v%XyimXVdaO#?flbvJ!{wwm!|UPfW*w7IAldKe$N}uGrbc-%TNV(W(z6yDu0dL^ zML?JS@|pV6r#1_0@1?Nq&(Gzhtye$)zPJ#UkRvsp{wC=7v_M^VxdagdoW5(GGAJ`v zk8cE??_ZB(4678C2L%VOr_eNKnKYEhlaY(${WB=)>7xVPU1VkmZ&5_7&?y7Y+p8A1 zDD)!~O{$|4Q={g3Zm3G?CJD>eeMqDgm?$=S$*yL8?EhqUITypo%`~iDI*diXxHce60 zRhVQ9*5Ss>I^Fc~8~x?`PvhqsoJ(z6-@Y^nwzrZDi?~(zxHWwFUHUjJ>Xuoc|4y-Ih5^kC}NEmIT;!N=YQ?At9#ZVdEM|x7pw~X2NGpa==177`J zZkt0Mkm06QAs1#hhsp4)SGTVN$(W^v6(LG1yP>l|2BkPa1EyFXxIUrwou8>f<|oAV zEI=YZ8YX7Qy310w!gAb0^uu|tsFp{zj*VAyzItK8Ol3M-nRp8roIhs)J9SC{H z5Np$~clJ8=nMT{Us1@sxG@HQcUsg`|)un#7=tF0DqKCykS%h)!JAsYi7?|<)vK6N~ zToXHgpUc1iB_!5Y>bx+2Hx-1Sc?LHO$0YP4|YjDAg;(fO#bIQA_u^$@!|~=RDiXw+R+%fC%dG ze1ndc!)e$M_YdvmgDv1ZV2Oq$RaO2uwa6G<2!Wv;33?4{p4624q$% zb-bk>M<~3{)2oH^s;eh4?J=Crf+v-Ytyi+TzvG6Y3Flu&nO=>*W{aZObj*Qr)jtSm zY>(6|!8(Z^%3lefUr^z9oIKToY(s}OIlUNrCc0lmZUm2}1aSFvnj44rm3m<;W&%R*XqnT4wbr)?NcC{{mz z$#?&*-j8~p(yg-|pqNwT-A7OE!>INFUuJFSwYdI>ALFI#9b)Ues?Cp^sWVv${hz00 zXPpLQc2-L7Bdxq)Zn$}6G4o zzk75Wn3lvlp`9dOy^G{HVJKhSWm?bA@1Jjz(sV)Hl*BR|yja zFUUQ|JqRgTt>x%--KY6tk3|R9W(Q<87kEkSdxVF8hmA5D6TKy(hRqQ>8-pDyCpp_S z7EfA@T<@n>BuCVOb9JNZi`x|;PQ7ba+pD@^yG(5mlA3%2uN?^P4TQqgJ9loj-=y;Q z_=w<+JMH6_B`F)(9&7o$x^hB4@!IVuz45Ocz--%R(|@*av#=FO<GQqL;5~Fk z&~Pyh8{KI%+Ay_hv77AZ+Wli*#84mCx-DzR?i+4{7xlXC-WARwfj2DNqORMQ=YDO% zwn+u4{QJ;J<6G(NQU*<0=UGcy#ZC?aR(B&hJ`>=Q z^O--s0?-RNIS7v~*04P(r*l#Fdn+vwNq2;k*yK4`UB|b#i%qVdrYg4YX(W0d zY|1O^-OacSHVzv=A0>vtTj9D6xF4EE}cSm)<^LCItW~u~_ zL{8F{3@HNV)EnUD{U5!ZWmH>jx31elsX%e}qQwgo_hO~EQy^$@r&!Pe#T!DQxP;zVhQ^S%I~5DR0DY_oB8&ict8 zxH~@kN*-f$Qlw%+Tjl-Hl$Z|BQweK0k5TU6T+415{%Z3R1+I5t zc#sCgigdi2`(28wK6Fl<{?R;H!$D+gt2{&wk|yi}VKK8L+~q(tYsewuhncN@yCPb>7xhLZr^iLhS_d+OGfx(aV3$_ORM(f5Oz~M~gQlf)C(sA#~z_gwMWrU!mZd z<0#PM@gTeZ{O0UQ717Qql~7E=5uP|um1u4dv#e>M08$s_vx0>aM>bJF_v2m0MyC*N zYhpp@lMO$Na$?BC%adq+%RfxC$ztTgY!1_wXRD8T9&bvLKqQ9Xa>suRt{80^laO`c zCzdN)!m8z4lc<`Of`)lR;Io@B%yflr+%?~@3-+ty>A^Q-5dB%NLe-ySs7qfiVTl?S z@YyZh#6jVi?FdGwn87K&hdf!0{;ZS8OxMo405prYntS6aKRe9@sj zcAwwcZc<2kqRrKy9Vp89 z?Snx6>3dlQmlE$I>cgFoGJ;bzj@(9yxRwZ)F|xQ_q_)DK*@C+6aA?77(e+bZX>_3(52F~<=F*GgB@9lkQk8SyBAX@A^(cRp07BvZvR^&Jkug?*zHq@tw|FXv9r-9cj9{Hi}?9bhz4=iKvv*`FFHq+a$m% zlG2Cc6j)KT@xhrK?m+NnMU}Xp?V11N+pFD8U3)d-@(8fS<-oMXbcsao;V&bQVma04 zp#BRSi2nt}Ou#WmS(EJ&bMmiaS9a9iLN!8;{Z8-d39aQw$;V6 zRTr13{ACG;1yF;7%)9BAopX@WQ|Y|NiYw37Bm7KmG{lnL4dS*v1DF2RU-zSO{)}0r ze~m4n01pR3;Gm%6W{g{re9)Z5yi%jzOpuTvW=2~9n^4Yddwp#{zaNm(nNrkhgcpI0mdaN*m1fAB@&x8IxFNd4U)sza8(b|Lq=5qkzLlRs>? zZ8iGKfsn2;yjwhHFdOho(3k~cLtv+FRJ=jr8dusmv@LqaAH=m@i5Bbb*Zbz~fjw!$)x&|*+ z?&IN_J5HdPaC?h;2u=Zu*&w57(M}QnN5!(%xlRnOA1caZ!&GFJD?|#vEnB9okC0ME z4#aWLW9kY6}>j@ZMWvPZhg~|2t^`e>4L4igs5nJb1Q_|nP)@Hy*@&nv`xIYLJBX;{{Q)#!u@8i2;a^!Ex04>fR0K{gEf677I_j^{X7lh2D ztxW?2$~sJAXp_2m9!nGkh2nJqBCHF8S~Ki&xJexqOg3eFJkNu8ktHUaGLe_#4h`y< z3aL~qi(WPW?|@R&;_%R7w}rTblcZqYcst1{JR<)VKBel4BF;_b>?jd34T+OQ@XBV4 zQy{=OHv$Wk2G>u0i$RujSNG@)%T4$Be+ZDO2ObqA*br{lwzgZ*Ssol$cB5KUP+de7 zT&-Si3Zk~-AoSdG=kMI}gEF0fA$&$Qrs6$)0jXmKIF&p`pKI{GNrZ{ec5CnM5ox9K zX-Y_*vjneC{Zn6D=bydP7`&ZFS7!B^mbJ7JY0iwi@P2oH2RVR!fE!{f%h3DeaO|?* z`c$ROr?k~up`Do+Z!WqTjkT>5Wv?*s{u%%TtUTXX-SqU$W~+DclRt*7r_XPCcb)ODO1Rtv^SmX;j1}?MA z0>*Dc+t&7x+d6es!eRc#XFf z{B4U+gp@zd7SpT>dnp%jyvtIht4<}Y5NqIDT^ojv-n79MQm0=qq2L=KVO)HgszH=~ z22IKDt#C=(++Gf&>%Ou)!Hkt5VxgmEOg~)eUo!jP5-(VP`}Nhvvx&25%50ZYtw)i| zVd#iu0a+{K)8J28*3b#capPMgE{DUa$F@fjl;Ap2iz9Z3P2R1ZfSv$Cg7ZEhBG=_P zzx^72XAMbp;XU80!{J5g_jux0K_*vi=oX>G2~VGSP_V&85oM(O=4(+Zu$Z5ac{=G! z%7u6-e>~+;ZznO%E>zMsuH7ItG9JL2yojOJUHpA^)TsLWcFV;a*1NHZoi(t4k6&%F%)vu6C>wj+__=Aj-qeA> z>~nhhd)IP%tNlx>tKdY3{-DlE_c|Vi9Ksmqa4ZVcUJvIv=Z1f&h|VX{gUQiEmpKJ> z+5l#>U)eWQ3)+E7*0MI(OzwAsClH_`}e4t+=v4Sr@9Mqfzp#I8?9rM9AM$v3@QSES#K> zyAqiVz&n7xaKIr{R+5x@p~#TP({ajTfbOVvTB;+tplJUe(Dg*nwvWpej(?{8Z6En>{FK{xG0 z(?bKDmX=o*(fNkN*Q2TbNg)h%p>KyFXsjAJ|66R#S)X+~r}$Z`G-H-=HY!bW@pqUW z)uXT2DU}P=e)&DIuN?p5_KNu_C})%ttw0eZmY`D~-MzgEy_G~;zJ{tjhuniLcl*|$ zJ>$D(m#w=gmCx-Va!n8^N|=_{f7O1Nlq6j_P$cY_RZpsD>ASh^o(V*aVEs14-32Fa za{sbU3|C(Ky*4Q4kUTulw$~boYu{;>+>$i*U#Rxo6RTH)VwdGkwifkxi{$T1R%<8> z(iq0Sv{r4`1#zZ)@~&T-rFymo>f|8I+(HD-$ioGXSzf$2zeBY9lFBx6U zwTdY|YkI}t>Jzvm)CAr0NgaUkkfUuIxw70ma0H`Q0ap#i5Ihdn3(>G9fGj=C6~0@+N!_mOIDE*|OL`r%~!Fzo1dsp`QSk zqxHq?ckJ~FGA?D!YSI;t^XvT_P3>_~nxQ16@A8P1^p*OXLfetQaQ5`d!Qn|soQ^}O zss+zsSjkmJK)t>{-JjhH zF2{7XhmX&BN5i@h(c@A5Zlq#ZeqKOq?!#&^x;pB_;hHi)BLISj;!`Rg46rMwLf;KJ zy$CcOdZ%DijTj3&Vr1a)mwqk@$AB`BC@Noxr#an#E>0E3zsg!S-)4;^ux?H!^FIYFSb0D2UhUE$qvhc+VFf7Q$c}TN z_9N1~2T!OcvTVGMtq0*51y0+&+oGl@J`qth)CW_&@w>()&1H06%P`XG+<1N{kzXZr zIt7|?At(DPTGJGvlXh~I&ua#r3HUvCaR&I1g%ORT`TXN5$Q)k}t?B>;mmB8aUk;o^ip8LXFwTg5YP z+fe?~nU_^iv?UnY|APlBN;kzZ*jK?Tl^;3`l*VQsJ&jjt`;&+o^%mLdV$hCbH%kBW z1L%+@sDBYaAv#wzPnWft6aI33Q^l)3u{lCFY`77 zfQmr{&R?urR+4YMDR}RPYL_Acci%gMJ^)x&&wJ@*@}tbmKNLQ($5gu%b+0pVt6R

~){UEh0*M3^|1Q;$*p_geu!`&6Dfs*S{~2uke|Osd>EAmeLCcegb^lr$ zCz`04NIu1Ws%aom3@*MOBqeqB=!}u+g+@O9-94;Z^BR_d^A))f;I)4irblG-7>3+QTw*65RXP^5Yxblm_^+CwGPS-54kiyvqhX^@e7F2lC%W|;=6;>{9CQi7&?N6%L7TasV_Px@fOwWpLLsQR3 zGpfv=A|-1z`*m3EVCc^cQcA5knmIqy1z~>B8ZCWik98qpWIdSfxowAH$<5+zjt{<0 z@~{Dla@5BxR?heP7F0!jpWQI7?K8^SseW)*%A>mh<-w*kPRKtOsa!?CdSA=)&=H(t zhPJ!2VzisO7R#D`CKr@TcXsy0dc>_0r=ar`jR(Q&^Px*05qHJMo<+v6Dtc=l>IA|b z>*U-yMRfYm8}>IX^h2!=6b=#{r^L~yr@oOwX5L0&`sB$a;B$;pdX?I*6_Utu(P&@n z>JdKEBFt7xH1i4e;Yqj5vZ0h{DP9&lnLJ117u^7aTt#%m&v4FhH7 zd!C{upn=*^98D_*=a+PpFq~cTvM8#VF6FYWz3^%zq9X&Wmb)!cCloIcq4=%v!en30 zY$eZd4SLKu_MI60dfJGVu>h6;h-%%)jp{S?o}4%~<&@7_@5HhtU+3L2&$#Zfa-QA9 zr4cN_to%^Uc@!wUw4}*p%WBh0E3b^5wl(MmJ5SSM_2ze5E;@S1EJPtR#=6 zSktl7hS5O*Y&2mkt-)Oa*S;(g@iGr=FFyjU?D$remNlKtO13+;UYuUoeOR{u>~=0! zNUL=y>9v_5MpRh<%t^XX=^`@~hn7{jo?TU5meeh(`BXt)W~~9v!yQA`O|M8wi=6Zd z-yLwwaA}o)KohqkdeGUgzA|&W*b^&f3jF(-Qwn94TqUira#kw>t-*zo5sj${&3-b= zrS+UC?oSFTkq{sscCFO{qxrA2jHoP>eDdyePlM} zRwuds>(^hX#WX_7?!5-?l)KiKm7JW{YzRN0b!u@UR-qKQhvOy?i2&YJbT^? zLakQ(m($&5+J9clv{Dgjy>#=1Px3d}c&~1tkTbXHh7*~HRDOP&Ie$NRswXVmI95CI zIvFSw8ubSjGYyErsFM<8rk^_E6TDQr({eQLqC=(I9l}q1wV-=*IO5W zwsvUK>Uhmj zGX8~Z^Gs#0UhujfbgIPfx4UhH=8h+9wwi_4Wf#l%t#X*Eo2hBFg52qv)z&uFH0vnu zg|xzu2j$|xV<=uq=F8DZQLHWW3}MTpnBTU*#`%F4vzU&3X4CM|(f0<|Eg6L_cTptD z=G&+m#4)qHkP78)tgcD3T^}T&Txgt>J1y{|`g~MA-Key@@{`W$v^egs+!=*OeEadS z_8QJz!Z^u?^FUbMq90w{hgngC`D{2+P$h7CPt+pD`#Uh`b#vAOxgn?%hNxOzNaGsQIb6Rwku<#7hzEXMBayB@={(*@m0VSY z%(*-skx+~U&Ax#T13z#Klg?#e6*twsBmkjp#y)&I5`pD};BtJ_qPb{6t(C5ealBVOk)3^3Z=R81vD9j#~!fCo~IVpQ|lK%_7#+SPj z*d-{_2e{N^efq;p^_>nc_dg zp--#e8$P$X{kQyH2ANjlHGj2icYx2aWMI_E)(?tG>()jta85{yVK)=M0L?4&;NU^vdi@_ty@E@pfZlBFaAh z_LQU?-!Vw-_GIlPLJ$2ikLSI*bAM)c`9zt6KL>6q?~sUJaBY;1bj$g^H)NFto3Z?1 zi6DP*;nW63LTViQUtW^Q<%uRfEitViXH16kZSqeYbbiDPBcxv8l=jeqKTZ_>oZ#h} z|DZY`VrvNO&RNc*m-j(V$aHZZ+md;Z#$AMP_lmwn-y=(y7(% zt3xAx+GF?oBJ?p7AR6~_cY1j9yf6!dOR?SbTj$e`7-Qx2`os(->^gY( zT95G?x$+8SE(u`P<}-dHwWYcjGDEHBP=s!>tK>12X=Vkos%x!V0~O9@E9QX@;9~q3 z)T)@7IKX%W-70lZyXKgsFM-=VNRA#XXSvw!z=1g^b{Iy+Qe5m{x3{7_l>I zkMT137R&Qh7=LtU7GoLP~{@cQ`cc&tgr*GCTbGLx3A4wg2 zS!$fJ3+E)FR-OfKk^Vw z^}PK8a6Bi76am?LGmri*4?VVK#uD!2lHi~!Pecp9#N9FrMpvzap`JRYL ziP5V0B6Dr*ae7?prQk>%dQyu^W55^^F{IhpX>~3)`LbN(vQDwV zJrOCfWA*4Y%S3Xv1Jtdih8WRqaS%$s`$RG@q*$6UsyHiMYuf=k@a~TgYHJ>N7V9*$ z>w5t8ni@>^OO+Kb%stx0B>-iFK!~pc(b)JvqC}FFHp5?+HxXSgIH{eDxzrIV>O-H$ z>dg|bZ%gysX=~}S7)}?t&0n&A5PY6h-jZ(=tlw}W^;$4G^E@%tYb<^`l>d54f9Lvmdn+D^Z)qF%B6 z6X@m`n?wI`jv+wH!@Qq>9wLP%(V?{?tz3MoGvM>A#@Iym^Uow?}YRJZ#zhP6a z)Yijr4XsxX!}uEF>FV9I;9A>8|M?NU6+0Vu-OWlQgQ|j8QyoeK9oi8r(@9QyMAG7k zcg%r9$(M>4R0-Yja`IB#G4D)bY~2^t&1@NI{>!RURqpBu&RvAY!0`0H-DsHq4bF3z z{>65DZU5I;CjQ^RR1?ir{AbIz{~fj|{R<9x5%sUN4fC>UoPmbM$JfXItlXx7sLVL*wwoRp%ksClThk=o;Ee@C6bQDY z8h2J=!{~G6a}SSB!Xo>x)GG&+x1f&-V`7P)=vHo3j}DYtmk2x{4ltdSS5hLdmNJUA zd5X=OBGvRyCjn3)L{^D6kwESVe(vFFcAigk)+T2~h#V@YB`R0yhUVt<{XJmA<=^1` tE9=s8fxj2;0z{8I;QLpzVEuV}e`Plx$^u*v5YT)7eqIbZ6q_R;7#_(< ze$wgiZUJLCqNKy%MJ27i$v) z7fn*-$)YHe2Dt#6XTC+)-osS2xkBX$?Jw%gcIcDSl@V<3_fGwl(Z}2uNbuAB_YiyO zkfQSM1^8U-rE30*|18J8pqu)dqkPo;6|o4dGzA}lg%`&T>xol#$^)o5kwY6Yr z{d7kR?2@0|*2DSv+qt=$OY+s#mFoh4w>mAK3!&@RujQ4z+m3n$*PXl^9*? z__~i(CEKdNb+cq*S)Ovksk%lh(C&av)5isQT}7!~0+vWsx0)zX9331xgiUYMk(Y-3 zbRHZKq?*M8>{|MYg{eAVZ1q~n-S*f;sa9$d7SZ_CRm52}FM&e{E0~!ldc+gcgt53k zjDJ)B6t5LhTo&J2E=i8Q-(XMECi_a?U85jm|KvZdmD0iIEsH@a^uvPI4~vtE9_?%M z7q@tKsZVXKRusYIC%-3;cT4$EzzJ90>Ulb3tP{Mm4DAo&lI_anCz6o&@C!L7SC7+f zR^ht&T71ix8A4$*R9s%~lwOoZxwA}&b*)U$*$Y6X znz~;#Fx*+<%oW}3!6c-(37BB(X6@iLQ;vRkK8<-ER!Q#4x*W(=5O{;EY^ZwaX1-8qB-=SUZ%j6^24hJ*X zgDPu8V8?8JQ;}(6JA(;`Qp%juWFEN$4V4P2=7^KaQ9Mv;sftuxBT`<~Kp8!?%ZiiR zvGfBOQsVO3`|=sj-6<0A7%x-JH@>Tx&n^;)AM8YT`JElF{)I_y{yocd=8e>R5p$mB z%aRKLq0`2FD_j-dv3q%kpJKW5VN`|2$8x|-2|YP>hSE&e*`mgB!ZcN5-_*n?uzH(Y zR(jx(4CG~*>f$<&-87^Jlb(cBL(u+oJ9WBE-BhG>P2+Kv-)I^dv`9#zkd8vV$cCra zsZ%IkB^Lswmvq(ZXt;SKWIP8`nb{5`O0>LW$^Kn9qMq?P&6B@jzS|@}a|5K)WcO=R zkDD8d?AZWv!zA0|rK}1}Eec}c=GaXCw)Zp>IQ<=(e{&ddoJ>A4epxgR8<`l=ZSjJI zpqyS78*TV#_y4p#?IfwG>A$YKo7u;CVhUAs>4==xC37M_U*R(O^x~aEo4-7-s5wEK zb}l2YBGsCmj-O8ZkJeQp(VcT$L$eJ z=Muc`G0ehyU`F#SeILo2b4m|wJVuH^x*zu#U(yW|X&i?&58AgeONf8Wwu)ik%(Zg@ zE>7&8uhdM7x37>U8C$*c6dyS+79i6>6tddnpQe9a!`J+M`eV$>?MbKCrF^3rg2$k+ z!|gD+l@C%Q0dCRz21lrg?Xz*x3pMayTFB5;aI z5&sd#?Q6FV{-l#TgQCiOE?#q6g(i%beq$zF4t$cAg8*~hO8?$ddy=*VEO=&m%Lwc9V|5>sGX0cd1K@2Vs&9k!!*X_tLiB ztw~?|&OTVYnNHHwjg@KN@lPUZN9&Ba!esPrbd_h5JYE3M@dyUG-z zP#i{Pt>D5E0HY5mCy}!DXW+glARQ^7d+fg3Vm$qNClf-vX8Eo%CPRbFAdwGU$Kh8K zKT278z)E{&cZ;v_8o}=)d&H@E59t#OqRlDH6ZR4_2j1A@ELfq`N_s8k=C)J_WE~;3sIMXx-Xw1i=AYWvrYT3 zK1&&B4E~f|G6E1J84HpFRX3Q+B;FYN+ZaEY8s}{#EsXH-;G5Wd951T;R9rC$ZW3f} zPNJH;L{sLiqYi;Ml#_Zk@QCLqUQ^`as9kFD6!#sSaFeA9H10R+e!y`kqf`pJ>JL($ z`&=quhc!^aiYf<`j&np)SG>E40;~bT4wAt|*!-GGayLMH zSpcuq(3=8DrDbNvr(5sk4)s3i()QgQ?U-1LLgVFxO8Tr1r<~DLs)N^X{~bCrt9!}T zyF0b>z`Lx-BY|O=#xg3^z!1-c5>>~*Se36hga`=voKKaRLSdGvRaoEFQnx2NfQFpZ zzCYrb>QzG1L@5}av9h0LB$Xq&OkrxrVuk|joVAOKn9NK}`vGlzGxG6%digZHAPn zDEdxvuGo}I$hR|iYBP4KI&)ku2|9@b9V{~Rf#0bL&kPnkB89>d6%lJ`XuyV;5irZ@-LZAPq9yt9DoP)*rD*?M_8p8hT$^n01_n0NT?rH%iJ*DWmb7d+ zG21wLO=phAv{vDwAZmfxRD(%7SsT{mb|?SNyJrp0?|OW>Z`JIuZphfD9iMqoD3*rR zjKkp!oDMQ(w2>%xVWN$L5oi&NL^m!J*~^TC+MZ1voGtebO&tU3+Yk^O!w9tU{l-LF z9-QtgMTozxSJEa^2r4_ z%PEC

ahqBIH%Lo1<5EeUJgq(hE&Hieg=}$AG?}Tdo(&*zDf<^ub@wG&+po{aMqT zrWFI%wEl;I33ij^)8m=I+;|IXB<0^d$a_oN-T=cNqJFD+j3=ZMWB232R1v)+gu2FL zQ@p<8yt5LQ87LGs#UvA#HcYA6r~G94R++C7roQ_}%S#mKggI18#Z>Y0%3Y^ur;T#U zoq@ynlagzHHsRMXn0414%w{cX8p)0P0B8Q_-;5vBizXGaRCBmRPq0EzBm^BLbFZ(s z?RNWel*-y*k@)wY=V5HaEM(2Hk?u>{gs|-RAWTk;zx}#hs9Z8652;mRbUj~+_g;qO zer6nhJgZz)6BYg*~N3|Xxok!ppcA*9?(damoBqDjZ=`^^C>K>I1x_=wz-5t06G3r+KnuOD*2=Ni; zJ#Pa2MmH}d>!OR8t0X^AVS}!87M3(WZMuYI`MWQ5gj|xsr~`g9u%qO~#cJ^3W^f|8 z3);Y$GNTueSld#nF0j91$YQ%(Baga3am!d|=SK-I2HQ0k|8?Dz_Idqzr?<;Xm$eUT zRR6*b(gr7k-k6`ua34MLai0;lij60D3)*~AAwTB+G(JzR78e9VmAD#%0sjIw{91k zYXT{6N*;q+dQr7qLIH1uJ47^sE1+5UvMz5EiFAz3SBv|bAA|akCcSFrU`C&}FZm};qv<(CBoj(Pb%}*5SVaBm6aluP`hEl>u1lMdF0t!0gy9*F1$e-i z76)-F3+sq@Q3~ddc;BsUiWRdbglQEOlPJzg>+)d=XQmJClRi>k$+>l}!#e zv@YvNI;2g;HxC!5?-1)fFSD4eSxhW`?lXOdS~@EnQqw{!yZI+T_6HF?=Zq& zWnrp{wrg4R{b>E^_V%q)`Ws1>cVv|3YF2XB3zm03S?EVtI8L&853hB_jDm@fUkAj+ z@h2zG{NMrON0*^I?Y=q;AmLpl(5e6RHGQBvF-&Z+eB{;39-&|y!i;dxrUpnSN?s-< z{paTnxZF;WGLQOMLE(m8KWpS}P8rI~5U#(yT(R?)EPM^CX_0I7z%4mnXTP;6F5!!5 zfd&)(745PX4`YA%L`ZuE(W{gt|*MJGh zg^!NO?*J*jVgb{LT8i%wlG9PRl>xg_$6A)>hHMxqKJ_HCi&krc=XV+S4T8I zC!sHG$F#z*_`!>)YZa1a{sOpU>RC{fNBs0&=&D z*CQ;rgvZkAoEVJH9VhYeUL!_uJk~9cM+eFmd@AB#C*!TkRtcF7E~e>$kPw~3Y8soT z=AFKv7H=1h#3-WPL2cv~Z&tU;NV}M9kS^S@S0J^nYp4@LMb)ytQk;?G3t$4~fYQTf z@^GqV^m{%TqbwF*=7ZXQr4EMJhk_Eo z6lB2QP5-2X_n-9_!h-whDZj;%)SGY@demL=KFzN(zxVSSs-Zlqm8|MW3?u-5iYo%# zy$U4y@%v43RI9fPWcTD^Lo!7=m2;GGlB8XDhTag70>=t+@cR|%g0WRW>g&7I2VVpP zg9MJfABqnx+_*f(*Q(h30!$UNeEkd_uEjh)nsj8bvjo($c>0BOyi_VM)j=}%o8k6< zCm7p!Qu^81NhvA`Qb?MGG+NXZi}^-5ib$0lQRgSgX5KU$%ntfSv`uTgHx`ZmGsEEe zv2mlBfWmz2MT62{MK_-{v4w=O*WF9$;u|-ru7Ad1G4ELBXQFLnu_+x;39Df`P*)g` z>8Zua-4RrEj_Q@%0j788$k$8)R)|EICvl#B`+1@J|Aa6r)Jio1x6^v%a$M0Nbt8Y; z5YNT8(4Tv!!CT0yrPFjSNhOb;3|SIarw5b2$x1D@dxQ#fQtP=Oy4x8S-VAez?989F z?tWE6;U5fCnKESL^&)Sq%++_m!f7%R6=uoCa;R7^@X%-knfP*wZ(VVks+p84nyb^r zd&eI$QqEil$6qJBbn#8MFjwc-%Ql`5?T;cN8&9r>x3gn-yu+8w0lzO{CD85Ds$SxI zSbhS5!W4Jlh$}5<^3rOg_|=p*N!HEM1y=-Z$F5%iukOdmLrKhO*6aM5aq^1%w!B&8 zmV;@1Z8W7~){aq#w473;AXKU6iW2xui_Xq5y}=4-5|)4-Q$o@>BT>1Yh63A=(u`y!I4dwVBE>NBNMMRK*PH9DuY@4dbUf?Zj4o9ieUAufe8m zr7?;=a&yGDvS?bvvDa^yF>VY@3ybk*0s7)nZ3xj8*C=*(9?lPiIw0oWYhWp(io|bL zv9=foGYQUT8)dUfP@3)UW|GxSDS8K;0C0Mtcu$RT558)HpBQ|NKAEST1G5GgP%}tS z6pJ#NG(GBL-^9=n+h$>+F2RY0NlUh!>qSh`TKXIVn8;s>g3>hh#_#~q_3<%0Oz_9y{C4Cf zn_=@Y0b@=K;YF~z{JZl^w5_|M9&NGiavbKVXwIgGcD~$@3YHj`;gb`()FE+F0Or}8 z#|O!g2tjGgo~`QNOD00>qj| z&cAmSyfFUECW6gyecSf<*oj9O;n~5FWY=?Iw@%=>Y=eL9kM0g?V!>G& zAG+G*ty}Z0*2E6_5+)Pldx}G%2EhD&!?bVmxt-B-Tia#Oy@{?&AqS+kNJ_cKd)$Vm!5 zHZ&b^tr7Sg^}5qG_9v0xWZc>M-vbw}*fUny*0%0ff=ya;?>L{b(2cWw(3AzWRyR&9 zMH-Da&xflS751IETXQzctn&ubS;MBk*A%_F4w2tFB$bOmIee8h@y;`AbMUke*oB1$ z%h=<~;zILu=3T$lVOxEjl>uMo!GvPvh_w+f)lg%o#6Fm5og5eXSCa8gb!3VO_P9wR z&Vu}oT2|pO<^aP_iDA5i*5J$X`8bad5KnzH~4&bods#-%QN2B$degKD>isZFzUmw^G<4hHiiLLDyWQs_*%tR(Qk^dyHZpt)_#V+BW3e;1We)g$ zv=B0QMKpMwXW@J)bxu6)F+bjTTh!w_2C*A=+0#v^5T;}({{xNV$Bj=cmD8aKcyoJy z7mMShMAA!P=K~D#4aizVrVl`I0FW>=JMHW=yW(DuAZt!We|WJF9I@~^U%({ zE7&gSw8evv3hS^qhRkfFoOKspu4$`Px~(-%g;VD!+RSRV*|98NY|t^;Z1N28$29lk z`%*)3KQFdS7L>Bb>;K}}xk%$`iG){5g{y`eNyv}t8HEZ~B&-BOGGz==<34;fHLKWT zf%I&(!vZJPH%v3)H&ALba_x0mbBJGp@`a13Prt5$^IAfI$<~Dr980WDe}JwmKXQN_ z(n+nRWq?)y{Ux2G*#0PuqcmcZ7XpW&t0!Vn28p=A!s&5v&f}}}q`mE|d@G`~i(IdK zx9!#2d5CvQd!B#UN8DdKOnar0=6ca1X>Q^dl7!bfQ#K=9LtSghAdqyE2R zrUy1|00;_cVtJ1kO%ueeR3=J3YVIoiV5h7Jc>cwa^(>JY36z;zjI2sm?BYqYPv_t~ zmn=qIG5RO>d@hsmh|u(ov%Z6BbJ1B^Isw09O45K>ieMlIqM_Y97isUI2ISEby3|ss zU|xChi;-+A9%N;64j0F6*aYv^C2y0K{`ZYX!KqDI0LXco~ zNOfLr#CqAMcK5ik&HNptBaF|zsdv5Zw0VnS-{?HTwJ~0!_!2FrPVrc*j6(F`{&;uG zhl6p;Si;#KB7RkI{8P9!UBjUE5Z*KE0Co(Xu7urkuo%{U6q>~7G zwg)*pn73JkIs&N&R|?F#LA$YG=i?r$R1k$IqsKzIs+)AW?28+vDPdI^PT)nQJrYL%5}5KqPqU_G&%>9iFx%;~)+w3n9!u;NHLX$+9NMGM_BfL9g! z;x^h70m;Gay)8MmC=M7hdpH<<0pBpo`56B%24mHKa1yMWTcBE4C9^l0@T0wF$qb>1 zAp?<%W^0Je(SU!M4>DO+mU|b( zad^dTMJTf?>oe%c&A;8E72M)br)z_VWxF-EMaudLhxJV7_MCq=>}{JH98SowRot`U zwj!>ca}H-UP;FN$w8U)paizw1E2ms8m2>C}DCidv{8D1ztQ3Xrr~bZHcr9_LkRu-T z$bln3VvjUpmr(FWvG-ANc~I!?jlhrY1%J$9jr8-xTm6s`!1|f`i74=NJ#KVKyFb4x zQ?);r)~ETczJ*uBc5%D&Ph~-mq%coYOS~!c-x1_;k^N@m3bs%-Ol-`=5_HakJZjw{>CHlE&I4lS4;RZmhNqXkq+QC|1u zJ|AZ4(2NVN@q=ktY5Xjuw0|7>`+b7$pe7X{AVa&eT;p}7!Wl1ZQT0l9{cxMo(VFDz z%rVW`<{gk+g4e#f_@IAp{xla=RCR-+O{N@ZmB6nfl;|Owdj`GO5aNV$vcsVDaIzsY zKUp*5!uR{0qU^@Um?WV;`aYvkGF2o$oDX*Le-mh8&Xrss?B(Vhy-7#=6QiiJ)TBvB zlv+;4L5?L}T^#z8DZE@o$<^zn9)s$&xqh>k7wL1X6XO zD~P-{#@0)cm>*easTi?Q7cB6vYy5rAS>Mlbmca($|3xfr26VhgO_ZFe{k&}9yb3C* zAYp7~vDYQwnE%XlQU}%N<8ztvGNY)cTftkQfRQiKpc^v< zeSwuJ-kbDmiFCB$AQ*L}46A>8EELDhPX@s2t{>x9@CmFa2_`qrXWFSa`5u~-{Ly{1 zaAX_kNKvy5z98O)>8LH!lk{t~P1!)8-`{cYT{*}EuOX~O%`W>-W)biN1k)=2QWFt_D@L{pB!d#F^?JV8wvrDl(?%sZo^SAfjAxm z`)hwB0EtS*KXTOBdzqw>wI2xVO8{J7ZI&Bp^k%rJsg@17CS91_#QO=YQ z*beJW8k`O;Fx#@HiGTVj1unRfLyL|b}9W`zq#x^y_w^?Z-DB&-Ve;Gy3 znHB0zM@%p*fDXE|OwHBQQ!fNqT>_b=;b?9AlYr1MbnBd6d}rVynQjmrvJ!evli*(1 zz*g-y#_YBuu4>24;gA~~c+8K(O%Djimq?tG_w2ye(Ft$*B7L2+Uti~bcl3U9kwYL@ zLXp&`eLTetU8;HWB9-F6yGOFFcON?Av*_UR({9=7_Kun^8ta$|sAN6ie(Ute4~7nNy7iG9EAxK-MrmW}RVlbOA2Lp;W!k(&SFfoTQRz!H zCzSb?@yEoih#Cy>!<&%uv4VB)7GHnQ`Lvdb8o@nQVQ>EE*|;IeF25QTT4`J+CyqFt z2B%x5Zx`%mW5h4K!tj?)MsF6ljNM18&5X zg2J|j$QE}f>B|5)kBGPI~0K%h&8_O%TTFW=R$-TuG!FIiEd8lKo!rscgWKbcrv%|dSF zQl4WgOKV4W7+}BT*7{y|=c6blepS~+8d`ei$ow;? zU>DwZYuEpHcNg1H!s4i#tFtCA9t8g>$p35*uwybE84XbT+F~M}1{H#KcahO6ezkqY z2FYW+$!-iIWW2kn->Fjk(Pmgj=JWDog8UUy2^K%=no`ckgUz3%2(Ef;*(3 zd0i>=Ei>x!Cg@P*(4-TCpFJDM!buH6LL7sf<9uge2#;SCwq73RA}>fUm+|4(y|c6b z&v54Rch15Ol)bM6=o47xrW%&cmMAth08a-mx`fCPWwIee-U9Wae>lI2DxBi13M|Y1`P2nT=1Y~jo7B#5Hx0gh(Iv| zQpAg=&ZN5<6XbO8X0#Pyr)(_=QN}|i?;cvDne3Nu96QP{Q#ouqn&c`S2+XTQRs3jT z{7H5fL}?2~b-|V728DvB5tS8ShkyFk3X1?jr`#uu?cFcKjA+3rpk~@cH*_oN z9g+(7DXr|)8f!~&zf+(#k39Uf?q!eJKt_*RE^ZaDRHwvK>FuRRYFS4 z!L_EpTlBmvu%cqJx-QIkn`bm=?j*5f;{P*IRQ^99iVS`8dH+I+*JP)UMI9r1srNC{ zK`8#Xl7;+hP1CwgShdNP-Lr)GtC?5}P$fVL5`*d*PsFG;v5?Z|d>A?9$TdIi2{N zy*9r^MT9{FxxQ^nS*^__b~)>5M<(X}83j2L(a6B}QKedKAdh+auvO#es}_(_v?RLH zljQZJD-HF$49QdXr4Xkwc?36%fPfZj+6O&)Es3v;@A{TBw)-7WZ`pTajS^u%YM)zV zo%O-2KCT}UNya;9e7b)!G=0du!?NEz%IP(qVZu@8Vk?#QR`E7F6c;4&&V7{y!ogc{ zOIaaQ8#+O3FmE->dSkH{{g5$+dwFH&U0F`g7n&HQNc<8d5%cKW?U=~pr}%AFSn>&w z0*K}+OVW`Cxt?-}E56&Lj~jf|@^fkvnONnW&j6upP|iyV($t$_s?Odqt|M&5+@C$N zG-vu#===^rgF0tBC9GL`#~krh@PT*OLYz|W>y0R>O&2Z9e#CEJWlUv@+ZeHF-suTLj`k`j+=ID) z2P=}PiIIx+Mcxdnpirx6+OZ<-PHn%~ya*{t@ZNg?O}KZ?OWo~?Fq7AmnD zE^?4<*i=XHrIu$KS6yVV?bYZL74oB0;>TsB6Fi2bVZkJ>M&sNw zCqG}|J~NvNs)9SXB!BKA-StE?V9S*hRP~rAKV4a8gnm!Chb|Xzi#U9P>i8+X^1>Sp zRJ2a=kx&Rq$}>PK3X_uh{4)Owzi&*sTukOfh5hj#)Pae!jX>HFwfM?%-DDLj{pame zV9y?(n%dNxofsHar;PBxXQY?UCB!wq;u#S zc3_M>CeMspePCwIeebeE-D7xG!SP57EoZ7;eyD#Cnc15i#MCKBUfZAc?b|&1{NvCR zg$z9rsP|->6|Jb(X@a>0?1ve%&VOL&zj6|JUQEOcr2V4MEeF)}3z$Z;8_qc!p>la|N79eArfX=Z z>6eJM>A`&PZ?3XYmWamLxM13|P4U5i+4BFvRfd=Mks(1i?;~6Mbb#8SsG8(O+WkJ( zWgtg|mi}o&v*UC~-p}%|rH?S7U6lB>lWmq@#G@tfJM4tAJzq;MD zoVwWoGWFBxPPkB1<77V+W`7JK7;aRGNs!ji*PENWO8j2i$*nFut4#Z|eAH1~ht+xcAf0TT0Qpe;#@Q3y6F{A{4&||@UMe|C*x^ah(0ciK zE^XBS6ZG^tAjZKC+q`y9?Kz*I6ZHAn-1EdS^|8Ny$hpT=zn2?1+x6CH`!4=?$?5|m zYm&{5=g#ze$i-vwZmZ+i?NR9F2u=a<{WOe&fhi&zsF_3^S29`)sW(!o@t?}2Jrt~K zTPTR8d@nPGb&I!|2k!l0%w^un+Sku; z3OajRd&&@6k|$BctbzN=-}vtKoqO&{DV2$W67_f2{XT=h1`)s%l2=$s)$ER-V!7SF zkl{h`zmQ=+M?hYq?drQ@sg9bRt||?JgeGirCBj0N-~WLO#X4o#F9yd*?56d@F2{4WmZu;UhI-#Kt*i(j*N6b;s@YzLDd*BIwN`yph1Z5p#zM6$0r)@6^$3% zqs~3cYOxh~k|cvN0G9YmW$*P!H9k1zXa2N5vB8o?Uh(*ToE}26iqRRH-NN79!r(Ug zeHU(CjHk*GvH4az3io`Gf|(Zm?z0p@W0FgHYYWJXMfok@hMU1qQSW~Ya2`(3a(x`5 zAz>9iE2_li_RTt#Mm429Ot>nydpI+aFwj3Utt7{)H&YLmvto(-Py3r|hgS%sz*`?% zXUgppZ~cO=w1}Y%znXYx`ZC=A77kWo6z!`3zX`uD$4sZO2?OMfR4?Vm$2(O>(011y zck2Eo^n^mGd=|m6gDm?ZV616#kpq5{Ifq(^0RgLB<`;9`lF_Cop(2Z#HP#RBDp-S1 zxUYcI5$KJSzpq3w97WTkUZc2*skJ)KC1OUk3Qg_QD%*)>4hBGnT%Y@=Nc7Tw#kTC- z3LGs1Xq*j#HVvk`-klv0wsXz#>#Pb7Rd*yb`5(BICIF#>a&tDdQ0vJ&spomYTaf43 z!a!qjqwgTybXI7V>D`cb5J<}Fcqt=)I6{t0&lhk#hT5f@h1Sc*x$WzGWY6`IiK72*G(d$}b_i4+=`-u!{4xqBYa7b4$a7CVBdhH3Y|Qe6x>$LiwZ$NPWk> zhF1*Sb*f0#!LrO*`O_kGBm?Q6hitYHyB~Pi$S8>Wx_p?p{9C|wJo57#fdQw5S?!As z?OGa*g6g>8J8~=za_orkwx}m*acY0cOD|l`G~5Fd?;uJC@XRED%{IO3?6Dxl|Mb5 zOLCC9=q1x%Y}rfdC|PYITeMyG1N=7J^u9QoNBxxC?$tMB(Rk@OdQV;~gw@DXg&$OuT&Sp$>i>>yLVuW-?Yhvc5^lq zbYXn8hrMZ(n_Dx?mm1vR+~@rW&WCrHoj)3R=PJfiSM_04u}2?gc$uF?r#l$Maqnyo zPd#LcWN8gpS>)TmAXPWCo3FOAhCjh;9Y>+v28rNGUA?hmfhbC0>zp;G${2ttopu{( z)#<@LGX7(UNIkCVH-E{Co=wvinz6>%a+|}3pLO^FWySOP)}`rjbR2!T68Z?Yw=!S8 zf6`%0RopFe&{yH)o6Pa|(`XJ3d>U0>y9!SbtV;?vNM`B!si^ieWLLD5uzN?) zftZycub<#!UYu$0vxZRpNU@aFCsm(xP(Qgl{P*k99mrKW`X)FwR%JjNfW&_pHmiDY zLmk`7JiOr68f+gxJhQOKQUAu^rzn)Bb+RVm@(I<9={6U;g6aG|cZ`Fj%3jc~T(G=u znk=u)&S`@q118y8C#}$ur{tCEx<6s^d8~yF^Nth7*$i$q_FM)^yy85rG?*XTMee{U z{Kc6Hw|rTC77PR?HD;&-8(BqmZj!?a^2^?Bx6FY!8C_YB6X*=F2U{dD!pUzfhbL0_Ao zB|$%_MG_EcV)V=(Ig`3i^t`{_ILz+}8Ur}i=}*YrI_F?0dTJCAZ6UY`*^F&D`c$Ix zzyO59m&vZ8uifdkKcW`uqf*5!SC4fZVtsq_x9ip!wp06}oSV$V(3}Y`; zpC?xwRwQeGRB24#e!qJR?aRQ5F}TRJ;xiS@sD_TVJ#+t5Fz!hd%9JNfkAFx-FH z8t(Apu7o@Ml;{6(_^IX_%A3cXzCg3JoL71vR)gp7xu6wAh0Fc(T}{jXo=t~2XJz3K zbLGxE#*Q#)dwe`O-olR`@YlMBMG*g<;Uhg70^4?OqH}HOkG9D0X<4Z>?9~n%^pt@j z0|KNnh-zZGqxYI*)(fo;jop5C=ODtOXv~AU4Py_A%PY2+z@KLjop<$C2KCwnPSy|- z&Z^&YjX&91%#1bldycLH_-XSO&bFq{s4p#=6dR$|Hg#|`%u`~K`_EbK;;2R6Lf&G1 zht&8+GOTp%4AN<5I%+@8MdXECtarcicAw&9mjJ_WtpurG_j8-ljPB$h3}V<$2|V|> z`y-BAzaxH6-Iu=zw*H$HsJRP0U=6i?pqFQjVKlyJMu+z%J8s)Ug`sfq(KWe3F7hNUEj4{AAd}qJJP+N>N=BkJP4B)_-nM~0ntTP@@lcm>bO zL+BcEjU(J4h_Sf)8ML1*q=AA=L`vGuMsFpQGYE!l#PNgBywG;2qyGvz@tTX9FuB(V z2Ke=&v1zEua4Q^4?ypBKkqp|E$GV&p@jIC<-ddrC!Y;`2nIu;tPZDb^ch5CiRILPM zau4UZhnUUekYjLwj?=K2BSb89PA_VSLmec!tXt@F>2q)=FlckPOy%A4za zVepT6RM9bk#^E6Wnny91EdVoS>7$0~;U#9^Rq-_n5V68oSk}UXU^VZVS>0N!fl@&^ zwg%Q-fE6kS_>i+Q7;~;(cWEP!yfUL(3F|BJG>yPhIIQku2pi7p(?X&kgDdL}9g)*(YTZNE~H7~w!BFY0mC4wYX%HKb22 zIYAn+bvA_P9noW;`!DZ)tI-7YJUbRe8W!c$FgY*_A1tDlH(Dv!VwQ;IVvW4(oD2mQ z)4rj{>(2q(F)2RghC1I1@dB3M8MMI$9#v9<&XsCk;d~XsT{}s7bA*D814*9AuErK3 zEGqlk(6AYYs&5aToV9krRhW`nn2{f2H5_cf4RHd*ja?hD-t!RG^A6D1v@Ya4Jpz$OtY@hYg`u+~;YQ=C(Gz zTzmhxq?6z388THm4C>FRNg?qaGEy$4y5q_y)J<<*v6(bHfAVkf4!6?a4OW6?A3!6h zDw**nS)CMfnXAL=HjyL?cqY@>2Q@;^!u4Ho{) zKk$j*HGJq3uDek?yeRu15j%SPpX7L+&oBokCDh6>nb#(rv9*KU^f89L(LDa9cO{70 zSH=C;1J>?FW3Rr{QdD!q-Cn}Zn{iFTJB-WyS(wbZ@_ZwIZ>{hRiq8}6yhR2$7}qcUN}B(sdt zYf0a={5mM_^;`yOCj1?3=Jy)LhljdXu!)LN~SOob?DE04#y`7U|_{ z&(`pz5_k3|#?-|RX(ii}X`+6uk`;zg*e@%+O6Ug+8DCD6sJvn7&q=2qL7Aph^t2`kRVm710^O z`#O{Ea>+U*%IMH@A^A&1tgJKEyL!eF76JEKT( zjl<%XF`}w^uJ;+Ypm9&+};G*s4`{sjDI3QNrCSWWZKUMu;zKqvP^nS=brRGVe3Q&;9 zUf@|b3eUzTcoUBd@s7}^#WmSV%Fe`ZoY796ISlJmHORDQtVpdVjTlbO5oNx-!o_LE zt(W_!>%+oa14>l)ZL8<5qqw_DG4i9uwYt8LBcCxnN78@jnE8(^n&fYI!W|4TLT0K| z>2D*==w^whG!GbK2*{foO;be0b&r41gi-5;y%s1;Ic9d^VaJ zXqo(R*m=L)@hz(lpMd5I|^redcRGhq9`Q5_oJ;$n^n#Q2y4lpp9krVNaZY>d8 z&rn?~Apgkvp$6Xj)djS?KEsS)WW==?mBPgplxQ?j8(jtbem!OARl|`&WdSgM|J4cK zKUze%a1Iyrknx1(AMA=3FS##wQ)?}J#S%Me#~%V39wk_!2#Ikn5A zNf@BfkuWVyEkN?(zl4ds_`QP_+d_=8;0fDU!%Xw?i)7;%SwWYkvJv}p3-dd#VD=@B zSr7r|ww*$a4ej3s(O(VGx5D zx*Hiufgy(&T1pr~T0ulXq(pK+`u;)Bx$k*D-0ydP{m*ajJv-J~dp+xU09^{vcr<_m z_XdD3XDlL@gJim~Qgq#w@~R=&-^txwJ3Pmil86sr_B-1uzvVV>vZk=m2k_);Ho`%F z2h6yYY)N__hTs%CHM}4;YA3JJ4S;>%6g-gA(YXdTD885Pwy=vB@W*5?k)+MBr@SrB z*+%nD+(SWF8MW{-5jugKN`@vGB8shyj?1qY*TjB~oO;P`@}a9btRp`SiHAe#bzsMf zuN023ar+rp=CZ}tF8D=X#%eU@jc8Z0oKq}9j0~a@${IArT58)Af`TNe-n%RIC;{Lx zi}aJN(A=Co%^O=yqpw^xcY?l6hx371CDr%VOq2Xdjf)w}%*4LhBs;tY2%a@yt!*kD zzAPMk>h!Ds8$dTTmid=GKGR~h0I9iV6#8jJ8o=uYpa@}@AxxhAh*&pp)IE|W1lje~ zNYRv1C)hVzSd!1G*le>rKUw0c-9p~_os!X+L{P*ZZpyvu*?2$a0p3aLZ{A7iKwW{b zL!<<%LCSdt9^=YI-GB>u127}74$Fu2u^!csdNMx-baJ=v)aV-`VNaR`+=!8)W(NGn-&j%j2Ir z$GiO^g;uDq+muO9{sJ=Ie%Vxz@rLXl^|;e#TPrWraIE}FPoL&JI57^?W?s(s3_t&R zKi1*PcIrf`vGel7a^HRC+`am(e}bToqeTS(!G(~wc*6R=d_?EoyZ$(jzYMmJ<* z2mLcrJ=fhXNte93saZX%4EaC@{bE)skw=h40M%0NmUux+=28ow;-X|Bs+rSdY^HRb zOzG3yE9u%C&2?CdMrv32qG{~)(xe3f6yX^EmjNPTwkJn5W43~^24S$NMV z0@$b7FUE|~?_UI*<~l`kD>F-o*h=|pCx-8ow)8D_<(b0NOH-dkM%DE~5mS|ljKbf< zNkb0Z2fy4>GF@giNUXOwF3z{hcur@oB4uVDX*z$Z0)047n)>l1q1rHL0K3!Y$^=qh z;yNL1aQ`7eJUfy?1Bb-Yl!cf^Q%D+AR*dSs+Yll}w=kh%T?~vJ^b^q2&?^XVKIoG zazBNO8tteo>-mcW75g2VRdkIxCtc?3RI0%SYmpS113$Wr%lff1!FMK!t*7r_%c=5{NIGg6VDVw|%;0@-gCp_Qbc4 zWF?a}{em-*oXSaTNwV5kAz8%%_`~9AfJRcZm2b?p(H7FsJh8L05P^o?^w&jXHj-I5 zfhf74aZ%s9t2kx}`FZglG|U(G^z^#~F09o?Abt$ns1!!7*BE`;GtV$JN3^;4rgeT3 z0P^0`ZRn;hN}X{6{h}au9g-uw#lXDZWloN7rVJOI2~o5;F#x&J@m5GUNzd$l8W10PpQ)a#(E7bi?|w>m z%ZTRuh4w>pp{asWT865f0ylTikjnSKdbL%O3v{qZ@pu4=QW<&!8fCa{hfZ_wpr&?y zTrynn$Z2IiO3#biUpo>NC1_`FKfEEfs@u32BaGD9Uun`yX7q~0=i%LjKB}Nei31=7 zqOp-4ewB>W*Rq#SunT-`AP&ISR7mmA1v#)>LFTm7d!=UZiFq@?%3^fNE?8vNLWnU%< zJNi~wWD^j20bUq0WWWofhRP6)79L%tR~r4J?9eFB1aJs81NIQ$y)!x!#|rN@8El^( zPQ^G0#`nBHNt7ntYXhfVd)O>K3> zj5@Sh&wFX&&;@R8sKR#al!LAP1S1!d8$C*r)knj*I)*u}J%y9LE*%2boYFzC(b z%(}=IA40FT@m%MTQ4#g6_ZaY$&sIII| z^&lT;y2cX&h5kINl6@Nu!b>TjLvvLh21k!2x7h8HsjplLX=Uf!B1Rvj9x`>2o~J{n zLhS6ig0pC!*th~4SH;Oww)2y^!WN@sXxCkyByZ7cefEvxrb`o6&Sj#eNoN0;s&r;` zzjVYq^g${^+)~{#Z&cufce1MI6As?y8P-9|T0W$VLVFqUH7dc_m9NkGBA1V!PpR%= zABeB1x9(4B&iMWu0t6%^K3)J_gb~*98jU^IK)h2+TfZow!GEq7cR9IHy5jUO^=r^> z3aeR!BVE>~I}qJ%1Kx2nSxdWWPRLynM8qxsaU45Onw?uRqDzOJ$d6N;6s>9L$^)TGRyZ{*UH69cPT1yMlF1q%lvdu}hiMU062hqs|E_6ve|EF4+S!&Nw4HT04Ry%lHz)V>L0;9YPw=Yd$HF&}vs#Lzt<*W+)mnzC@7Uu>y_*z_sF$I}_L7YpuE*~@ z-O%-5GxAb7oXRnd5e$lz8a;`dNC%$z4>;^(|##? z?y0b6ElzD`Xd`@V-`kgxdf*^>;k`hCY#qU^hC#)AvFiwH=xE*4{v=)ZhM5zX_+41J zagjMkS#=iP6~ZPfulG@st;pFGF}j0U3%K3H!(uxR`3ljy6R9m{%A^9uzx~G}wsj-! zw3dc?WtwNSMl#0hJS{~j7Lu;;`N&Iv1_0+dzfx2or&=<*XY6|`hEefNSJ(l4f;3|b z1Sk-GB9Oh~cUDlAtmxM&YF%40C&Z6{Cu!b5Lyao~5@^N;G+6W7GzOYNWNU+J7oxEo z=0PKwOF$Y#+%|zG613dNpW9SuClXdmml1{u=hNj-w<+Xz6nrPK=f(9rtqOhZa(xHk zX3w6;zgfQ~XidXvoX{K5Y@Z}c+Sn18<+$2lR%JdC-8B^#@kUvl_H(uVUBZV0)>94@ zV~SN%jgLQCb?{H4ypr}Km$D|rxC_H)d_wbz@P*d{xWil=)efRwB(L|b@TTJ$KJE(U z@vyfeFRFtqXn|aUKi1Tjd;pXMGPf}r_md6Wpq$Z;m@s5E8@3;?&f9V(Q0HR(LIMMb zxa@hv`trT!m7R~ntUlQ^7|~21-zd-;kgOerO?RZd^0S&ui*L9z_wiRzbSDlc&rZ#G zt3Jvx$etvOELxs#eZP{1Z@de+OSfe%DgME?^2ac2pl}dLyabxFnOhSekEadCGKzfe zdPRfMHji5;ln&|Gz3#xJ(OSMuq`(6C&Ck5I2XU{CCSODwK>L18w}#&@TMS5>Tal1eRxcPOL-jkjy>wj@P|FRU zryyAmcVhsUB7&B-i+#Pso8{8zd$Ugd79#lFi3UM)eK3M_LLWM;0V@7|g}AK#=nZr* z-~!qO9a0qaxcYrZ>(^@577NC0`OZ-vc}npm#}j$A&$h{%4n=H##m#j{k0< zH~ve$;J+Sv57{nt5pc}zU+5D8wRl4edf{!Q$n;IMiPg2viyY<{n7J~ean~xxZy)jv zt9`*gr&Os!H>vxl(qiQslt{C4@Gk%XoNU3bF+_=U%Vt7M%Xdhs{s<$8#DaPVe0E>C z#@6cmi_1@kV4r=R#blCTfXY7U(42l)Lg>3qNx z{y;UMyUReHAY?-5s&_X~(#ou!4~}}6$glr?Cwc`qLR+_~t_Jreq!C7smU~S|<+mBJ zt1$QO7#O5Rlg3%BEFx5V#%u_8UoHp0>ppB!@_S|4=9O^LKyz^6< z5_UJ)ON;8>@R2iGIvnzs=V=-qI#6B7!8YYChRHWin$urQq|vdLl0y)7R@23ekKP&2 zf4eti8kqf3ILSZSK98#+|3M~7c{i=k^a68GHJ;_#@PF8{z6Gct8n-KdtM7J9wsldF~yHQ$;!q34B> z^x-awl_9mtvN|zJy?MHO4(kHydEh+jc zlPZg$MO$zIbrJ8aqhA-qz8>MNLpEsnJFv2MC*W0c$@tU!l*>oEE4-^`C!=P#%+~~* z4~8dk=Pd`}T4~omlgzVxXTa&O4S-C1%gbM2tj-|ml;ybPPKj-c=b}pI(P#Q+<@fFk z#45H)-m&a0TsN(Eaw*6!H05i<6cOhu=^?3CqJl%9*(Rx#@73~Wops%t7#_}F?~Vrv zDqB07Nuzij;w(<@e3s{0ryC*?jFp(cO6{O4^2O-c+;Yt4Cw>sZEN`Iw+wST8%$Z@> zAndPk!U^rK*^fTwCUbp$MxY;{tu(&PbwsC;<`nz)Qd%7aH!BO45$zR^1o440CC9pllyU|(k5+>{2*7h z3*KQ!+hq-rSwJ+gJ<_$+!z5qV*~g8~nqiKB)kZ)z zS061|yd&@X2n*}Dl0J2h&x3BI@{g+!;(CXC1DyXn*`S#6VH_N91!$flAaD9&o`P8B z_S)7mWZ0ZlKBVX$vkLgYXh4N#B^0T@OM!#|8Ck2;e_W9exFrB4mpe16{|T&+rFb!h zKEWTG_sb^%*Lf~MahJ!@UZu}iF^o2X`DMF^ga}f5z~qZ?5D)If0AEmlHmZfmhXl1#Jj2eD zJR+ZA2!@Cv{q1JEH4{)A2mwo@*EXzUzZ&Z5PbwX))xy{2VcIX+Px4Anxw%dk+P4=U z{F;VUV(rdH(h&DA^^~5t{y9pOv?wtI35*7^aHsz{|CAZ68rT)nZT$97OEchQFTwdZ zTOgI8^knJAJ*QZ-Q|?n0O~j70w-6J`{Y)0SIwxxAM1tLDFTjOr4;?8`UiQ5e_@^ku zLLaRtru)*B9GltuU^{*h-fP0oGo<&``;#c4`odGEFME1VOT2${JFWR-x1)^Q!YU-3$YlQ;a^;%zxO(lRqBT-}WMOIt}&cw8`3ktGtY3Y+}Z~Px+?rEU0hB^ho4-5R#M` z?pDEmizIq1$*1W4tv1so+Y+*X6VF724eTmRdkNN`cQ%|2Q;In^@g-mk|0I10+*z1@ znTZU&Y#vR|Jo|4nOU{t_jrxq#{>-^+VaA@~Qp`P0JhdbeKka!Sn$Psk0NjTj^t(Mo zVElzohX2b#DNo-=@RQTDErm3K01u(x%=F2a%tcbb mrNhqtQ`Y4GR%a|n=YQP^DGdIV*S~Smi<**_BJ#df*#7{_j;>n( literal 0 HcmV?d00001 From 3500e14f83adb3d251cee1235198d0f05dbfd87b Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:01:19 -0700 Subject: [PATCH 11/44] Correct conflicting names --- modules/postgres-cloud-native/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index d63ca190..f4bd1bd4 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -15,7 +15,7 @@ resource "kubectl_manifest" "argo-deployment-operator" { apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: postgres-cloud-native + name: postgres-cloud-native-operator namespace: argocd spec: project: default @@ -56,7 +56,7 @@ resource "kubectl_manifest" "argo-deployment-database" { apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: postgres-cloud-native + name: postgres-cloud-native-database namespace: argocd spec: project: default From bb2ad915f1715aec4c397e238af8607719d92b11 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:10:28 -0700 Subject: [PATCH 12/44] Set secret --- modules/postgres-cloud-native/templates/cluster-values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index e66ff44c..4b08589c 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -199,7 +199,8 @@ cluster: initdb: database: airflow-pg owner: "" # Defaults to the database name - secret: "airflow-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch + secret: + name: "airflow-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch # postInitSQL: # - CREATE EXTENSION IF NOT EXISTS vector; From ec4ea7df0d266a8e6d00d636c13339d9f79c4db4 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:29:46 -0700 Subject: [PATCH 13/44] Turn off special --- modules/postgres-cloud-native/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index f4bd1bd4..4238e610 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -86,7 +86,8 @@ YAML # TODO: Secrets should be moved out to AWS secrets manager resource "random_password" "airflow-pg-password" { - length = 20 + length = 20 + special = false } # TODO: This will need to be copied over to the airflow NS as well From e9b05fbd98d082035eee9807795a968202f8cb23 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:32:26 -0700 Subject: [PATCH 14/44] Let secret automatically be created --- modules/postgres-cloud-native/templates/cluster-values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index 4b08589c..b36333f4 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -199,8 +199,7 @@ cluster: initdb: database: airflow-pg owner: "" # Defaults to the database name - secret: - name: "airflow-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch + secret: "" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch # postInitSQL: # - CREATE EXTENSION IF NOT EXISTS vector; From 6af391e2f7915be8742865c342a195dff9ae4493 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:32:50 -0700 Subject: [PATCH 15/44] Don't create secret --- modules/postgres-cloud-native/main.tf | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index 4238e610..d4f4f756 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -85,28 +85,28 @@ YAML } # TODO: Secrets should be moved out to AWS secrets manager -resource "random_password" "airflow-pg-password" { - length = 20 - special = false -} +# resource "random_password" "airflow-pg-password" { +# length = 20 +# special = false +# } -# TODO: This will need to be copied over to the airflow NS as well -resource "kubernetes_secret" "airflow-user-secret" { - metadata { - name = "airflow-user-secret" - namespace = "cnpg-database" - labels = { - "cnpg.io/reload" = "true" - } - } +# # TODO: This will need to be copied over to the airflow NS as well +# resource "kubernetes_secret" "airflow-user-secret" { +# metadata { +# name = "airflow-user-secret" +# namespace = "cnpg-database" +# labels = { +# "cnpg.io/reload" = "true" +# } +# } - type = "kubernetes.io/basic-auth" +# type = "kubernetes.io/basic-auth" - data = { - "username" = "apache-airflow" - "password" = random_password.airflow-pg-password.result - } +# data = { +# "username" = "apache-airflow" +# "password" = random_password.airflow-pg-password.result +# } - depends_on = [kubernetes_namespace.cnpg-database] -} +# depends_on = [kubernetes_namespace.cnpg-database] +# } From 8e010653397e9d46e66e06f06f2592f3066febbe Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:35:16 -0700 Subject: [PATCH 16/44] Don't specify secret --- modules/postgres-cloud-native/templates/cluster-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index b36333f4..7334deb4 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -199,7 +199,7 @@ cluster: initdb: database: airflow-pg owner: "" # Defaults to the database name - secret: "" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch + # secret: "" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch # postInitSQL: # - CREATE EXTENSION IF NOT EXISTS vector; From 4f59b7d17aad60098fba1e5892cee38015a6a3da Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:47:43 -0700 Subject: [PATCH 17/44] Separate postgres operator to cluster --- .../dpe-k8s-deployments-testing/main.tf | 16 +++- .../postgres-cloud-native-operator/README.md | 16 ++++ .../postgres-cloud-native-operator/main.tf | 44 ++++++++++ .../templates/operator-values.yaml | 0 .../variables.tf | 17 ++++ .../versions.tf | 16 ++++ modules/postgres-cloud-native/README.md | 70 ++++------------ modules/postgres-cloud-native/main.tf | 80 +------------------ .../templates/cluster-values.yaml | 2 +- modules/postgres-cloud-native/variables.tf | 10 +++ 10 files changed, 135 insertions(+), 136 deletions(-) create mode 100644 modules/postgres-cloud-native-operator/README.md create mode 100644 modules/postgres-cloud-native-operator/main.tf rename modules/{postgres-cloud-native => postgres-cloud-native-operator}/templates/operator-values.yaml (100%) create mode 100644 modules/postgres-cloud-native-operator/variables.tf create mode 100644 modules/postgres-cloud-native-operator/versions.tf diff --git a/deployments/stacks/dpe-k8s-deployments-testing/main.tf b/deployments/stacks/dpe-k8s-deployments-testing/main.tf index eb481444..7ab8c5e0 100644 --- a/deployments/stacks/dpe-k8s-deployments-testing/main.tf +++ b/deployments/stacks/dpe-k8s-deployments-testing/main.tf @@ -1,8 +1,20 @@ -module "postgres-cloud-native" { +module "postgres-cloud-native-operator" { # source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" - source = "../../../modules/postgres-cloud-native/" + source = "../../../modules/postgres-cloud-native-operator/" # version = "0.2.1" auto_deploy = true auto_prune = true git_revision = "ibcdpe-1004-airflow-ops" } + + +module "postgres-cloud-native" { + # source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" + source = "../../../modules/postgres-cloud-native/" + # version = "0.2.1" + auto_deploy = true + auto_prune = true + git_revision = "ibcdpe-1004-airflow-ops" + namespace = "airflow" + argo_deployment_name = "airflow-postgres-cloud-native" +} diff --git a/modules/postgres-cloud-native-operator/README.md b/modules/postgres-cloud-native-operator/README.md new file mode 100644 index 00000000..bfa988d3 --- /dev/null +++ b/modules/postgres-cloud-native-operator/README.md @@ -0,0 +1,16 @@ +# Purpose +The purpose of this module is to deploy the `Cloudnative PG` helm chart . +This will deploy both the operator and a database cluster. + + +Future work: + +- Since each microservice/application is meant to recieve it's own database the deployment model within this module should be changed slightly to install the operator at a cluster level, with each application having its own database. + + + +# How many databases?? + +From their documentation: + +"Our recommendation is to dedicate a single PostgreSQL cluster (intended as primary and multiple standby servers) to a single database, entirely managed by a single microservice application." \ No newline at end of file diff --git a/modules/postgres-cloud-native-operator/main.tf b/modules/postgres-cloud-native-operator/main.tf new file mode 100644 index 00000000..932f1367 --- /dev/null +++ b/modules/postgres-cloud-native-operator/main.tf @@ -0,0 +1,44 @@ +locals { + git_revision = "ibcdpe-1004-airflow-ops" +} + +resource "kubernetes_namespace" "cnpg-system" { + metadata { + name = "cnpg-system" + } +} + +resource "kubectl_manifest" "argo-deployment-operator" { + depends_on = [kubernetes_namespace.cnpg-system] + + yaml_body = <. - - -## What resources are being deployed through this module - -First we create a namespace to deploy all of the related resources to: -```terraform -resource "kubernetes_namespace" "airflow" { - metadata { - name = "airflow" - } -} -``` - -A secret key is also created, however, the implementation needs to be reviewed before -production: - -Finally, we are creating the ArgoCD Resource definition for the application: -```terraform -resource "kubectl_manifest" "argo-deployment" { - # Does not deploy the resource until the namespace is created - depends_on = [kubernetes_namespace.airflow] - - yaml_body = <. +This will deploy both the operator and a database cluster. + + +Future work: + +- Since each microservice/application is meant to recieve it's own database the deployment model within this module should be changed slightly to install the operator at a cluster level, with each application having its own database. + + + +# How many databases?? + +From their documentation: + +"Our recommendation is to dedicate a single PostgreSQL cluster (intended as primary and multiple standby servers) to a single database, entirely managed by a single microservice application." \ No newline at end of file diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index d4f4f756..3acd6b7c 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -2,61 +2,12 @@ locals { git_revision = "ibcdpe-1004-airflow-ops" } -resource "kubernetes_namespace" "cnpg-system" { - metadata { - name = "cnpg-system" - } -} - -resource "kubectl_manifest" "argo-deployment-operator" { - depends_on = [kubernetes_namespace.cnpg-system] - - yaml_body = < Date: Fri, 16 Aug 2024 15:54:13 -0700 Subject: [PATCH 18/44] Correct path --- modules/postgres-cloud-native-operator/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native-operator/main.tf b/modules/postgres-cloud-native-operator/main.tf index 932f1367..85f941c8 100644 --- a/modules/postgres-cloud-native-operator/main.tf +++ b/modules/postgres-cloud-native-operator/main.tf @@ -33,7 +33,7 @@ spec: helm: releaseName: cloudnative-pg valueFiles: - - $values/modules/postgres-cloud-native/templates/operator-values.yaml + - $values/modules/postgres-cloud-native-operator/templates/operator-values.yaml - repoURL: 'https://github.com/Sage-Bionetworks-Workflows/eks-stack.git' targetRevision: ${local.git_revision} ref: values From 553789dcaa1fc1a4cb1c3abd614be0ff19bfccc0 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:03:12 -0700 Subject: [PATCH 19/44] Set airflow secret --- modules/apache-airflow/templates/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index 6baeee04..83b4a64e 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -368,7 +368,7 @@ data: # data: # connection: base64_encoded_connection_string - metadataSecretName: ~ + metadataSecretName: cluster-pg-app # When providing secret names and using the same database for metadata and # result backend, for Airflow < 2.4.0 it is necessary to create a separate # secret for result backend but with a db+ scheme prefix. @@ -2077,7 +2077,7 @@ cleanup: # Configuration for postgresql subchart # Not recommended for production postgresql: - enabled: true + enabled: false image: tag: "11" auth: From 7adaf405b3d759263141ab34872cba691e602a8c Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:18:18 -0700 Subject: [PATCH 20/44] Set secret here to satisfy airflow --- modules/apache-airflow/main.tf | 2 +- modules/postgres-cloud-native/main.tf | 36 ++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/modules/apache-airflow/main.tf b/modules/apache-airflow/main.tf index 9546d371..32c5dbce 100644 --- a/modules/apache-airflow/main.tf +++ b/modules/apache-airflow/main.tf @@ -31,7 +31,7 @@ resource "kubernetes_secret" "airflow_webserver_secret" { # TODO: Should a long-term deployment use a managed RDS instance? # https://github.com/apache/airflow/blob/main/chart/values.yaml#L2321-L2329 -resource "kubectl_manifest" "argo-deployment" { +resource "kubectl_manifest" "airflow-deployment" { depends_on = [kubernetes_namespace.airflow] yaml_body = < Date: Fri, 16 Aug 2024 16:20:02 -0700 Subject: [PATCH 21/44] Correct label --- modules/postgres-cloud-native/main.tf | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index 31a6dfe7..e9665383 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -45,8 +45,8 @@ resource "kubernetes_secret" "connection-secret" { name = "pg-user-secret" namespace = var.namespace labels = { - "cnpg.io/reload" = "true" - "cnpg.io/cluster" = ${var.argo_deployment_name} + "cnpg.io/reload" = "true" + "cnpg.io/cluster" = var.argo_deployment_name } } @@ -54,15 +54,15 @@ resource "kubernetes_secret" "connection-secret" { data = { - "dbname" = "application-database" - "host" = "cluster-pg-rw" - "jdbc-uri" = "jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" - "password" = random_password.pg-password.result - "pgpass" = "cluster-pg-rw:5432:application-database:application-database:${random_password.pg-password.result}" - "port" = "5432" - "uri" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" - "user" = "application-database" - "username" = "application-database" + "dbname" = "application-database" + "host" = "cluster-pg-rw" + "jdbc-uri" = "jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" + "password" = random_password.pg-password.result + "pgpass" = "cluster-pg-rw:5432:application-database:application-database:${random_password.pg-password.result}" + "port" = "5432" + "uri" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" + "user" = "application-database" + "username" = "application-database" "connection" = "jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" } From 213c2dfdeb72c0db70304d2d053cd3d5fc8b2d88 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:23:21 -0700 Subject: [PATCH 22/44] Corrections --- modules/postgres-cloud-native/main.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index e9665383..febb72bb 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -3,6 +3,9 @@ locals { } resource "kubectl_manifest" "argo-deployment-database" { + depends_on = [ + kubernetes_secret.connection-secret + ] yaml_body = < Date: Fri, 16 Aug 2024 16:31:29 -0700 Subject: [PATCH 23/44] Set secret --- modules/postgres-cloud-native/templates/cluster-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index 75ec6541..ec7b627e 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -199,7 +199,7 @@ cluster: initdb: database: application-database owner: "" # Defaults to the database name - # secret: "" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch + secret: "pg-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch # postInitSQL: # - CREATE EXTENSION IF NOT EXISTS vector; From 87edd9706e358162ab759efb5baadfa9b00e3e03 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:32:00 -0700 Subject: [PATCH 24/44] Correct key --- modules/postgres-cloud-native/templates/cluster-values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index ec7b627e..712f4dd2 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -199,7 +199,8 @@ cluster: initdb: database: application-database owner: "" # Defaults to the database name - secret: "pg-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch + secret: + name: "pg-user-secret" # Name of the secret containing the initial credentials for the owner of the user database. If empty a new secret will be created from scratch # postInitSQL: # - CREATE EXTENSION IF NOT EXISTS vector; From b720650687f4a84814919c986b9de41ad8abe33e Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:33:13 -0700 Subject: [PATCH 25/44] Correct secret name --- modules/apache-airflow/templates/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index 83b4a64e..460f60a2 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -368,7 +368,7 @@ data: # data: # connection: base64_encoded_connection_string - metadataSecretName: cluster-pg-app + metadataSecretName: pg-user-secret # When providing secret names and using the same database for metadata and # result backend, for Airflow < 2.4.0 it is necessary to create a separate # secret for result backend but with a db+ scheme prefix. From 4e92ed43f7e10d0e0e8a6a1b71c4535ff456bfe4 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:40:24 -0700 Subject: [PATCH 26/44] Set with db+ --- modules/postgres-cloud-native/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index febb72bb..3bb5c3cc 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -66,6 +66,6 @@ resource "kubernetes_secret" "connection-secret" { "uri" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" "user" = "application-database" "username" = "application-database" - "connection" = "jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" + "connection" = "db+jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" } } From 7134efa59a26c972111c27b6d81bce93f150ca66 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:41:08 -0700 Subject: [PATCH 27/44] enable postgres again to see connection format --- modules/apache-airflow/templates/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index 460f60a2..bf7eb7e3 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -2077,7 +2077,7 @@ cleanup: # Configuration for postgresql subchart # Not recommended for production postgresql: - enabled: false + enabled: true image: tag: "11" auth: From 23c237131d315a78248cc1743e86d784e2ac2c31 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:43:50 -0700 Subject: [PATCH 28/44] disable postgres --- modules/apache-airflow/templates/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index bf7eb7e3..460f60a2 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -2077,7 +2077,7 @@ cleanup: # Configuration for postgresql subchart # Not recommended for production postgresql: - enabled: true + enabled: false image: tag: "11" auth: From 74696e363b0ac719a96637b460ce811dd08a7673 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:51:52 -0700 Subject: [PATCH 29/44] Correct connection name in secret --- modules/postgres-cloud-native/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index 3bb5c3cc..d3b3bdc6 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -66,6 +66,6 @@ resource "kubernetes_secret" "connection-secret" { "uri" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" "user" = "application-database" "username" = "application-database" - "connection" = "db+jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" + "connection" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" } } From 03aefc5d8a5f908191fb341eb0beb34486881694 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:59:51 -0700 Subject: [PATCH 30/44] Run with verbose --- modules/apache-airflow/templates/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index 460f60a2..1c9b16bf 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -976,7 +976,8 @@ migrateDatabaseJob: # The format below is necessary to get `helm lint` happy - |- exec \ - airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb" }} + airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb" }} \ + --verbose # Annotations on the database migration pod annotations: {} @@ -1138,6 +1139,7 @@ webserver: # memory: 128Mi # Create initial user. + # TODO: Create the initial user via a random secret defaultUser: enabled: true role: Admin From 31d98a0ff35fb6eff565df4002cc073cb886dcfe Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 17:06:19 -0700 Subject: [PATCH 31/44] Correct host url --- modules/postgres-cloud-native/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index d3b3bdc6..b25400d3 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -58,14 +58,14 @@ resource "kubernetes_secret" "connection-secret" { data = { "dbname" = "application-database" - "host" = "cluster-pg-rw" + "host" = "${var.argo_deployment_name}-cluster-rw.${var.namespace}" "jdbc-uri" = "jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" "password" = random_password.pg-password.result - "pgpass" = "cluster-pg-rw:5432:application-database:application-database:${random_password.pg-password.result}" + "pgpass" = "${var.argo_deployment_name}-cluster-rw:5432:application-database:application-database:${random_password.pg-password.result}" "port" = "5432" - "uri" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" + "uri" = "postgresql://application-database:${random_password.pg-password.result}@${var.argo_deployment_name}-cluster-rw.${var.namespace}:5432/application-database" "user" = "application-database" "username" = "application-database" - "connection" = "postgresql://application-database:${random_password.pg-password.result}@cluster-pg-rw.${var.namespace}:5432/application-database" + "connection" = "postgresql://application-database:${random_password.pg-password.result}@${var.argo_deployment_name}-cluster-rw.${var.namespace}:5432/application-database" } } From 1c9e5d3ff53fa879417205023e73fe6202b5744b Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 17:07:27 -0700 Subject: [PATCH 32/44] Correct JDBC uri --- modules/postgres-cloud-native/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/main.tf b/modules/postgres-cloud-native/main.tf index b25400d3..419cef7f 100644 --- a/modules/postgres-cloud-native/main.tf +++ b/modules/postgres-cloud-native/main.tf @@ -59,7 +59,7 @@ resource "kubernetes_secret" "connection-secret" { data = { "dbname" = "application-database" "host" = "${var.argo_deployment_name}-cluster-rw.${var.namespace}" - "jdbc-uri" = "jdbc:postgresql://cluster-pg-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" + "jdbc-uri" = "jdbc:postgresql://${var.argo_deployment_name}-cluster-rw.${var.namespace}:5432/application-database?password=${random_password.pg-password.result}&user=application-database" "password" = random_password.pg-password.result "pgpass" = "${var.argo_deployment_name}-cluster-rw:5432:application-database:application-database:${random_password.pg-password.result}" "port" = "5432" From 494aabd3d29ded726d3d1dd83fcb3d342fd6b7d0 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 17:23:44 -0700 Subject: [PATCH 33/44] Remove verbose migrate --- modules/apache-airflow/templates/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index 1c9b16bf..d4ce0f5b 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -976,8 +976,7 @@ migrateDatabaseJob: # The format below is necessary to get `helm lint` happy - |- exec \ - airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb" }} \ - --verbose + airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb" }} # Annotations on the database migration pod annotations: {} From 2d010917a58539cd6c666a8c9deec1b9c9aa981e Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Fri, 16 Aug 2024 17:25:37 -0700 Subject: [PATCH 34/44] Cleanup --- .../templates/operator-values.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/modules/postgres-cloud-native-operator/templates/operator-values.yaml b/modules/postgres-cloud-native-operator/templates/operator-values.yaml index be7d92ee..caede3e1 100644 --- a/modules/postgres-cloud-native-operator/templates/operator-values.yaml +++ b/modules/postgres-cloud-native-operator/templates/operator-values.yaml @@ -1,21 +1,3 @@ -# -# Copyright The CloudNativePG Contributors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Default values for CloudNativePG. -# This is a YAML-formatted file. -# Please declare variables to be passed to your templates. replicaCount: 1 From ecd89b4331037ed0099b15e8babd8eb7bcc5c32a Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 08:59:30 -0700 Subject: [PATCH 35/44] Enable scraping --- modules/apache-airflow/main.tf | 8 +++----- modules/apache-airflow/variables.tf | 5 +++++ modules/main.tf | 6 +++--- .../postgres-cloud-native-operator/README.md | 18 ++++++++++++++---- .../templates/operator-values.yaml | 2 +- .../templates/cluster-values.yaml | 2 +- modules/victoria-metrics/templates/values.yaml | 4 ++++ 7 files changed, 31 insertions(+), 14 deletions(-) diff --git a/modules/apache-airflow/main.tf b/modules/apache-airflow/main.tf index 32c5dbce..8e26906c 100644 --- a/modules/apache-airflow/main.tf +++ b/modules/apache-airflow/main.tf @@ -5,7 +5,7 @@ resource "kubernetes_namespace" "airflow" { metadata { - name = "airflow" + name = var.namespace } } @@ -18,7 +18,7 @@ resource "random_password" "airflow" { resource "kubernetes_secret" "airflow_webserver_secret" { metadata { name = "airflow-webserver-secret" - namespace = "airflow" + namespace = var.namespace } data = { @@ -29,8 +29,6 @@ resource "kubernetes_secret" "airflow_webserver_secret" { } -# TODO: Should a long-term deployment use a managed RDS instance? -# https://github.com/apache/airflow/blob/main/chart/values.yaml#L2321-L2329 resource "kubectl_manifest" "airflow-deployment" { depends_on = [kubernetes_namespace.airflow] @@ -60,6 +58,6 @@ spec: ref: values destination: server: 'https://kubernetes.default.svc' - namespace: airflow + namespace: ${var.namespace} YAML } diff --git a/modules/apache-airflow/variables.tf b/modules/apache-airflow/variables.tf index 07ea3c7a..1a263d40 100644 --- a/modules/apache-airflow/variables.tf +++ b/modules/apache-airflow/variables.tf @@ -15,3 +15,8 @@ variable "git_revision" { type = string default = "main" } + +variable "namespace" { + description = "The namespace to deploy into" + type = string +} \ No newline at end of file diff --git a/modules/main.tf b/modules/main.tf index cf7bf2ab..f3864cfe 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -66,7 +66,7 @@ locals { description = "Helm chart deployment for a single node Victoria Metrics instance" project_root = "modules/victoria-metrics" space_id = "root" - version_number = "0.4.7" + version_number = "0.4.8" } trivy-operator = { @@ -100,7 +100,7 @@ locals { description = "Helm chart deployment for apache airflow." project_root = "modules/apache-airflow" space_id = "root" - version_number = "0.3.1" + version_number = "0.3.2" } argo-cd = { @@ -135,7 +135,7 @@ locals { description = "Helm chart deployment for postgres-cloud-native." project_root = "modules/postgres-cloud-native" space_id = "root" - version_number = "0.2.1" + version_number = "0.2.2" } private-workerpool = { diff --git a/modules/postgres-cloud-native-operator/README.md b/modules/postgres-cloud-native-operator/README.md index bfa988d3..52b8e662 100644 --- a/modules/postgres-cloud-native-operator/README.md +++ b/modules/postgres-cloud-native-operator/README.md @@ -1,15 +1,25 @@ # Purpose The purpose of this module is to deploy the `Cloudnative PG` helm chart . -This will deploy both the operator and a database cluster. +This will deploy both the operator. +The `database` deployment is a part of another module. This allows us to add a single +operator to a cluster and deploy 1 or more databases to that cluster. -Future work: -- Since each microservice/application is meant to recieve it's own database the deployment model within this module should be changed slightly to install the operator at a cluster level, with each application having its own database. +## Future work to expand the capabilities +- Setting up backups to S3: https://cloudnative-pg.io/documentation/current/backup/ +- Moving to database only node groups: https://cloudnative-pg.io/documentation/current/architecture/#postgresql-architecture +- Assign database persistent volumes to an expandable storage class +Reading: +- https://www.cncf.io/blog/2023/09/29/recommended-architectures-for-postgresql-in-kubernetes/ + - "The next level is to separate the Kubernetes worker nodes for PostgreSQL workloads from the other workloads’, using Kubernetes’ native scheduling capabilities, such as affinity, anti-affinity, node selectors and taints. You’ll still insist on the same storage, but you can get more predictability in terms of CPU and memory usage." +- Assign database persistent volumes to an expandable storage class + - https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/ -# How many databases?? + +# How many databases should I use? From their documentation: diff --git a/modules/postgres-cloud-native-operator/templates/operator-values.yaml b/modules/postgres-cloud-native-operator/templates/operator-values.yaml index caede3e1..3298a422 100644 --- a/modules/postgres-cloud-native-operator/templates/operator-values.yaml +++ b/modules/postgres-cloud-native-operator/templates/operator-values.yaml @@ -132,7 +132,7 @@ affinity: {} monitoring: # -- Specifies whether the monitoring should be enabled. Requires Prometheus Operator CRDs. - podMonitorEnabled: false + podMonitorEnabled: true # -- Metrics relabel configurations to apply to samples before ingestion. podMonitorMetricRelabelings: [] # -- Relabel configurations to apply to samples before scraping. diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index 712f4dd2..78d40be5 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -165,7 +165,7 @@ cluster: monitoring: # -- Whether to enable monitoring - enabled: false + enabled: true podMonitor: # -- Whether to enable the PodMonitor enabled: true diff --git a/modules/victoria-metrics/templates/values.yaml b/modules/victoria-metrics/templates/values.yaml index 3c4ffc64..466578a2 100644 --- a/modules/victoria-metrics/templates/values.yaml +++ b/modules/victoria-metrics/templates/values.yaml @@ -798,6 +798,10 @@ grafana: gnetId: 17813 revision: 2 datasource: VictoriaMetrics + cloudnativepg: + gnetId: 20417 + revision: 3 + datasource: VictoriaMetrics defaultDashboardsTimezone: utc From da1ead4c69f1b0d1e4cfc164ed382e7a287fe121 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:05:54 -0700 Subject: [PATCH 36/44] Bump metrics --- deployments/stacks/dpe-k8s-deployments/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/stacks/dpe-k8s-deployments/main.tf b/deployments/stacks/dpe-k8s-deployments/main.tf index 57df99d3..ffa0146b 100644 --- a/deployments/stacks/dpe-k8s-deployments/main.tf +++ b/deployments/stacks/dpe-k8s-deployments/main.tf @@ -13,7 +13,7 @@ module "sage-aws-eks-autoscaler" { module "victoria-metrics" { depends_on = [module.argo-cd, module.sage-aws-eks-autoscaler] source = "spacelift.io/sagebionetworks/victoria-metrics/aws" - version = "0.4.7" + version = "0.4.8" auto_deploy = var.auto_deploy auto_prune = var.auto_prune git_revision = var.git_revision From 8f6ad739c31ef30c93b5a17ed353012cfcf1613c Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:13:55 -0700 Subject: [PATCH 37/44] Disable prometheusRule --- modules/postgres-cloud-native/templates/cluster-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index 78d40be5..28a8a6d9 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -171,7 +171,7 @@ cluster: enabled: true prometheusRule: # -- Whether to enable the PrometheusRule automated alerts - enabled: true + enabled: false # -- Exclude specified rules excludeRules: [] # - CNPGClusterZoneSpreadWarning From 6da8ebdb64dfbf4112ed3be483911d13244b533c Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:29:12 -0700 Subject: [PATCH 38/44] Try installing prometheus CRDs --- modules/postgres-cloud-native-operator/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/postgres-cloud-native-operator/main.tf b/modules/postgres-cloud-native-operator/main.tf index 85f941c8..8588fbd8 100644 --- a/modules/postgres-cloud-native-operator/main.tf +++ b/modules/postgres-cloud-native-operator/main.tf @@ -37,6 +37,9 @@ spec: - repoURL: 'https://github.com/Sage-Bionetworks-Workflows/eks-stack.git' targetRevision: ${local.git_revision} ref: values + - repoURL: 'https://github.com/prometheus-operator/prometheus-operator.git' + targetRevision: v0.76.0 + path: example/prometheus-operator-crd destination: server: 'https://kubernetes.default.svc' namespace: cnpg-system From 1eccf9182bfd8e3ea64511651cd94553e3408b70 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:30:02 -0700 Subject: [PATCH 39/44] Remove --- modules/postgres-cloud-native-operator/main.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/postgres-cloud-native-operator/main.tf b/modules/postgres-cloud-native-operator/main.tf index 8588fbd8..85f941c8 100644 --- a/modules/postgres-cloud-native-operator/main.tf +++ b/modules/postgres-cloud-native-operator/main.tf @@ -37,9 +37,6 @@ spec: - repoURL: 'https://github.com/Sage-Bionetworks-Workflows/eks-stack.git' targetRevision: ${local.git_revision} ref: values - - repoURL: 'https://github.com/prometheus-operator/prometheus-operator.git' - targetRevision: v0.76.0 - path: example/prometheus-operator-crd destination: server: 'https://kubernetes.default.svc' namespace: cnpg-system From de68d9a6a5b3c559758e8331d002cdb011d4c1fd Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:31:33 -0700 Subject: [PATCH 40/44] Install CRDs --- .gitignore | 3 ++- main.tf | 2 +- modules/main.tf | 2 +- modules/victoria-metrics/templates/values.yaml | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 9b15dfdc..71bcd867 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.tfstate* .terraform -terraform.tfvars \ No newline at end of file +terraform.tfvars +settings.json \ No newline at end of file diff --git a/main.tf b/main.tf index fcdc2390..b1ce7b66 100644 --- a/main.tf +++ b/main.tf @@ -10,7 +10,7 @@ # } locals { - git_branch = "main" + git_branch = "ibcdpe-1004-airflow-ops" } resource "spacelift_stack" "root_administrative_stack" { diff --git a/modules/main.tf b/modules/main.tf index f3864cfe..89587fe2 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -66,7 +66,7 @@ locals { description = "Helm chart deployment for a single node Victoria Metrics instance" project_root = "modules/victoria-metrics" space_id = "root" - version_number = "0.4.8" + version_number = "0.4.9" } trivy-operator = { diff --git a/modules/victoria-metrics/templates/values.yaml b/modules/victoria-metrics/templates/values.yaml index 466578a2..b50a54b9 100644 --- a/modules/victoria-metrics/templates/values.yaml +++ b/modules/victoria-metrics/templates/values.yaml @@ -1147,7 +1147,7 @@ crds: ## install prometheus operator crds prometheus-operator-crds: - enabled: false + enabled: true # -- Add extra objects dynamically to this chart extraObjects: [] From cd6dfc1a6ee20844fe87e5edf56af3bb09edb585 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:38:33 -0700 Subject: [PATCH 41/44] Add PromRule --- modules/postgres-cloud-native/templates/cluster-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postgres-cloud-native/templates/cluster-values.yaml b/modules/postgres-cloud-native/templates/cluster-values.yaml index 28a8a6d9..78d40be5 100644 --- a/modules/postgres-cloud-native/templates/cluster-values.yaml +++ b/modules/postgres-cloud-native/templates/cluster-values.yaml @@ -171,7 +171,7 @@ cluster: enabled: true prometheusRule: # -- Whether to enable the PrometheusRule automated alerts - enabled: false + enabled: true # -- Exclude specified rules excludeRules: [] # - CNPGClusterZoneSpreadWarning From c4e87957e994fad479ce359f19ed1ddaf3f7cc27 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:14:50 -0700 Subject: [PATCH 42/44] randomize first pass --- modules/apache-airflow/main.tf | 18 ++++++++++++++++++ modules/apache-airflow/templates/values.yaml | 16 +++++++++++++--- modules/main.tf | 2 +- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/modules/apache-airflow/main.tf b/modules/apache-airflow/main.tf index 8e26906c..ad6cb18f 100644 --- a/modules/apache-airflow/main.tf +++ b/modules/apache-airflow/main.tf @@ -28,6 +28,24 @@ resource "kubernetes_secret" "airflow_webserver_secret" { depends_on = [kubernetes_namespace.airflow] } +resource "random_password" "airflow-admin-user" { + length = 32 + special = false +} + +resource "kubernetes_secret" "airflow-admin-user-secret" { + metadata { + name = "airflow-admin-user-secret" + namespace = var.namespace + } + + data = { + "password" = random_password.airflow-admin-user.result + "username" = "admin" + } + + depends_on = [kubernetes_namespace.airflow] +} resource "kubectl_manifest" "airflow-deployment" { depends_on = [kubernetes_namespace.airflow] diff --git a/modules/apache-airflow/templates/values.yaml b/modules/apache-airflow/templates/values.yaml index d4ce0f5b..d7352cf7 100644 --- a/modules/apache-airflow/templates/values.yaml +++ b/modules/apache-airflow/templates/values.yaml @@ -880,7 +880,7 @@ createUserJob: - "-r" - "{{ .Values.webserver.defaultUser.role }}" - "-u" - - "{{ .Values.webserver.defaultUser.username }}" + - "${AIRFLOW_USERNAME}" - "-e" - "{{ .Values.webserver.defaultUser.email }}" - "-f" @@ -888,7 +888,7 @@ createUserJob: - "-l" - "{{ .Values.webserver.defaultUser.lastName }}" - "-p" - - "{{ .Values.webserver.defaultUser.password }}" + - "${AIRFLOW_PASSWORD}" # Annotations on the create user job pod annotations: {} @@ -952,7 +952,17 @@ createUserJob: useHelmHooks: false applyCustomEnv: true - env: [] + env: + - name: AIRFLOW_USERNAME + valueFrom: + secretKeyRef: + name: airflow-admin-user-secret + key: username + - name: AIRFLOW_PASSWORD + valueFrom: + secretKeyRef: + name: airflow-admin-user-secret + key: password resources: {} # limits: diff --git a/modules/main.tf b/modules/main.tf index 89587fe2..10b47b05 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -100,7 +100,7 @@ locals { description = "Helm chart deployment for apache airflow." project_root = "modules/apache-airflow" space_id = "root" - version_number = "0.3.2" + version_number = "0.3.3" } argo-cd = { From 639c94d72a26349d8a231b30830bcea3261bf40c Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:15:45 -0700 Subject: [PATCH 43/44] Tear down airflow temporary --- .../dpe-k8s-deployments-testing/main.tf | 20 +++++++++---------- .../stacks/dpe-k8s-deployments/main.tf | 16 +++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/deployments/stacks/dpe-k8s-deployments-testing/main.tf b/deployments/stacks/dpe-k8s-deployments-testing/main.tf index 7ab8c5e0..aab2823a 100644 --- a/deployments/stacks/dpe-k8s-deployments-testing/main.tf +++ b/deployments/stacks/dpe-k8s-deployments-testing/main.tf @@ -8,13 +8,13 @@ module "postgres-cloud-native-operator" { } -module "postgres-cloud-native" { - # source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" - source = "../../../modules/postgres-cloud-native/" - # version = "0.2.1" - auto_deploy = true - auto_prune = true - git_revision = "ibcdpe-1004-airflow-ops" - namespace = "airflow" - argo_deployment_name = "airflow-postgres-cloud-native" -} +# module "postgres-cloud-native" { +# # source = "spacelift.io/sagebionetworks/postgres-cloud-native/aws" +# source = "../../../modules/postgres-cloud-native/" +# # version = "0.2.1" +# auto_deploy = true +# auto_prune = true +# git_revision = "ibcdpe-1004-airflow-ops" +# namespace = "airflow" +# argo_deployment_name = "airflow-postgres-cloud-native" +# } diff --git a/deployments/stacks/dpe-k8s-deployments/main.tf b/deployments/stacks/dpe-k8s-deployments/main.tf index ffa0146b..591d81e0 100644 --- a/deployments/stacks/dpe-k8s-deployments/main.tf +++ b/deployments/stacks/dpe-k8s-deployments/main.tf @@ -28,14 +28,14 @@ module "trivy-operator" { git_revision = var.git_revision } -module "airflow" { - depends_on = [module.victoria-metrics, module.argo-cd, module.sage-aws-eks-autoscaler] - source = "spacelift.io/sagebionetworks/airflow/aws" - version = "0.3.1" - auto_deploy = var.auto_deploy - auto_prune = var.auto_prune - git_revision = var.git_revision -} +# module "airflow" { +# depends_on = [module.victoria-metrics, module.argo-cd, module.sage-aws-eks-autoscaler] +# source = "spacelift.io/sagebionetworks/airflow/aws" +# version = "0.3.1" +# auto_deploy = var.auto_deploy +# auto_prune = var.auto_prune +# git_revision = var.git_revision +# } module "argo-cd" { depends_on = [module.sage-aws-eks-autoscaler] From 6b6b2591baf2c34daf21f513159a1e34a53c024e Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:23:40 -0700 Subject: [PATCH 44/44] Add module for operator --- deployments/README.md | 1 - modules/main.tf | 29 ++++++++++++++----- .../postgres-cloud-native-operator/README.md | 2 -- 3 files changed, 22 insertions(+), 10 deletions(-) delete mode 100644 deployments/README.md diff --git a/deployments/README.md b/deployments/README.md deleted file mode 100644 index bd0bab18..00000000 --- a/deployments/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory is not actively used and will be removed in the future \ No newline at end of file diff --git a/modules/main.tf b/modules/main.tf index 10b47b05..81729dfd 100644 --- a/modules/main.tf +++ b/modules/main.tf @@ -120,7 +120,7 @@ locals { version_number = "0.3.1" } - postgres-cloud-native = { + postgres-cloud-native-database = { github_enterprise = { namespace = "Sage-Bionetworks-Workflows" id = "sage-bionetworks-workflows-gh" @@ -130,14 +130,29 @@ locals { name = "postgres-cloud-native" terraform_provider = "aws" administrative = false - # branch = var.git_branch - branch = "ibcdpe-1004-airflow-ops" - description = "Helm chart deployment for postgres-cloud-native." - project_root = "modules/postgres-cloud-native" - space_id = "root" - version_number = "0.2.2" + branch = var.git_branch + description = "Helm chart deployment for a postgres database instance using the postgres-cloud-native-operator." + project_root = "modules/postgres-cloud-native" + space_id = "root" + version_number = "0.3.0" } + postgres-cloud-native-operator = { + github_enterprise = { + namespace = "Sage-Bionetworks-Workflows" + id = "sage-bionetworks-workflows-gh" + } + repository = "eks-stack" + + name = "postgres-cloud-native" + terraform_provider = "aws" + administrative = false + branch = var.git_branch + description = "Helm chart deployment for postgres-cloud-native operator." + project_root = "modules/postgres-cloud-native-operator" + space_id = "root" + version_number = "0.3.0" + } private-workerpool = { github_enterprise = { namespace = "Sage-Bionetworks-Workflows" diff --git a/modules/postgres-cloud-native-operator/README.md b/modules/postgres-cloud-native-operator/README.md index 52b8e662..eb4d3077 100644 --- a/modules/postgres-cloud-native-operator/README.md +++ b/modules/postgres-cloud-native-operator/README.md @@ -9,14 +9,12 @@ operator to a cluster and deploy 1 or more databases to that cluster. ## Future work to expand the capabilities - Setting up backups to S3: https://cloudnative-pg.io/documentation/current/backup/ - Moving to database only node groups: https://cloudnative-pg.io/documentation/current/architecture/#postgresql-architecture -- Assign database persistent volumes to an expandable storage class Reading: - https://www.cncf.io/blog/2023/09/29/recommended-architectures-for-postgresql-in-kubernetes/ - "The next level is to separate the Kubernetes worker nodes for PostgreSQL workloads from the other workloads’, using Kubernetes’ native scheduling capabilities, such as affinity, anti-affinity, node selectors and taints. You’ll still insist on the same storage, but you can get more predictability in terms of CPU and memory usage." - Assign database persistent volumes to an expandable storage class - - https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/ # How many databases should I use?