Skip to content

Commit

Permalink
Feat/backup script (#2604)
Browse files Browse the repository at this point in the history 
* install mount-s3

* install mount-s3

* install mount-s3

* install mount-s3

* Update Dockerfile

* Update Dockerfile

* Update Dockerfile

* Update Dockerfile

* Update Dockerfile

* Create psql-db-backup-encrypt-job.yaml

* Update psql-db-backup-encrypt-job.yaml

* Update dbbackup.sh

* Update Dockerfile

* Update dbbackup.sh

* Update dbbackup.sh

* Update dbbackup.sh

* Update psql-db-backup-encrypt-job.yaml

* Update dbbackup.sh

* Update dbbackup.sh

* Update dbbackup.sh

* Update dbbackup.sh

* Update dbbackup.sh

* Update dbbackup.sh

updated cron time

* add kube-setup-s3-csi-driver

* update dbbackup.sh

* Update kube-setup-s3-csi-driver fix tmp file name

fix tmp file name

* Update dbbackup.sh

added service account creation

* Update kube-setup-s3-csi-driver.sh

Add all oidc_url's to the trust plicy

* Update kube-setup-s3-csi-driver.sh

separate policies, roles per cluster
  • Loading branch information
@ajoaugustine
ajoaugustine authored Aug 14, 2024
1 parent 2cbd1ca commit d7ffd04
Show file tree
Hide file tree
Showing 3 changed files with 588 additions and 46 deletions.
208 changes: 162 additions & 46 deletions gen3/bin/dbbackup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
# or copy to Aurora.
#
# Usage:
# gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora <source_namespace>]
# gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora|encrypt|setup-cron <source_namespace>]
#
# dump - Initiates a database dump, creating the essential AWS resources if they are absent.
# The dump operation is intended to be executed from the namespace/commons that requires
Expand All @@ -21,19 +21,12 @@
# va-dump - Runs a va-testing DB dump.
# create-sa - Creates the necessary service account and roles for DB copy.
# migrate-to-aurora - Triggers a service account creation and a job to migrate a Gen3 commons to an AWS RDS Aurora instance.
# copy-to-aurora - Triggers a service account creation and a job to copy the databases Indexd, Sheepdog & Metadata to new databases within an RDS Aurora cluster.
#
# copy-to-aurora - Triggers a service account creation and a job to copy the databases Indexd, Sheepdog & Metadata to new databases within an RDS Aurora cluster. The source_namespace must be provided. The job should be run at the destination, not at the source.
# encrypt - Perform encrypted backup.
# setup-cron - Set up a cronjob for encrypted backup.
#
####################################################################################################

# Exit on error
#set -e

# Print commands before executing
#set -x

#trap 'echo "Error at Line $LINENO"' ERR

source "${GEN3_HOME}/gen3/lib/utils.sh"
gen3_load "gen3/lib/kube-setup-init"

Expand All @@ -42,20 +35,36 @@ account_id=$(aws sts get-caller-identity --query "Account" --output text)
vpc_name="$(gen3 api environment)"
namespace="$(gen3 db namespace)"
sa_name="dbbackup-sa"
bucket_name="gen3-db-backups-${account_id}"
bucket_name_encrypted="gen3-db-backups-encrypted-${account_id}"
kms_key_alias="alias/gen3-db-backups-kms-key"

cluster_arn=$(kubectl config current-context)
eks_cluster=$(echo "$cluster_arn" | awk -F'/' '{print $2}')

gen3_log_info "policy_name: $policy_name"
gen3_log_info "account_id: $account_id"
gen3_log_info "vpc_name: $vpc_name"
gen3_log_info "namespace: $namespace"
gen3_log_info "sa_name: $sa_name"
gen3_log_info "bucket_name: $bucket_name"
gen3_log_info "bucket_name_encrypted: $bucket_name_encrypted"
gen3_log_info "kms_key_alias: $kms_key_alias"
gen3_log_info "eks_cluster: $eks_cluster"

# Create or get the KMS key
create_or_get_kms_key() {
kms_key_id=$(aws kms list-aliases --query "Aliases[?AliasName=='$kms_key_alias'].TargetKeyId" --output text)
if [ -z "$kms_key_id" ]; then
gen3_log_info "Creating new KMS key with alias $kms_key_alias"
kms_key_id=$(aws kms create-key --query "KeyMetadata.KeyId" --output text)
aws kms create-alias --alias-name $kms_key_alias --target-key-id $kms_key_id
else
gen3_log_info "KMS key with alias $kms_key_alias already exists"
fi
kms_key_arn=$(aws kms describe-key --key-id $kms_key_id --query "KeyMetadata.Arn" --output text)
}

# Create an S3 access policy if it doesn't exist
create_policy() {
# Check if policy exists
if ! aws iam list-policies --query "Policies[?PolicyName == '$policy_name'] | [0].Arn" --output text | grep -q "arn:aws:iam"; then
# Create the S3 access policy - policy document
access_policy=$(cat <<-EOM
{
"Version": "2012-10-17",
Expand All @@ -70,15 +79,14 @@ create_policy() {
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::gen3-db-backups-*"
"arn:aws:s3:::gen3-db-backups-*",
"arn:aws:s3:::gen3-db-backups-encrypted-*"
]
}
]
}
EOM
)

# Create the S3 access policy from the policy document
policy_arn=$(aws iam create-policy --policy-name "$policy_name" --policy-document "$access_policy" --query "Policy.Arn" --output text)
gen3_log_info "policy_arn: $policy_arn"
else
Expand All @@ -90,16 +98,10 @@ EOM

# Create or update the Service Account and its corresponding IAM Role
create_service_account_and_role() {
cluster_arn=$(kubectl config current-context)
eks_cluster=$(echo "$cluster_arn" | awk -F'/' '{print $2}')
oidc_url=$(aws eks describe-cluster --name $eks_cluster --query 'cluster.identity.oidc.issuer' --output text | sed -e 's/^https:\/\///')
role_name="${vpc_name}-${namespace}-${sa_name}-role"
role_arn="arn:aws:iam::${account_id}:role/${role_name}"
local trust_policy=$(mktemp -p "$XDG_RUNTIME_DIR" "tmp_policy.XXXXXX")
gen3_log_info "trust_policy: $trust_policy"
gen3_log_info "eks_cluster: $eks_cluster"
gen3_log_info "oidc_url: $oidc_url"
gen3_log_info "role_name: $role_name"

cat > ${trust_policy} <<EOF
{
Expand All @@ -123,16 +125,9 @@ create_service_account_and_role() {
}
EOF

echo ${trust_policy}
gen3_log_info "Exiting create_assume_role_policy"

# Create or Update IAM Role
gen3_log_info " Create or Update IAM Role"
if aws iam get-role --role-name $role_name 2>&1; then
gen3_log_info "Updating existing role: $role_name"
aws iam update-assume-role-policy --role-name $role_name --policy-document "file://$trust_policy"
else
gen3_log_info "Creating new role: $role_name"
aws iam create-role --role-name $role_name --assume-role-policy-document "file://$trust_policy"
fi

Expand All @@ -143,20 +138,26 @@ EOF
if ! kubectl get serviceaccount -n $namespace $sa_name 2>&1; then
kubectl create serviceaccount -n $namespace $sa_name
fi
# Annotate the KSA with the IAM role ARN
gen3_log_info "Annotating Service Account with IAM role ARN"
# Annotate the KSA with the IAM role ARN
kubectl annotate serviceaccount -n ${namespace} ${sa_name} eks.amazonaws.com/role-arn=${role_arn} --overwrite

}

# Create an S3 bucket if it doesn't exist
# Create an S3 bucket with SSE-KMS if it doesn't exist
create_s3_bucket() {
local bucket_name=$1
local kms_key_arn=$2
# Check if bucket already exists
if aws s3 ls "s3://$bucket_name" 2>&1 | grep -q 'NoSuchBucket'; then
gen3_log_info "Bucket does not exist, creating..."
aws s3 mb "s3://$bucket_name"
else
gen3_log_info "Bucket $bucket_name already exists, skipping bucket creation."
# Enable SSE-KMS encryption on the bucket
aws s3api put-bucket-encryption --bucket $bucket_name --server-side-encryption-configuration '{
"Rules": [{
"ApplyServerSideEncryptionByDefault": {
"SSEAlgorithm": "aws:kms",
"KMSMasterKeyID": "'"$kms_key_arn"'"
}
}]
}'
fi
}

Expand All @@ -181,7 +182,7 @@ apiVersion: v1
kind: ServiceAccount
metadata:
name: psql-db-copy-sa
namespace: ${namespace}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand All @@ -191,7 +192,6 @@ rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
Expand All @@ -208,6 +208,90 @@ subjects:
EOF
}

# Function to create the persistent volume and persistent volume claim
create_pv_pvc() {
if ! kubectl get pv s3-pv-db-backups 2>&1; then
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolume
metadata:
name: s3-pv-db-backups
spec:
capacity:
storage: 120Gi
accessModes:
- ReadWriteMany
mountOptions:
- allow-delete
- allow-other
- uid=1000
- gid=1000
- region us-east-1
- sse aws:kms
- sse-kms-key-id ${kms_key_arn}
csi:
driver: s3.csi.aws.com
volumeHandle: s3-csi-db-backups-volume
volumeAttributes:
bucketName: ${bucket_name_encrypted}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: s3-pvc-db-backups
spec:
accessModes:
- ReadWriteMany
storageClassName: ""
resources:
requests:
storage: 120Gi
volumeName: s3-pv-db-backups
EOF
fi
}

# Create the service account, cluster role, and cluster role binding for the backup encryption job
create_backup_encryption_sa() {
if ! kubectl get serviceaccount -n ${namespace} dbencrypt-sa 2>&1; then
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: dbencrypt-sa
namespace: ${namespace}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: dbencrypt-role
rules:
- apiGroups: [""]
resources: ["secrets", "pods", "pods/exec", "services", "endpoints", "persistentvolumeclaims", "persistentvolumes", "configmaps"]
verbs: ["get", "watch", "list", "create", "delete", "patch", "update"]
- apiGroups: ["batch"]
resources: ["jobs", "cronjobs"]
verbs: ["get", "watch", "list", "create", "delete", "patch", "update"]
- apiGroups: ["apps"]
resources: ["deployments", "statefulsets", "daemonsets"]
verbs: ["get", "watch", "list", "create", "delete", "patch", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: dbencrypt-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: dbencrypt-role
subjects:
- kind: ServiceAccount
name: dbencrypt-sa
namespace: ${namespace}
EOF
fi
}

# Function to run the Aurora migration job
migrate_to_aurora() {
create_db_copy_service_account
Expand All @@ -222,28 +306,50 @@ copy_to_aurora() {
gen3 job run psql-db-copy-aurora SOURCE_NAMESPACE "$1"
}

# main function to determine whether dump, restore, or create service account
# Function to perform encrypted backup
encrypt_backup() {
gen3 job run psql-db-backup-encrypt
}

# Function to set up cronjob for encrypted backup
setup_cronjob() {
gen3 job cron psql-db-backup-encrypt "15 1 * * *"
}

# Check prerequisites for encrypted backup and cronjob
check_prerequisites() {
create_or_get_kms_key
create_s3_bucket $bucket_name_encrypted $kms_key_arn
gen3 kube-setup-s3-csi-driver $bucket_name_encrypted
create_pv_pvc
create_backup_encryption_sa
}

# main function to determine whether dump, restore, create service account, encrypt backup, or setup cronjob
main() {
case "$1" in
dump)
gen3_log_info "Triggering database dump..."
create_policy
create_service_account_and_role
create_s3_bucket
create_or_get_kms_key
create_s3_bucket $bucket_name_encrypted $kms_key_arn
db_dump
;;
restore)
gen3_log_info "Triggering database restore..."
create_policy
create_service_account_and_role
create_s3_bucket
create_or_get_kms_key
create_s3_bucket $bucket_name_encrypted $kms_key_arn
db_restore
;;
va-dump)
gen3_log_info "Running a va-testing DB dump..."
create_policy
create_service_account_and_role
create_s3_bucket
create_or_get_kms_key
create_s3_bucket $bucket_name_encrypted $kms_key_arn
va_testing_db_dump
;;
create-sa)
Expand All @@ -262,8 +368,18 @@ main() {
gen3_log_info "Copying databases within Aurora..."
copy_to_aurora "$2"
;;
encrypt)
gen3_log_info "Performing encrypted backup..."
check_prerequisites
encrypt_backup
;;
setup-cron)
gen3_log_info "Setting up cronjob for encrypted backup..."
check_prerequisites
setup_cronjob
;;
*)
echo "Invalid command. Usage: gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora <source_namespace>]"
echo "Invalid command. Usage: gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora|encrypt|setup-cron <source_namespace>]"
return 1
;;
esac
Expand Down
Loading

0 comments on commit d7ffd04

Please sign in to comment.