From a26dba947b129c6e40aab63e1e52f083ea40161f Mon Sep 17 00:00:00 2001 From: Frank Muise Date: Sun, 30 Jun 2024 14:53:31 -0400 Subject: [PATCH 1/3] Fix issues with Azure blobs --- README.md | 26 ++++++++++++++++---------- install/assets/defaults/10-db-backup | 1 + install/assets/functions/10-db-backup | 11 +++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e14f7d1..1151f47 100644 --- a/README.md +++ b/README.md @@ -356,11 +356,14 @@ If `DEFAULT_BACKUP_LOCATION` = `S3` then the following options are used: If `DEFAULT_BACKUP_LOCATION` = `blobxfer` then the following options are used:. -| Parameter | Description | Default | `_FILE` | -| -------------------------------------- | ------------------------------------------- | ------------------- | ------- | -| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | -| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | -| `DEFAULT_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | +| Parameter | Description | Default | `_FILE` | +| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- | +| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | +| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | +| `DEFAULT_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | +| `DEFAULT_BLOBXFER_REMOTE_MODE` | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto` | x | + +- When `DEFAULT_BLOBXFER_REMOTE_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. > This service uploads files from backup targed directory `DEFAULT_FILESYSTEM_PATH`. > If the a cleanup configuration in `DEFAULT_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. @@ -636,11 +639,14 @@ If `DB01_BACKUP_LOCATION` = `S3` then the following options are used: If `DB01_BACKUP_LOCATION` = `blobxfer` then the following options are used:. -| Parameter | Description | Default | `_FILE` | -| ----------------------------------- | ------------------------------------------- | ------------------- | ------- | -| `DB01_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | -| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | -| `DB01_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | +| Parameter | Description | Default | `_FILE` | +| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- | +| `DB01_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | +| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | +| `DB01_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | +| `DB01_BLOBXFER_REMOTE_MODE` | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto` | x | + +- When `DEFAULT_BLOBXFER_REMOTE_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. > This service uploads files from backup directory `DB01_BACKUP_FILESYSTEM_PATH`. > If the a cleanup configuration in `DB01_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 0645a68..3ad56f6 100644 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -8,6 +8,7 @@ DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0} DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440} DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"} DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"} +DEFAULT_BLOBXFER_MODE=${DEFAULT_BLOBXFER_MODE:-"auto"} DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"} DEFAULT_COMPRESSION=${DEFAULT_COMPRESSION:-"ZSTD"} DEFAULT_COMPRESSION_LEVEL=${DEFAULT_COMPRESSION_LEVEL:-"3"} diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 5929502..c485552 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -66,6 +66,7 @@ bootstrap_variables() { DEFAULT_BLOBXFER_STORAGE_ACCOUNT \ DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY \ DEFAULT_BLOBXFER_REMOTE_PATH \ + DEFAULT_BLOBXFER_MODE \ DB"${backup_instance_number}"_AUTH \ DB"${backup_instance_number}"_TYPE \ DB"${backup_instance_number}"_HOST \ @@ -93,6 +94,7 @@ bootstrap_variables() { DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT \ DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT_KEY \ DB"${backup_instance_number}"_BLOBXFER_REMOTE_PATH \ + DB"${backup_instance_number}"_BLOBXFER_MODE \ BLOBXFER_STORAGE_ACCOUNT \ BLOBXFER_STORAGE_ACCOUNT_KEY \ DB_HOST \ @@ -199,6 +201,7 @@ bootstrap_variables() { transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_PATH backup_job_blobxfer_remote_path transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT backup_job_blobxfer_storage_account transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT_KEY backup_job_blobxfer_storage_account_key + transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_MODE backup_job_blobxfer_mode transform_backup_instance_variable "${backup_instance_number}" CHECKSUM backup_job_checksum transform_backup_instance_variable "${backup_instance_number}" CLEANUP_TIME backup_job_cleanup_time transform_backup_instance_variable "${backup_instance_number}" COMPRESSION backup_job_compression @@ -1072,7 +1075,7 @@ cleanup_old_data() { write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions" else write_log info "Syncing changes via blobxfer" - silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only + silent run_as_user blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only fi ;; "file" | "filesystem" ) @@ -1635,8 +1638,8 @@ EOF if [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions" else - write_log info "Synchronize local storage from S3 Bucket with blobxfer" - ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete + write_log info "Synchronize local storage from blob with blobxfer" + ${play_fair} blobxfer download --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete write_log info "Moving backup to external storage with blobxfer" mkdir -p "${backup_job_filesystem_path}" @@ -1644,7 +1647,7 @@ EOF run_as_user mv "${temporary_directory}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}" - silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} + silent run_as_user ${play_fair} blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} move_exit_code=$? if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${temporary_directory}"/"${backup_job_filename}"."${checksum_extension}" ; fi From 4786ea9c7f4326a3b5f51a47620b2f140fe1309e Mon Sep 17 00:00:00 2001 From: Frank Muise Date: Sun, 30 Jun 2024 14:56:50 -0400 Subject: [PATCH 2/3] Update log entry for blob sync --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index c485552..467fead 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -1638,7 +1638,7 @@ EOF if [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions" else - write_log info "Synchronize local storage from blob with blobxfer" + write_log info "Synchronize local storage from blob container with blobxfer" ${play_fair} blobxfer download --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete write_log info "Moving backup to external storage with blobxfer" From 0d23c2645c6dfcbb69f82f3cdb304286ce3befc7 Mon Sep 17 00:00:00 2001 From: Frank Muise Date: Sun, 30 Jun 2024 16:28:16 -0400 Subject: [PATCH 3/3] Add --no-overwrite to blobxfer download --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 467fead..a536937 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -1639,7 +1639,7 @@ EOF write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions" else write_log info "Synchronize local storage from blob container with blobxfer" - ${play_fair} blobxfer download --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete + ${play_fair} blobxfer download --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete write_log info "Moving backup to external storage with blobxfer" mkdir -p "${backup_job_filesystem_path}"