From ed7ae611399a49600f4406691e1c1d7f65880c7e Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 14 Jun 2024 11:39:45 -0600 Subject: [PATCH 01/23] bumps tower version --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 54f3a56..0700274 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,7 +3,7 @@ profile: {{ var.profile | default() }} region: {{ var.region | default("us-east-1") }} aws_infra_templates_root_url: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra admincentral_cf_bucket: bootstrap-awss3cloudformationbucket-19qromfd235z9 -tower_version: v23.1.4 +tower_version: v23.4.3 default_stack_tags: Department: IBC Project: Infrastructure From bc823321014261eae5b05a73470d558c8c0f92f0 Mon Sep 17 00:00:00 2001 From: bwmac Date: Thu, 20 Jun 2024 12:26:39 -0600 Subject: [PATCH 02/23] updates for 23.4.3 --- .../nextflow-ecs-task-definition.yaml | 25 ++++++++-------- .../nextflow-ecs-task-definition.yaml | 24 +++++++-------- templates/nextflow-ecs-task-definition.j2 | 30 +++++++++++++++++-- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index fa8ba96..6fce159 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -6,12 +6,11 @@ dependencies: - infra-dev/nextflow-efs-file-system.yaml - infra-dev/nextflow-elasticache-cluster.yaml - parameters: - TowerSmtpHost: 'email-smtp.us-east-1.amazonaws.com' - TowerSmtpPort: '587' - TowerSmtpUser: !ssm 'smtp-username' - TowerSmtpPassword: !ssm 'smtp-password' + TowerSmtpHost: "email-smtp.us-east-1.amazonaws.com" + TowerSmtpPort: "587" + TowerSmtpUser: !ssm "smtp-username" + TowerSmtpPassword: !ssm "smtp-password" TowerContactEmail: nextflow-admins@sagebase.org TowerServerUrl: https://tower-dev.sagebionetworks.org TowerRedisUrl: !stack_output_external nextflow-elasticache-cluster::RedisEndpoint @@ -23,19 +22,19 @@ parameters: TowerDbPassword: !aws_secrets_manager nextflow-aurora-mysql-NextflowTowerDatabaseUserSecret::SecretString::password TowerGoogleClientId: !aws_secrets_manager nextflow/google_oauth_app::SecretString::client TowerGoogleSecret: !aws_secrets_manager nextflow/google_oauth_app::SecretString::secret - CronContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' - FrontendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' - BackendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + CronContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + FrontendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}" + BackendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + MigrateDBContainerImage: "cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}" EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId - EfsVolumeMountPath: '/efs' - TowerUserWorkspace: 'false' + EfsVolumeMountPath: "/efs" + TowerUserWorkspace: "false" TowerRootUsers: - thomas.yu@sagebase.org - khai.do@sagebase.org - TowerConfigFileName: 'tower.yaml' + TowerConfigFileName: "tower.yaml" -stack_tags: - {{stack_group_config.default_stack_tags}} +stack_tags: { { stack_group_config.default_stack_tags } } sceptre_user_data: environment: !file src/tower/resources/environment.yaml diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index ba4df17..76d19e5 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -7,10 +7,10 @@ dependencies: - infra-prod/nextflow-elasticache-cluster.yaml parameters: - TowerSmtpHost: 'email-smtp.us-east-1.amazonaws.com' - TowerSmtpPort: '587' - TowerSmtpUser: !ssm 'smtp-username' - TowerSmtpPassword: !ssm 'smtp-password' + TowerSmtpHost: "email-smtp.us-east-1.amazonaws.com" + TowerSmtpPort: "587" + TowerSmtpUser: !ssm "smtp-username" + TowerSmtpPassword: !ssm "smtp-password" TowerContactEmail: nextflow-admins@sagebase.org TowerServerUrl: https://tower.sagebionetworks.org TowerRedisUrl: !stack_output_external nextflow-elasticache-cluster::RedisEndpoint @@ -22,18 +22,18 @@ parameters: TowerDbPassword: !aws_secrets_manager nextflow-aurora-mysql-NextflowTowerDatabaseUserSecret::SecretString::password TowerGoogleClientId: !aws_secrets_manager nextflow/google_oauth_app::SecretString::client TowerGoogleSecret: !aws_secrets_manager nextflow/google_oauth_app::SecretString::secret - CronContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' - FrontendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' - BackendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + CronContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + FrontendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}" + BackendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + MigrateDBContainerImage: "cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}" EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId - EfsVolumeMountPath: '/efs' - TowerUserWorkspace: 'false' + EfsVolumeMountPath: "/efs" + TowerUserWorkspace: "false" TowerRootUsers: - thomas.yu@sagebase.org - TowerConfigFileName: 'tower.yaml' + TowerConfigFileName: "tower.yaml" -stack_tags: - {{stack_group_config.default_stack_tags}} +stack_tags: { { stack_group_config.default_stack_tags } } sceptre_user_data: environment: !file src/tower/resources/environment.yaml diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 0cd9c06..01667b3 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -83,6 +83,15 @@ Parameters: Type: String Description: Redis container docker image, e.g. 'redis:5.0.8' {%- endif %} + MigrateDBContainerName: + Type: String + Description: (Optional) Name of the migrate-db container + Default: migrate-db + MigrateDBContainerImage: + Type: String + Description: > + (Optional) migrate-db container docker image, + e.g. 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:v23.4.3' CronContainerName: Type: String Description: (Optional) Name of the cron container @@ -180,6 +189,21 @@ Resources: EFSVolumeConfiguration: FilesystemId: !Ref EfsFileSystemId ContainerDefinitions: + - image: !Ref RedisContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref MigrateDBContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref FrontendContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref BackendContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref CronContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} # The following container definition is a stop-gap solution for # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 @@ -229,8 +253,8 @@ Resources: awslogs-group: !Ref TowerTaskLogGroup awslogs-stream-prefix: !Ref AwslogsStreamPrefix {%- endif %} - - Name: !Sub '${CronContainerName}-MigrateDb' - Image: !Ref CronContainerImage + - Name: !Sub '${MigrateDBContainerName}' + Image: !Ref MigrateDBContainerImage Memory: 2000 Cpu: 0 Essential: false @@ -275,7 +299,7 @@ Resources: - ContainerName: !Ref RedisContainerName Condition: START {%- endif %} - - ContainerName: !Sub '${CronContainerName}-MigrateDb' + - ContainerName: !Sub '${MigrateDBContainerName}' Condition: SUCCESS WorkingDirectory: /work EntryPoint: From df40beb4cdeb91879cb7c12136e13fad8ce71c97 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:04:56 -0600 Subject: [PATCH 03/23] Update nextflow-ecs-task-definition.j2 --- templates/nextflow-ecs-task-definition.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 01667b3..6b233c3 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -100,7 +100,7 @@ Parameters: Type: String Description: > (Optional) Cron container docker image, - e.g. '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:v21.06.0' + e.g. 'cr.seqera.io/private/nf-tower-enterprise/backend:v21.06.0' FrontendContainerName: Type: String Description: (Optional) Name of the container that runs the tower ui @@ -109,7 +109,7 @@ Parameters: Type: String Description: > Frontend container docker image, - e.g. '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/frontend:v21.06.0' + e.g. 'cr.seqera.io/private/nf-tower-enterprise/frontend:v21.06.0' FrontendContainerPort: Type: Number Description: (Optional) Port to open in frontend container @@ -126,7 +126,7 @@ Parameters: Type: String Description: > Backend container docker image, - e.g. '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:v21.06.0' + e.g. 'cr.seqera.io/private/nf-tower-enterprise/backend:v21.06.0' BackendContainerPort: Type: Number Description: (Optional) Port to open in backend container @@ -194,10 +194,10 @@ Resources: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - image: !Ref MigrateDBContainerImage repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - image: !Ref FrontendContainerImage repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - image: !Ref BackendContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' From 05d9a672df8343c8b2e75d3848f07dcb38d207e6 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:17:46 -0600 Subject: [PATCH 04/23] Update nextflow-ecs-task-definition.j2 --- templates/nextflow-ecs-task-definition.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 6b233c3..8b82b99 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -253,7 +253,7 @@ Resources: awslogs-group: !Ref TowerTaskLogGroup awslogs-stream-prefix: !Ref AwslogsStreamPrefix {%- endif %} - - Name: !Sub '${MigrateDBContainerName}' + - Name: !Ref MigrateDBContainerName Image: !Ref MigrateDBContainerImage Memory: 2000 Cpu: 0 @@ -299,7 +299,7 @@ Resources: - ContainerName: !Ref RedisContainerName Condition: START {%- endif %} - - ContainerName: !Sub '${MigrateDBContainerName}' + - ContainerName: !Ref MigrateDBContainerName Condition: SUCCESS WorkingDirectory: /work EntryPoint: From 0993cc1450750dbf14ea41dfb8a57f70136b925e Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:36:46 -0600 Subject: [PATCH 05/23] Update nextflow-ecs-task-definition.yaml --- config/infra-dev/nextflow-ecs-task-definition.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index 6fce159..da75ad7 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -34,7 +34,8 @@ parameters: - khai.do@sagebase.org TowerConfigFileName: "tower.yaml" -stack_tags: { { stack_group_config.default_stack_tags } } +stack_tags: + {{stack_group_config.default_stack_tags}} sceptre_user_data: environment: !file src/tower/resources/environment.yaml From 1edc3fa7fdff7a3dfdefa79892d226b201628d23 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:37:06 -0600 Subject: [PATCH 06/23] Update nextflow-ecs-task-definition.yaml --- config/infra-prod/nextflow-ecs-task-definition.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index 76d19e5..60452ed 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -33,8 +33,9 @@ parameters: - thomas.yu@sagebase.org TowerConfigFileName: "tower.yaml" -stack_tags: { { stack_group_config.default_stack_tags } } - +stack_tags: + {{stack_group_config.default_stack_tags}} + sceptre_user_data: environment: !file src/tower/resources/environment.yaml EnableRedisDocker: false From 3a15e7a63e47884359cabb6828b0a7909c4b50b4 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:41:19 -0600 Subject: [PATCH 07/23] Update nextflow-ecs-task-definition.yaml --- config/infra-dev/nextflow-ecs-task-definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index da75ad7..5958300 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -34,7 +34,7 @@ parameters: - khai.do@sagebase.org TowerConfigFileName: "tower.yaml" -stack_tags: +stack_tags: {{stack_group_config.default_stack_tags}} sceptre_user_data: From c54a2fddc5998b83dd7c0d8eac0b05b6939539b5 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:41:45 -0600 Subject: [PATCH 08/23] Update nextflow-ecs-task-definition.yaml --- config/infra-prod/nextflow-ecs-task-definition.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index 60452ed..2978271 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -33,9 +33,9 @@ parameters: - thomas.yu@sagebase.org TowerConfigFileName: "tower.yaml" -stack_tags: +stack_tags: {{stack_group_config.default_stack_tags}} - + sceptre_user_data: environment: !file src/tower/resources/environment.yaml EnableRedisDocker: false From 1787a9e4d970b4871db31937cd6b2c0c5d06ea10 Mon Sep 17 00:00:00 2001 From: bwmac Date: Thu, 20 Jun 2024 14:53:20 -0600 Subject: [PATCH 09/23] revert formatting changes --- .../nextflow-ecs-task-definition.yaml | 22 +++++++++---------- .../nextflow-ecs-task-definition.yaml | 22 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index 5958300..fc236df 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -7,10 +7,10 @@ dependencies: - infra-dev/nextflow-elasticache-cluster.yaml parameters: - TowerSmtpHost: "email-smtp.us-east-1.amazonaws.com" - TowerSmtpPort: "587" - TowerSmtpUser: !ssm "smtp-username" - TowerSmtpPassword: !ssm "smtp-password" + TowerSmtpHost: 'email-smtp.us-east-1.amazonaws.com' + TowerSmtpPort: '587' + TowerSmtpUser: !ssm 'smtp-username' + TowerSmtpPassword: !ssm 'smtp-password' TowerContactEmail: nextflow-admins@sagebase.org TowerServerUrl: https://tower-dev.sagebionetworks.org TowerRedisUrl: !stack_output_external nextflow-elasticache-cluster::RedisEndpoint @@ -22,17 +22,17 @@ parameters: TowerDbPassword: !aws_secrets_manager nextflow-aurora-mysql-NextflowTowerDatabaseUserSecret::SecretString::password TowerGoogleClientId: !aws_secrets_manager nextflow/google_oauth_app::SecretString::client TowerGoogleSecret: !aws_secrets_manager nextflow/google_oauth_app::SecretString::secret - CronContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" - FrontendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}" - BackendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" - MigrateDBContainerImage: "cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}" + CronContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' + BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId - EfsVolumeMountPath: "/efs" - TowerUserWorkspace: "false" + EfsVolumeMountPath: '/efs' + TowerUserWorkspace: 'false' TowerRootUsers: - thomas.yu@sagebase.org - khai.do@sagebase.org - TowerConfigFileName: "tower.yaml" + TowerConfigFileName: 'tower.yaml' stack_tags: {{stack_group_config.default_stack_tags}} diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index 2978271..497feff 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -7,10 +7,10 @@ dependencies: - infra-prod/nextflow-elasticache-cluster.yaml parameters: - TowerSmtpHost: "email-smtp.us-east-1.amazonaws.com" - TowerSmtpPort: "587" - TowerSmtpUser: !ssm "smtp-username" - TowerSmtpPassword: !ssm "smtp-password" + TowerSmtpHost: 'email-smtp.us-east-1.amazonaws.com' + TowerSmtpPort: '587' + TowerSmtpUser: !ssm 'smtp-username' + TowerSmtpPassword: !ssm 'smtp-password' TowerContactEmail: nextflow-admins@sagebase.org TowerServerUrl: https://tower.sagebionetworks.org TowerRedisUrl: !stack_output_external nextflow-elasticache-cluster::RedisEndpoint @@ -22,16 +22,16 @@ parameters: TowerDbPassword: !aws_secrets_manager nextflow-aurora-mysql-NextflowTowerDatabaseUserSecret::SecretString::password TowerGoogleClientId: !aws_secrets_manager nextflow/google_oauth_app::SecretString::client TowerGoogleSecret: !aws_secrets_manager nextflow/google_oauth_app::SecretString::secret - CronContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" - FrontendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}" - BackendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" - MigrateDBContainerImage: "cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}" + CronContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' + BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId - EfsVolumeMountPath: "/efs" - TowerUserWorkspace: "false" + EfsVolumeMountPath: '/efs' + TowerUserWorkspace: 'false' TowerRootUsers: - thomas.yu@sagebase.org - TowerConfigFileName: "tower.yaml" + TowerConfigFileName: 'tower.yaml' stack_tags: {{stack_group_config.default_stack_tags}} From 917d022d7f71ceb302d4e87f319677113e41af54 Mon Sep 17 00:00:00 2001 From: bwmac Date: Thu, 20 Jun 2024 16:07:16 -0600 Subject: [PATCH 10/23] fix redis config --- config/infra-dev/nextflow-ecs-task-definition.yaml | 1 + config/infra-prod/nextflow-ecs-task-definition.yaml | 1 + templates/nextflow-ecs-task-definition.j2 | 2 ++ 3 files changed, 4 insertions(+) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index fc236df..ac404e4 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -26,6 +26,7 @@ parameters: FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' + RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId EfsVolumeMountPath: '/efs' TowerUserWorkspace: 'false' diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index 497feff..c7f5251 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -26,6 +26,7 @@ parameters: FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' + RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId EfsVolumeMountPath: '/efs' TowerUserWorkspace: 'false' diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 8b82b99..8da9e6b 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -189,9 +189,11 @@ Resources: EFSVolumeConfiguration: FilesystemId: !Ref EfsFileSystemId ContainerDefinitions: + {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} - image: !Ref RedisContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + {%- endif %} - image: !Ref MigrateDBContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' From 7e06c4c04defc76bdeaa91f74b96b43938ea2ce3 Mon Sep 17 00:00:00 2001 From: bwmac Date: Thu, 20 Jun 2024 16:09:02 -0600 Subject: [PATCH 11/23] remove redis credential config --- templates/nextflow-ecs-task-definition.j2 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 8da9e6b..21750c9 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -189,11 +189,6 @@ Resources: EFSVolumeConfiguration: FilesystemId: !Ref EfsFileSystemId ContainerDefinitions: - {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} - - image: !Ref RedisContainerImage - repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - {%- endif %} - image: !Ref MigrateDBContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' From f4256adc078d5d93276f3d6873a46e32f504583a Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 10:41:01 -0600 Subject: [PATCH 12/23] comments out RedisContainerImage --- config/infra-dev/nextflow-ecs-task-definition.yaml | 2 +- config/infra-prod/nextflow-ecs-task-definition.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index ac404e4..fcc6db0 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -26,7 +26,7 @@ parameters: FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' - RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' + # RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId EfsVolumeMountPath: '/efs' TowerUserWorkspace: 'false' diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index c7f5251..ff97f90 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -26,7 +26,7 @@ parameters: FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' - RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' + # RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId EfsVolumeMountPath: '/efs' TowerUserWorkspace: 'false' From 3a8c7090e1416e47830d318d333bb1def6219f09 Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 10:45:14 -0600 Subject: [PATCH 13/23] remove RedisContainerImage --- config/infra-dev/nextflow-ecs-task-definition.yaml | 1 - config/infra-prod/nextflow-ecs-task-definition.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index fcc6db0..fc236df 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -26,7 +26,6 @@ parameters: FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' - # RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId EfsVolumeMountPath: '/efs' TowerUserWorkspace: 'false' diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index ff97f90..497feff 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -26,7 +26,6 @@ parameters: FrontendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' BackendContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' MigrateDBContainerImage: 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}' - # RedisContainerImage: 'cr.seqera.io/public/redis:5.0.8' EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId EfsVolumeMountPath: '/efs' TowerUserWorkspace: 'false' From aa8d03de6b6efde7e15398081c1a75e6cb0a966d Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 10:56:51 -0600 Subject: [PATCH 14/23] adds name to ContainerDefinitions --- templates/nextflow-ecs-task-definition.j2 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 21750c9..6e9caee 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -189,16 +189,20 @@ Resources: EFSVolumeConfiguration: FilesystemId: !Ref EfsFileSystemId ContainerDefinitions: - - image: !Ref MigrateDBContainerImage + - Name: !Ref MigrateDBContainerName + Image: !Ref MigrateDBContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - - image: !Ref FrontendContainerImage + - Name: !Ref FrontendContainerName + Image: !Ref FrontendContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - - image: !Ref BackendContainerImage + - Name: !Ref BackendContainerName + Image: !Ref BackendContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - - image: !Ref CronContainerImage + - Name: !Ref CronContainerName + Image: !Ref CronContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} From 60c6fca6a3b04e2626f139b946d62660302c280c Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 11:14:55 -0600 Subject: [PATCH 15/23] Corrects parameter capitalization --- templates/nextflow-ecs-task-definition.j2 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 6e9caee..78f6676 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -191,20 +191,20 @@ Resources: ContainerDefinitions: - Name: !Ref MigrateDBContainerName Image: !Ref MigrateDBContainerImage - repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - Name: !Ref FrontendContainerName Image: !Ref FrontendContainerImage - repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - Name: !Ref BackendContainerName Image: !Ref BackendContainerImage - repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - Name: !Ref CronContainerName Image: !Ref CronContainerImage - repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} # The following container definition is a stop-gap solution for # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 From 38066f6a389fcbd0afb5f92b21fcbd00b7e2771b Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 11:30:07 -0600 Subject: [PATCH 16/23] remove CronContainer from config --- templates/nextflow-ecs-task-definition.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 78f6676..68939cc 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -201,10 +201,6 @@ Resources: Image: !Ref BackendContainerImage RepositoryCredentials: CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - - Name: !Ref CronContainerName - Image: !Ref CronContainerImage - RepositoryCredentials: - CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} # The following container definition is a stop-gap solution for # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 From cd71fe43209cb9f6b55967fee0caa96eddef879c Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 13:53:33 -0600 Subject: [PATCH 17/23] removes duplicate container defs and adds creds to existing blocks --- templates/nextflow-ecs-task-definition.j2 | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 68939cc..ad26c0e 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -189,18 +189,6 @@ Resources: EFSVolumeConfiguration: FilesystemId: !Ref EfsFileSystemId ContainerDefinitions: - - Name: !Ref MigrateDBContainerName - Image: !Ref MigrateDBContainerImage - RepositoryCredentials: - CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - - Name: !Ref FrontendContainerName - Image: !Ref FrontendContainerImage - RepositoryCredentials: - CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - - Name: !Ref BackendContainerName - Image: !Ref BackendContainerImage - RepositoryCredentials: - CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} # The following container definition is a stop-gap solution for # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 @@ -252,6 +240,8 @@ Resources: {%- endif %} - Name: !Ref MigrateDBContainerName Image: !Ref MigrateDBContainerImage + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' Memory: 2000 Cpu: 0 Essential: false @@ -285,6 +275,8 @@ Resources: awslogs-stream-prefix: !Ref AwslogsStreamPrefix - Name: !Ref CronContainerName Image: !Ref CronContainerImage + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' Memory: 2000 Cpu: 0 {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} @@ -326,6 +318,8 @@ Resources: awslogs-stream-prefix: !Ref AwslogsStreamPrefix - Name: !Ref FrontendContainerName Image: !Ref FrontendContainerImage + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' Memory: 2000 Cpu: 0 Essential: false @@ -348,6 +342,8 @@ Resources: Memory: 2000 Cpu: 0 Image: !Ref BackendContainerImage + RepositoryCredentials: + CredentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' PortMappings: - ContainerPort: !Ref BackendContainerPort HostPort: !Ref BackendHostPort From 0010cf1ce17b735f67e6d7eb26d06985d2d48bab Mon Sep 17 00:00:00 2001 From: bwmac Date: Fri, 21 Jun 2024 14:57:25 -0600 Subject: [PATCH 18/23] adds new iam role --- templates/nextflow-ecs-task-definition.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index ad26c0e..3cace33 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -183,6 +183,7 @@ Resources: TowerTask: Type: AWS::ECS::TaskDefinition Properties: + ExecutionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/EcsTaskExecutionRole' NetworkMode: bridge Volumes: - Name: !Ref EfsVolumeName From 843c48de6e7f9eef7bb7a71bae4ff0f4db09db6a Mon Sep 17 00:00:00 2001 From: bwmac Date: Mon, 24 Jun 2024 09:54:40 -0600 Subject: [PATCH 19/23] lower case 'e' --- templates/nextflow-ecs-task-definition.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 3cace33..779aabe 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -183,7 +183,7 @@ Resources: TowerTask: Type: AWS::ECS::TaskDefinition Properties: - ExecutionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/EcsTaskExecutionRole' + executionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/EcsTaskExecutionRole' NetworkMode: bridge Volumes: - Name: !Ref EfsVolumeName From 923ffebe8cf0ee568b4e58a838c58943c8985f15 Mon Sep 17 00:00:00 2001 From: bwmac Date: Mon, 24 Jun 2024 10:05:59 -0600 Subject: [PATCH 20/23] revert --- templates/nextflow-ecs-task-definition.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 779aabe..3cace33 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -183,7 +183,7 @@ Resources: TowerTask: Type: AWS::ECS::TaskDefinition Properties: - executionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/EcsTaskExecutionRole' + ExecutionRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/EcsTaskExecutionRole' NetworkMode: bridge Volumes: - Name: !Ref EfsVolumeName From a310c161d811dee572b4f976b0db0ee4eed025b7 Mon Sep 17 00:00:00 2001 From: bwmac Date: Tue, 25 Jun 2024 10:37:23 -0600 Subject: [PATCH 21/23] updates README --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 960ab5c..d5182e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,6 +127,13 @@ The following secrets were created in all AWS accounts (including `strides-ampad - `nextflow/ghcr_service_acct`: The GHCR service account credentials for the Wave service - `nextflow/quayio_service_acct`: The Quay.io service account credentials for the Wave service +## Deployment Testing + +After a new deployment has successfully completed, it is important to ensure things are working as expected by doing the following: + +1. Launch a simple workflow such as `nextflow-io/hello` from the UI using both `spot` and `on-demand` compute environments. +1. Run the `demo.py` [script](https://github.com/Sage-Bionetworks-Workflows/py-orca/blob/main/demo.py) from the `py-orca` repository. Make sure that your connection URI environment variable points to the correct URL and workspace. This will check that the API is working as expected and that individual workspaces are able to access their associated S3 buckets. + ## Additional Notes - The CIDR ranges of IP addresses specifies in the VPC configurations were added to the [Sage VPN](https://sagebionetworks.jira.com/wiki/spaces/IT/pages/352976898/Sage+VPN) table. From 04f96bdbe47e5e2d63b8634dae2c553dc9b6c2ff Mon Sep 17 00:00:00 2001 From: bwmac Date: Tue, 2 Jul 2024 15:41:06 -0600 Subject: [PATCH 22/23] adds EcsTaskExecutionRole to task definition template --- templates/nextflow-ecs-task-definition.j2 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 3cace33..82b49fa 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -180,6 +180,21 @@ Resources: LogGroupName: '/aws/ecs/task/nf-tower' RetentionInDays: 30 + EcsTaskExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: EcsTaskExecutionRole + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + - arn:aws:iam::aws:policy/SecretsManagerReadWrite + TowerTask: Type: AWS::ECS::TaskDefinition Properties: From 3b29390310f35c10bba756fa5037eeeb03098fb9 Mon Sep 17 00:00:00 2001 From: bwmac Date: Tue, 2 Jul 2024 16:28:58 -0600 Subject: [PATCH 23/23] changes S3ReadOnlyAccessArns to S3ReadWriteAccessArns --- config/projects-prod/robert-allaway-project.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/projects-prod/robert-allaway-project.yaml b/config/projects-prod/robert-allaway-project.yaml index 0e30828..65402ad 100644 --- a/config/projects-prod/robert-allaway-project.yaml +++ b/config/projects-prod/robert-allaway-project.yaml @@ -12,7 +12,7 @@ stack_tags: CostCenter: NO PROGRAM / 000000 # Valid values here: https://github.com/Sage-Bionetworks/aws-infra/tree/master/templates/tags parameters: - S3ReadOnlyAccessArns: + S3ReadWriteAccessArns: - "{{stack_group_config.tower_viewer_arn_prefix}}/robert.allaway@sagebase.org" - "{{stack_group_config.tower_viewer_arn_prefix}}/jineta.banerjee@sagebase.org" - "{{stack_group_config.tower_viewer_arn_prefix}}/sasha.scott@sagebase.org"