From e3e5651e0d3ac5bb79c83a6d086629dde4f5768a Mon Sep 17 00:00:00 2001 From: Khai Do Date: Tue, 20 Jun 2023 09:25:26 -0700 Subject: [PATCH 1/5] bump memory and add debug for redis check aof the redis-check-aof command is really so. try giving it more memory to make it run faster. Also add some debug statements so that we can log how long it takes to run. --- templates/nextflow-ecs-task-definition.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 6d6c9809..14d2b3db 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -178,7 +178,7 @@ Resources: # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 - Name: !Sub '${RedisContainerName}-CheckAOF' Image: !Ref RedisContainerImage - Memory: 2000 + Memory: 4000 Cpu: 0 Essential: false EntryPoint: @@ -187,7 +187,7 @@ Resources: # interactive prompt and doesn't have an option to disable it Command: - -c - - "yes | redis-check-aof --fix appendonly.aof" + - echo 'start redis-check-aof' && 'yes | redis-check-aof --fix appendonly.aof' && echo 'redis-check-aof complete' MountPoints: - ContainerPath: /data SourceVolume: !Ref EfsVolumeName From 4b322ae976f0bb3a4d66629fd34a58c1d66c3319 Mon Sep 17 00:00:00 2001 From: Khai Do Date: Tue, 20 Jun 2023 13:15:29 -0700 Subject: [PATCH 2/5] Revert "bump memory and add debug for redis check aof" This reverts commit e3e5651e0d3ac5bb79c83a6d086629dde4f5768a. --- templates/nextflow-ecs-task-definition.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 14d2b3db..6d6c9809 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -178,7 +178,7 @@ Resources: # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 - Name: !Sub '${RedisContainerName}-CheckAOF' Image: !Ref RedisContainerImage - Memory: 4000 + Memory: 2000 Cpu: 0 Essential: false EntryPoint: @@ -187,7 +187,7 @@ Resources: # interactive prompt and doesn't have an option to disable it Command: - -c - - echo 'start redis-check-aof' && 'yes | redis-check-aof --fix appendonly.aof' && echo 'redis-check-aof complete' + - "yes | redis-check-aof --fix appendonly.aof" MountPoints: - ContainerPath: /data SourceVolume: !Ref EfsVolumeName From 8f3f0084ff6801fc58fa267e4effd33bcbe9d5ae Mon Sep 17 00:00:00 2001 From: Joni Harker Date: Tue, 20 Jun 2023 12:51:51 -0700 Subject: [PATCH 3/5] Attempt to fix 'redis-check-aof' command Re-add the echo commands before and after 'redis-check-aof'. Replace the single quotes with parantheses, which will fork a sub-process for the piped commands to run in, and wrap the entire YAML string in double quotes to prevent YAML from parsing the pipe operator. --- templates/nextflow-ecs-task-definition.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 6d6c9809..baab64c3 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -185,9 +185,12 @@ Resources: - /bin/sh # The `yes` command is needed because `redis-check-aof` has an # interactive prompt and doesn't have an option to disable it + # Use parentheses around the piped commands to enforce order + # between `|` and `&&` operators, and wrap the entire string + # in double-quotes to prevent YAML parsing of the `|` operator. Command: - -c - - "yes | redis-check-aof --fix appendonly.aof" + - "echo 'start redis-check-aof' && ( yes | redis-check-aof --fix appendonly.aof ) && echo 'redis-check-aof complete'" MountPoints: - ContainerPath: /data SourceVolume: !Ref EfsVolumeName From 5c855cb6a2e58c3c8fa14430d569d179f8848792 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:00:15 -0700 Subject: [PATCH 4/5] bump memory for redis check aof (#211) the redis-check-aof command is really so. try giving it more memory to make it run faster. --- templates/nextflow-ecs-task-definition.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index baab64c3..6847aa15 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -178,7 +178,7 @@ Resources: # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 - Name: !Sub '${RedisContainerName}-CheckAOF' Image: !Ref RedisContainerImage - Memory: 2000 + Memory: 4000 Cpu: 0 Essential: false EntryPoint: From 4a32ca16ac0deb762c08d9091ded20bd51a1c92f Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Tue, 20 Jun 2023 18:48:34 -0700 Subject: [PATCH 5/5] log timestap for redis-check-aof command (#212) Add timestamp to help us figure out how long it takes for redis-check-aof scan to execute. --- templates/nextflow-ecs-task-definition.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 6847aa15..3c3f7d02 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -190,7 +190,7 @@ Resources: # in double-quotes to prevent YAML parsing of the `|` operator. Command: - -c - - "echo 'start redis-check-aof' && ( yes | redis-check-aof --fix appendonly.aof ) && echo 'redis-check-aof complete'" + - "echo 'start redis-check-aof' && (date) && ( yes | redis-check-aof --fix appendonly.aof ) && echo 'redis-check-aof complete' && (date)" MountPoints: - ContainerPath: /data SourceVolume: !Ref EfsVolumeName