From ff13eb3923cb85ffc26815bc0563ef4e4deba393 Mon Sep 17 00:00:00 2001 From: Tim O'Guin Date: Mon, 4 Mar 2024 16:01:43 -0600 Subject: [PATCH 1/3] feat: Add DLQ for Ocean Metadata queue Also tweak FifoThroughputLimit and MessageRetentionPeriod. --- infra/function/template.yml | 58 +++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/infra/function/template.yml b/infra/function/template.yml index c9bbd51..16ef9f6 100644 --- a/infra/function/template.yml +++ b/infra/function/template.yml @@ -1,3 +1,4 @@ +--- AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: Core Ocean Iceberg Metadata Function @@ -32,6 +33,25 @@ Resources: ContentBasedDeduplication: true DeduplicationScope: messageGroup FifoQueue: true + FifoThroughputLimit: perMessageGroupId + MessageRetentionPeriod: 604800 + RedrivePolicy: + deadLetterTargetArn: !GetAtt DeadLetterQueue.Arn + maxReceiveCount: 5 + + DeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: !Sub "${QueueName}-DLQ.fifo" + ContentBasedDeduplication: true + DeduplicationScope: messageGroup + FifoQueue: true + FifoThroughputLimit: perMessageGroupId + MessageRetentionPeriod: 604800 + RedriveAllowPolicy: + redrivePermission: byQueue + sourceQueueArns: + - !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:${QueueName}.fifo" Function: Type: AWS::Serverless::Function @@ -46,6 +66,10 @@ Resources: Tracing: Active SnapStart: ApplyOn: PublishedVersions + LoggingConfig: + LogFormat: JSON + ApplicationLogLevel: TRACE + SystemLogLevel: DEBUG Environment: Variables: AWS_LAMBDA_EXEC_WRAPPER: "/opt/otel-sqs-handler" @@ -86,10 +110,40 @@ Resources: Layers: - !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:layer:${UtilsName}:${UtilsVersion}" - !Sub "arn:aws:lambda:${AWS::Region}:901920570463:layer:aws-otel-java-wrapper-amd64-ver-1-30-0:1" + Outputs: Function: - Description: Lambda Arn + Description: ARN of the Lambda function + Value: !GetAtt Function.Arn + + FunctionArn: + Description: ARN of the Lambda function Value: !GetAtt Function.Arn + Queue: - Description: SQS Arn + Description: ARN of the SQS queue for metadata events + Value: !GetAtt Queue.Arn + + QueueArn: + Description: ARN of the SQS queue for metadata events Value: !GetAtt Queue.Arn + + QueueName: + Description: Name of the SQS queue for metadata events + Value: !GetAtt Queue.QueueName + + QueueUrl: + Description: URL of the SQS queue for metadata events + Value: !GetAtt Queue.QueueUrl + + DeadLetterQueueArn: + Description: ARN of the DLQ for metadata events + Value: !GetAtt DeadLetterQueue.Arn + + DeadLetterQueueName: + Description: Name of the DLQ for metadata events + Value: !GetAtt DeadLetterQueue.QueueName + + DeadLetterQueueUrl: + Description: URL of the DQL for metadata events + Value: !GetAtt DeadLetterQueue.QueueUrl From 1241bbeb34993af76fc0a715082b198d71d93ffe Mon Sep 17 00:00:00 2001 From: GH Action Date: Mon, 4 Mar 2024 22:03:22 +0000 Subject: [PATCH 2/3] version bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 497fc8d..fc02bc5 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ 4.0.0 com.mytiki core-iceberg-metadata - 0.1.11 + 0.2.0 jar Iceberg Metadata From 45fd11a7bbdf905fd038969c6fb3c931d929c23a Mon Sep 17 00:00:00 2001 From: Tim O'Guin Date: Mon, 4 Mar 2024 16:09:09 -0600 Subject: [PATCH 3/3] Trigger release workflow on new PR commits --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5aee3bd..67f72b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,7 @@ on: - main types: - closed + - synchronize concurrency: group: ${{ github.ref }}-${{ github.workflow }}