diff --git a/.github/workflows/container_builds.yml b/.github/workflows/container_builds.yml index 71787834..422f174b 100644 --- a/.github/workflows/container_builds.yml +++ b/.github/workflows/container_builds.yml @@ -108,6 +108,7 @@ jobs: echo BUILD_DIR=$BUILD_DIR >> $GITHUB_ENV cp ./pipeline/container_images/build_images/common/build.py ./$BUILD_DIR + cp ./pipeline/container_images/build_images/common/build_optimized.py ./$BUILD_DIR - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/README.MD b/README.MD index 7b84cab3..de69bb3d 100644 --- a/README.MD +++ b/README.MD @@ -2,7 +2,7 @@ 🐍 A collection of Python Packages as AWS Lambda(λ) Layers 🐍 -[![Python 3.8](https://img.shields.io/badge/python-3.8-green.svg)](https://www.python.org/downloads/release/python-380/) [![Python 3.8](https://img.shields.io/badge/python-3.9-green.svg)](https://www.python.org/downloads/release/python-390/) [![Python 3.10](https://img.shields.io/badge/python-3.10-green.svg)](https://www.python.org/downloads/release/python-3100/)[![Python 3.11](https://img.shields.io/badge/python-3.11-green.svg)](https://www.python.org/downloads/release/python-3110/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Python 3.10](https://img.shields.io/badge/python-3.10-green?style=for-the-badge)](https://www.python.org/downloads/release/python-3100/) [![Python 3.11](https://img.shields.io/badge/python-3.11-green?style=for-the-badge)](https://www.python.org/downloads/release/python-3110/) [![Python 3.12](https://img.shields.io/badge/python-3.12-green?style=for-the-badge)](https://www.python.org/downloads/release/python-3120/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000?style=for-the-badge)](https://github.com/psf/black) - [List of ARNs](#list-of-arns) - [Python Packages](#python-packages) @@ -12,6 +12,7 @@ - [Option 3: Using Serverless Framework](#option-3-using-serverless-framework) - [Option 4: Using AWS Serverless Application Model (SAM)](#option-4-using-aws-serverless-application-model-sam) - [Option 5: Using Terraform with the Klayer provider](#option-5-using-terraform-with-the-klayer-provider) + - [Option 6: Using CDK with cdk-klayers](#option-6-using-cdk-with-cdk-klayers) - [Status of layers](#status-of-layers) - [Layer expiry](#layer-expiry) - [Architecture Diagram](#architecture-diagram) @@ -25,25 +26,25 @@ List of the latest layer version arns are available by region: -* [Python 3.8](deployments/python3.8) -* [Python 3.9](deployments/python3.9) * [Python 3.10](deployments/python3.10) * [Python 3.11](deployments/python3.11) +* [Python 3.12](deployments/python3.12) * [Python 3.10-arm64](deployments/python3.10-arm64) * [Python 3.11-arm64](deployments/python3.11-arm64) +* [Python 3.12-arm64](deployments/python3.12-arm64) -*Note: We have deprecated layers for python3.6 and python 3.7 and will deprecate 3.8 shortly. Please use python 3.10 going forward.* +*Note: We have deprecated layers for python3.6, python3.7, python 3.8 and python3.9, Please use the latest version of python (currently python3.12) going forward* ## Python Packages For the full list of Python packages, refer to the following, feel free to make a `pull requests` modifying the files below to requests for a specific package. -* [Python 3.8](pipeline/config/packages_p38.csv) -* [Python 3.9](pipeline/config/packages_p39.csv) * [Python 3.10](pipeline/config/packages_p310.csv) * [Python 3.11](pipeline/config/packages_p311.csv) +* [Python 3.12](pipeline/config/packages_p312.csv) * [Python 3.10-arm64](pipeline/config/packages_p310-arm64.csv) * [Python 3.11-arm64](pipeline/config/packages_p311-arm64.csv) +* [Python 3.12-arm64](pipeline/config/packages_p312-arm64.csv) ## Using the Layers @@ -131,6 +132,42 @@ resource "aws_lambda_function" "test_lambda" { } ``` +### Option 6: Using CDK with cdk-klayers + +Using CDK, you can use the [cdk-klayers](https://github.com/keithrozario/cdk-klayers) package to help you pull in the latest layers for your Stack or App. + +```python + +from cdk_klayers import Klayers + +class MockStack(Stack): + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + runtime = aws_lambda.Runtime.PYTHON_3_12 + + # Initialize Klayers Class + klayers = Klayers( + self, + python_version = runtime + region = "ap-southeast-1" + ) + + # get the latest layer version for the requests package + requests_layer = klayers.layer_version(self, "requests") + + lambda_function = aws_lambda.Function( + self, 'HelloHandler', + runtime=runtime, + layers=[requests_layer], + code=aws_lambda.Code.from_asset('lambda'), + handler='hello.handler' + ) + +``` + + ## Status of layers Layers are built with the latest package version at 2am UTC on the first day of the Month. diff --git a/deployments/python3.12-arm64/README.MD b/deployments/python3.12-arm64/README.MD new file mode 100644 index 00000000..56e01ff8 --- /dev/null +++ b/deployments/python3.12-arm64/README.MD @@ -0,0 +1,26 @@ +# Layers for Python 3.12-arm64 + + +| Region (Code) | Region (Name)| ARNs| +| :------------- |:--------|:--------| +| af-south-1 |Africa (Cape Town)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/af-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/af-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/af-south-1/html)| +| ap-east-1 | Asia Pacific (Hong Kong)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-east-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-east-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-east-1/html)| +| ap-northeast-1 |Asia Pacific (Tokyo)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-northeast-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-northeast-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-northeast-1/html)| +| ap-northeast-2 |Asia Pacific (Seoul)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-northeast-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-northeast-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-northeast-2/html)| +| ap-south-1 |Asia Pacific (Mumbai)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-south-1/html)| +| ap-southeast-1 |Asia Pacific (Singapore)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-1/html)| +| ap-southeast-2 |Asia Pacific (Sydney)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-2/html)| +| ap-southeast-3 |Asia Pacific (Jakarta)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-3/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-3/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ap-southeast-3/html)| +| ca-central-1 |Canada (Central)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ca-central-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ca-central-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/ca-central-1/html)| +| eu-central-1 |EU (Frankfurt)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-central-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-central-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-central-1/html)| +| eu-north-1 |EU (Stockholm)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-north-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-north-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-north-1/html)| +| eu-south-1 |EU (Milan)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-south-1/html)| +| eu-west-1 |EU (Ireland)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-1/html)| +| eu-west-2 |EU (London)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-2/html)| +| eu-west-3 |EU (Paris)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-3/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-3/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/eu-west-3/html)| +| me-south-1 |Middle East (Bahrain)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/me-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/me-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/me-south-1/html)| +| sa-east-1 |South America (São Paulo)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/sa-east-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/sa-east-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/sa-east-1/html)| +| us-east-1 |US East (N. Virginia)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-east-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-east-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-east-1/html)| +| us-east-2 |US East (Ohio)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-east-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-east-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-east-2/html)| +| us-west-1 |US West (N. California)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-west-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-west-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-west-1/html)| +| us-west-2 |US West (Oregon)| [json](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-west-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-west-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12-arm64/layers/latest/us-west-2/html)| diff --git a/deployments/python3.12/README.MD b/deployments/python3.12/README.MD new file mode 100644 index 00000000..d341ded8 --- /dev/null +++ b/deployments/python3.12/README.MD @@ -0,0 +1,26 @@ +# Layers for Python 3.11 + + +| Region (Code) | Region (Name)| ARNs| +| :------------- |:--------|:--------| +| af-south-1 |Africa (Cape Town)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/af-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/af-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/af-south-1/html)| +| ap-east-1 | Asia Pacific (Hong Kong)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-east-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-east-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-east-1/html)| +| ap-northeast-1 |Asia Pacific (Tokyo)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-northeast-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-northeast-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-northeast-1/html)| +| ap-northeast-2 |Asia Pacific (Seoul)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-northeast-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-northeast-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-northeast-2/html)| +| ap-south-1 |Asia Pacific (Mumbai)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-south-1/html)| +| ap-southeast-1 |Asia Pacific (Singapore)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-1/html)| +| ap-southeast-2 |Asia Pacific (Sydney)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-2/html)| +| ap-southeast-3 |Asia Pacific (Jakarta)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-3/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-3/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ap-southeast-3/html)| +| ca-central-1 |Canada (Central)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ca-central-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ca-central-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/ca-central-1/html)| +| eu-central-1 |EU (Frankfurt)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-central-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-central-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-central-1/html)| +| eu-north-1 |EU (Stockholm)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-north-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-north-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-north-1/html)| +| eu-south-1 |EU (Milan)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-south-1/html)| +| eu-west-1 |EU (Ireland)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-1/html)| +| eu-west-2 |EU (London)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-2/html)| +| eu-west-3 |EU (Paris)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-3/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-3/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/eu-west-3/html)| +| me-south-1 |Middle East (Bahrain)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/me-south-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/me-south-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/me-south-1/html)| +| sa-east-1 |South America (São Paulo)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/sa-east-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/sa-east-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/sa-east-1/html)| +| us-east-1 |US East (N. Virginia)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-east-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-east-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-east-1/html)| +| us-east-2 |US East (Ohio)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-east-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-east-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-east-2/html)| +| us-west-1 |US West (N. California)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-west-1/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-west-1/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-west-1/html)| +| us-west-2 |US West (Oregon)| [json](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-west-2/json) \| [csv](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-west-2/csv) \| [html](https://api.klayers.cloud/api/v2/p3.12/layers/latest/us-west-2/html)| \ No newline at end of file diff --git a/pipeline/Serverless/README.MD b/pipeline/Serverless/README.MD index 8beb9e0f..678e4826 100644 --- a/pipeline/Serverless/README.MD +++ b/pipeline/Serverless/README.MD @@ -9,3 +9,11 @@ Functions and their corresponding `.yml` files are stored in individual folders. ## Step Functions Step function configuration uses the `serverless-step-functions` plugin, with corresponding `.yml` files stored in the /state_machines folder. + +## Installation + +serverless plugin install -n serverless-step-functions +serverless plugin install -n serverless-iam-roles-per-function +serverless plugin install -n serverless-better-credentials + +export AWS_SDK_LOAD_CONFIG=1 diff --git a/pipeline/Serverless/package-lock.json b/pipeline/Serverless/package-lock.json index eaf54796..5d2da1d6 100644 --- a/pipeline/Serverless/package-lock.json +++ b/pipeline/Serverless/package-lock.json @@ -12,53 +12,24 @@ "serverless": "^2.69.1" }, "devDependencies": { - "serverless-better-credentials": "^1.1.3", + "serverless-better-credentials": "^2.0.0", "serverless-iam-roles-per-function": "^3.2.0", - "serverless-step-functions": "^3.11.0" + "serverless-step-functions": "^3.21.0" } }, - "node_modules/@hapi/address": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", - "integrity": "sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ==", - "deprecated": "Moved to 'npm install @sideway/address'", - "dev": true - }, - "node_modules/@hapi/bourne": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-1.3.2.tgz", - "integrity": "sha512-1dVNHT76Uu5N3eJNTYcvxee+jzX4Z9lfciqRRHCU27ihbUcYi+iSc2iml5Ke1LXe1SyJCLA0+14Jh4tXJgOppA==", - "deprecated": "This version has been deprecated and is no longer supported or maintained", - "dev": true - }, "node_modules/@hapi/hoek": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-8.5.1.tgz", - "integrity": "sha512-yN7kbciD87WzLGc5539Tn0sApjyiGHAJgKvG9W8C7O+6c7qmoQMfVs0W4bX17eqz6C78QJqqFrtgdK5EWf6Qow==", - "deprecated": "This version has been deprecated and is no longer supported or maintained", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", "dev": true }, - "node_modules/@hapi/joi": { - "version": "15.1.1", - "resolved": "https://registry.npmjs.org/@hapi/joi/-/joi-15.1.1.tgz", - "integrity": "sha512-entf8ZMOK8sc+8YfeOlM8pCfg3b5+WZIKBfUaaJT8UsjAAPjartzxIYm3TIbjvA4u+u++KbcXD38k682nVHDAQ==", - "deprecated": "Switch to 'npm install joi'", - "dev": true, - "dependencies": { - "@hapi/address": "2.x.x", - "@hapi/bourne": "1.x.x", - "@hapi/hoek": "8.x.x", - "@hapi/topo": "3.x.x" - } - }, "node_modules/@hapi/topo": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-3.1.6.tgz", - "integrity": "sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ==", - "deprecated": "This version has been deprecated and is no longer supported or maintained", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz", + "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==", "dev": true, "dependencies": { - "@hapi/hoek": "^8.3.0" + "@hapi/hoek": "^9.0.0" } }, "node_modules/@kwsites/file-exists": { @@ -1716,6 +1687,27 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/@sideway/address": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", + "integrity": "sha512-IqO/DUQHUkPeixNQ8n0JA6102hT9CmaljNTPmQ1u8MEhBo/R4Q8eKLN/vGZxuebwOroDB4cbpjheD4+/sKFK4Q==", + "dev": true, + "dependencies": { + "@hapi/hoek": "^9.0.0" + } + }, + "node_modules/@sideway/formula": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz", + "integrity": "sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==", + "dev": true + }, + "node_modules/@sideway/pinpoint": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz", + "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==", + "dev": true + }, "node_modules/@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -2086,20 +2078,53 @@ "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz", "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==" }, + "node_modules/asl-path-validator": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/asl-path-validator/-/asl-path-validator-0.12.0.tgz", + "integrity": "sha512-pzBX2mKp8NQ7p1xM6sfSd2vFQJDX0UdUCun/YcRKMNSv7j93erTomK7iIU79N5rjJD++kPr9qwWhA67pFVpdhA==", + "dev": true, + "dependencies": { + "jsonpath-plus": "^7.0.0" + } + }, "node_modules/asl-validator": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-1.10.0.tgz", - "integrity": "sha512-N7/ouOzYaEJqUNf82NarTs6Cg8sZMnHIDVGULjhzw7GLyXgoXkQUZrhvHXCyucvkU4FfeHmzY3TynEjl0/UN3Q==", + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-3.8.2.tgz", + "integrity": "sha512-wSGCbzbTz5hfuSQl33u/OZEGXT8IE9taRbAPtCNg4boK7oHLlXZqH7VlwKGeKY4QwvDwlyktERBXs3r32JwIeQ==", "dev": true, "dependencies": { - "ajv": "^6.12.6", - "commander": "^5.1.0", - "jsonpath": "^1.1.0" + "ajv": "^8.12.0", + "asl-path-validator": "^0.12.0", + "commander": "^10.0.1", + "jsonpath-plus": "^7.2.0", + "yaml": "^2.3.1" }, "bin": { - "asl-validator": "bin/asl-validator.js" + "asl-validator": "dist/bin/asl-validator.js" + } + }, + "node_modules/asl-validator/node_modules/ajv": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.14.0.tgz", + "integrity": "sha512-oYs1UUtO97ZO2lJ4bwnWeQW8/zvOIQLGKcvPTsWmvc2SYgBb+upuNS5NxoLaMU4h8Ju3Nbj6Cq8mD2LQoqVKFA==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/asl-validator/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -2833,12 +2858,12 @@ } }, "node_modules/commander": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", - "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", "dev": true, "engines": { - "node": ">= 6" + "node": ">=14" } }, "node_modules/component-bind": { @@ -3159,12 +3184,6 @@ "node": ">=4.0.0" } }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, "node_modules/defaults": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", @@ -3481,38 +3500,6 @@ "node": ">=0.8.0" } }, - "node_modules/escodegen": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", - "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", - "dev": true, - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=4.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/escodegen/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/esniff": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/esniff/-/esniff-1.1.0.tgz", @@ -3542,24 +3529,6 @@ "uni-global": "^1.0.0" } }, - "node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/event-emitter": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", @@ -3676,12 +3645,6 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", - "dev": true - }, "node_modules/fastest-levenshtein": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz", @@ -4619,6 +4582,19 @@ "node": ">= 0.6.0" } }, + "node_modules/joi": { + "version": "17.13.1", + "resolved": "https://registry.npmjs.org/joi/-/joi-17.13.1.tgz", + "integrity": "sha512-vaBlIKCyo4FCUtCm7Eu4QZd/q02bWcxfUO6YSXAZOWF6gzcLBeba8kwotUdYJjDLW8Cz8RywsSOqiNJZW0mNvg==", + "dev": true, + "dependencies": { + "@hapi/hoek": "^9.3.0", + "@hapi/topo": "^5.1.0", + "@sideway/address": "^4.1.5", + "@sideway/formula": "^3.0.1", + "@sideway/pinpoint": "^2.0.0" + } + }, "node_modules/js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", @@ -4701,28 +4677,13 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/jsonpath": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/jsonpath/-/jsonpath-1.1.1.tgz", - "integrity": "sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==", - "dev": true, - "dependencies": { - "esprima": "1.2.2", - "static-eval": "2.0.2", - "underscore": "1.12.1" - } - }, - "node_modules/jsonpath/node_modules/esprima": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz", - "integrity": "sha1-dqD9Zvz+FU/SkmZ9wmQBl1CxZXs=", + "node_modules/jsonpath-plus": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", + "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==", "dev": true, - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, "engines": { - "node": ">=0.4.0" + "node": ">=12.0.0" } }, "node_modules/jsprim": { @@ -4883,19 +4844,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "dev": true, - "dependencies": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -5507,23 +5455,6 @@ "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" }, - "node_modules/optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dev": true, - "dependencies": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/ora": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", @@ -5868,15 +5799,6 @@ "once": "^1.3.1" } }, - "node_modules/prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/prepend-http": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", @@ -6222,6 +6144,15 @@ "node": ">= 0.12" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve-alpn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", @@ -6455,12 +6386,12 @@ } }, "node_modules/serverless-better-credentials": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/serverless-better-credentials/-/serverless-better-credentials-1.1.3.tgz", - "integrity": "sha512-specJ3DH70DkstDo0NibL7rETHdrMQ6RLbL1lz0ijjmx6fuV/8XAd6StxoNoXoCCbTT0lXU+OIlfDiyivuca6A==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/serverless-better-credentials/-/serverless-better-credentials-2.0.0.tgz", + "integrity": "sha512-CscjdWPmK58M/X6l/nOgSvlP8U2fQUt1tmlFDp5+/aRMYrLrcZk9JtcYUmnJBTcwk92JYXPjsKei3jk3e///pg==", "dev": true, "dependencies": { - "open": "^8.4.0" + "open": "^8.4.2" }, "engines": { "node": ">=12" @@ -6472,9 +6403,9 @@ } }, "node_modules/serverless-better-credentials/node_modules/open": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz", - "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", "dev": true, "dependencies": { "define-lazy-prop": "^2.0.0", @@ -6501,16 +6432,16 @@ } }, "node_modules/serverless-step-functions": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/serverless-step-functions/-/serverless-step-functions-3.11.0.tgz", - "integrity": "sha512-P/AbwI0MLLHhKq96G3oc3oAH2GWjD/rl55aMjTtNAXC7f/vCQ55k1KDtcvmWN3J6n5cTaEDzKlkWNsnabHy6Qw==", + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/serverless-step-functions/-/serverless-step-functions-3.21.0.tgz", + "integrity": "sha512-u6rMkTQaM4z/GSSy36Q/Utl0c7zaOv3i7sqVpZfvKVSV7LH+E40tptSSKRddECFWIKVERaT14JI7Dnwx9luoCQ==", "dev": true, "dependencies": { - "@hapi/joi": "^15.0.2", "@serverless/utils": "^6.7.0", - "asl-validator": "^1.9.8", + "asl-validator": "^3.8.0", "bluebird": "^3.4.0", "chalk": "^4.1.2", + "joi": "^17.7.0", "lodash": "^4.17.11" }, "peerDependencies": { @@ -7273,15 +7204,6 @@ "node": "*" } }, - "node_modules/static-eval": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz", - "integrity": "sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==", - "dev": true, - "dependencies": { - "escodegen": "^1.8.1" - } - }, "node_modules/stealthy-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", @@ -7970,18 +7892,6 @@ "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" }, - "node_modules/type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "dev": true, - "dependencies": { - "prelude-ls": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", @@ -8045,12 +7955,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/underscore": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", - "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==", - "dev": true - }, "node_modules/uni-global": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/uni-global/-/uni-global-1.0.0.tgz", @@ -8251,15 +8155,6 @@ "lodash": "^4.17.14" } }, - "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -8382,6 +8277,18 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "node_modules/yaml": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.2.tgz", + "integrity": "sha512-B3VqDZ+JAg1nZpaEmWtTXUlBneoGx6CPM9b0TENK6aoSu5t73dItudwdgmi6tHlIZZId4dZ9skcAQ2UbcyAeVA==", + "dev": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/yaml-ast-parser": { "version": "0.0.43", "resolved": "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz", @@ -8429,43 +8336,19 @@ } }, "dependencies": { - "@hapi/address": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", - "integrity": "sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ==", - "dev": true - }, - "@hapi/bourne": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-1.3.2.tgz", - "integrity": "sha512-1dVNHT76Uu5N3eJNTYcvxee+jzX4Z9lfciqRRHCU27ihbUcYi+iSc2iml5Ke1LXe1SyJCLA0+14Jh4tXJgOppA==", - "dev": true - }, "@hapi/hoek": { - "version": "8.5.1", - "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-8.5.1.tgz", - "integrity": "sha512-yN7kbciD87WzLGc5539Tn0sApjyiGHAJgKvG9W8C7O+6c7qmoQMfVs0W4bX17eqz6C78QJqqFrtgdK5EWf6Qow==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", "dev": true }, - "@hapi/joi": { - "version": "15.1.1", - "resolved": "https://registry.npmjs.org/@hapi/joi/-/joi-15.1.1.tgz", - "integrity": "sha512-entf8ZMOK8sc+8YfeOlM8pCfg3b5+WZIKBfUaaJT8UsjAAPjartzxIYm3TIbjvA4u+u++KbcXD38k682nVHDAQ==", - "dev": true, - "requires": { - "@hapi/address": "2.x.x", - "@hapi/bourne": "1.x.x", - "@hapi/hoek": "8.x.x", - "@hapi/topo": "3.x.x" - } - }, "@hapi/topo": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-3.1.6.tgz", - "integrity": "sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-5.1.0.tgz", + "integrity": "sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==", "dev": true, "requires": { - "@hapi/hoek": "^8.3.0" + "@hapi/hoek": "^9.0.0" } }, "@kwsites/file-exists": { @@ -9736,6 +9619,27 @@ } } }, + "@sideway/address": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", + "integrity": "sha512-IqO/DUQHUkPeixNQ8n0JA6102hT9CmaljNTPmQ1u8MEhBo/R4Q8eKLN/vGZxuebwOroDB4cbpjheD4+/sKFK4Q==", + "dev": true, + "requires": { + "@hapi/hoek": "^9.0.0" + } + }, + "@sideway/formula": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sideway/formula/-/formula-3.0.1.tgz", + "integrity": "sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==", + "dev": true + }, + "@sideway/pinpoint": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@sideway/pinpoint/-/pinpoint-2.0.0.tgz", + "integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==", + "dev": true + }, "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -10064,15 +9968,46 @@ "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz", "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==" }, + "asl-path-validator": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/asl-path-validator/-/asl-path-validator-0.12.0.tgz", + "integrity": "sha512-pzBX2mKp8NQ7p1xM6sfSd2vFQJDX0UdUCun/YcRKMNSv7j93erTomK7iIU79N5rjJD++kPr9qwWhA67pFVpdhA==", + "dev": true, + "requires": { + "jsonpath-plus": "^7.0.0" + } + }, "asl-validator": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-1.10.0.tgz", - "integrity": "sha512-N7/ouOzYaEJqUNf82NarTs6Cg8sZMnHIDVGULjhzw7GLyXgoXkQUZrhvHXCyucvkU4FfeHmzY3TynEjl0/UN3Q==", + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-3.8.2.tgz", + "integrity": "sha512-wSGCbzbTz5hfuSQl33u/OZEGXT8IE9taRbAPtCNg4boK7oHLlXZqH7VlwKGeKY4QwvDwlyktERBXs3r32JwIeQ==", "dev": true, "requires": { - "ajv": "^6.12.6", - "commander": "^5.1.0", - "jsonpath": "^1.1.0" + "ajv": "^8.12.0", + "asl-path-validator": "^0.12.0", + "commander": "^10.0.1", + "jsonpath-plus": "^7.2.0", + "yaml": "^2.3.1" + }, + "dependencies": { + "ajv": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.14.0.tgz", + "integrity": "sha512-oYs1UUtO97ZO2lJ4bwnWeQW8/zvOIQLGKcvPTsWmvc2SYgBb+upuNS5NxoLaMU4h8Ju3Nbj6Cq8mD2LQoqVKFA==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + } } }, "asn1": { @@ -10646,9 +10581,9 @@ } }, "commander": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", - "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", "dev": true }, "component-bind": { @@ -10917,12 +10852,6 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, - "deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, "defaults": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", @@ -11198,28 +11127,6 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, - "escodegen": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", - "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", - "dev": true, - "requires": { - "esprima": "^4.0.1", - "estraverse": "^4.2.0", - "esutils": "^2.0.2", - "optionator": "^0.8.1", - "source-map": "~0.6.1" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "optional": true - } - } - }, "esniff": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/esniff/-/esniff-1.1.0.tgz", @@ -11242,18 +11149,6 @@ "uni-global": "^1.0.0" } }, - "estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true - }, - "esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true - }, "event-emitter": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", @@ -11346,12 +11241,6 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", - "dev": true - }, "fastest-levenshtein": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz", @@ -12057,6 +11946,19 @@ "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, + "joi": { + "version": "17.13.1", + "resolved": "https://registry.npmjs.org/joi/-/joi-17.13.1.tgz", + "integrity": "sha512-vaBlIKCyo4FCUtCm7Eu4QZd/q02bWcxfUO6YSXAZOWF6gzcLBeba8kwotUdYJjDLW8Cz8RywsSOqiNJZW0mNvg==", + "dev": true, + "requires": { + "@hapi/hoek": "^9.3.0", + "@hapi/topo": "^5.1.0", + "@sideway/address": "^4.1.5", + "@sideway/formula": "^3.0.1", + "@sideway/pinpoint": "^2.0.0" + } + }, "js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", @@ -12126,24 +12028,11 @@ "graceful-fs": "^4.1.6" } }, - "jsonpath": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/jsonpath/-/jsonpath-1.1.1.tgz", - "integrity": "sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==", - "dev": true, - "requires": { - "esprima": "1.2.2", - "static-eval": "2.0.2", - "underscore": "1.12.1" - }, - "dependencies": { - "esprima": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz", - "integrity": "sha1-dqD9Zvz+FU/SkmZ9wmQBl1CxZXs=", - "dev": true - } - } + "jsonpath-plus": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", + "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==", + "dev": true }, "jsprim": { "version": "1.4.2", @@ -12298,16 +12187,6 @@ } } }, - "levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "dev": true, - "requires": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - } - }, "lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -12781,20 +12660,6 @@ "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" }, - "optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dev": true, - "requires": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" - } - }, "ora": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", @@ -13041,12 +12906,6 @@ } } }, - "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", - "dev": true - }, "prepend-http": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", @@ -13294,6 +13153,12 @@ "tough-cookie": "^2.3.3" } }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true + }, "resolve-alpn": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", @@ -13774,18 +13639,18 @@ } }, "serverless-better-credentials": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/serverless-better-credentials/-/serverless-better-credentials-1.1.3.tgz", - "integrity": "sha512-specJ3DH70DkstDo0NibL7rETHdrMQ6RLbL1lz0ijjmx6fuV/8XAd6StxoNoXoCCbTT0lXU+OIlfDiyivuca6A==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/serverless-better-credentials/-/serverless-better-credentials-2.0.0.tgz", + "integrity": "sha512-CscjdWPmK58M/X6l/nOgSvlP8U2fQUt1tmlFDp5+/aRMYrLrcZk9JtcYUmnJBTcwk92JYXPjsKei3jk3e///pg==", "dev": true, "requires": { - "open": "^8.4.0" + "open": "^8.4.2" }, "dependencies": { "open": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz", - "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", "dev": true, "requires": { "define-lazy-prop": "^2.0.0", @@ -13805,16 +13670,16 @@ } }, "serverless-step-functions": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/serverless-step-functions/-/serverless-step-functions-3.11.0.tgz", - "integrity": "sha512-P/AbwI0MLLHhKq96G3oc3oAH2GWjD/rl55aMjTtNAXC7f/vCQ55k1KDtcvmWN3J6n5cTaEDzKlkWNsnabHy6Qw==", + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/serverless-step-functions/-/serverless-step-functions-3.21.0.tgz", + "integrity": "sha512-u6rMkTQaM4z/GSSy36Q/Utl0c7zaOv3i7sqVpZfvKVSV7LH+E40tptSSKRddECFWIKVERaT14JI7Dnwx9luoCQ==", "dev": true, "requires": { - "@hapi/joi": "^15.0.2", "@serverless/utils": "^6.7.0", - "asl-validator": "^1.9.8", + "asl-validator": "^3.8.0", "bluebird": "^3.4.0", "chalk": "^4.1.2", + "joi": "^17.7.0", "lodash": "^4.17.11" }, "dependencies": { @@ -14080,15 +13945,6 @@ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" }, - "static-eval": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-2.0.2.tgz", - "integrity": "sha512-N/D219Hcr2bPjLxPiV+TQE++Tsmrady7TqAJugLy7Xk1EumfDWS/f5dtBbkRCGE7wKKXuYockQoj8Rm2/pVKyg==", - "dev": true, - "requires": { - "escodegen": "^1.8.1" - } - }, "stealthy-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", @@ -14651,15 +14507,6 @@ "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" }, - "type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "dev": true, - "requires": { - "prelude-ls": "~1.1.2" - } - }, "type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", @@ -14702,12 +14549,6 @@ } } }, - "underscore": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", - "integrity": "sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==", - "dev": true - }, "uni-global": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/uni-global/-/uni-global-1.0.0.tgz", @@ -14880,12 +14721,6 @@ "triple-beam": "^1.2.0" } }, - "word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -14969,6 +14804,12 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, + "yaml": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.2.tgz", + "integrity": "sha512-B3VqDZ+JAg1nZpaEmWtTXUlBneoGx6CPM9b0TENK6aoSu5t73dItudwdgmi6tHlIZZId4dZ9skcAQ2UbcyAeVA==", + "dev": true + }, "yaml-ast-parser": { "version": "0.0.43", "resolved": "https://registry.npmjs.org/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz", diff --git a/pipeline/Serverless/package.json b/pipeline/Serverless/package.json index 561df27e..c6fa1b4a 100644 --- a/pipeline/Serverless/package.json +++ b/pipeline/Serverless/package.json @@ -6,9 +6,9 @@ "serverless": "^2.69.1" }, "devDependencies": { - "serverless-better-credentials": "^1.1.3", + "serverless-better-credentials": "^2.0.0", "serverless-iam-roles-per-function": "^3.2.0", - "serverless-step-functions": "^3.11.0" + "serverless-step-functions": "^3.21.0" }, "main": "index.js", "scripts": { diff --git a/pipeline/Terraform/.gitignore b/pipeline/Terraform/.gitignore index 610af6ce..13e39ad8 100644 --- a/pipeline/Terraform/.gitignore +++ b/pipeline/Terraform/.gitignore @@ -1,2 +1,3 @@ .terraform/ -.terraform.* \ No newline at end of file +.terraform.* +!.terraform.lock.hcl \ No newline at end of file diff --git a/pipeline/Terraform/.terraform.lock.hcl b/pipeline/Terraform/.terraform.lock.hcl new file mode 100644 index 00000000..bb4eb28a --- /dev/null +++ b/pipeline/Terraform/.terraform.lock.hcl @@ -0,0 +1,85 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.67.0" + constraints = "~> 4.0" + hashes = [ + "h1:P43vwcDPG99x5WBbmqwUPgfJrfXf6/ucAIbGlRb7k1w=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.1" + constraints = ">= 1.2.0" + hashes = [ + "h1:tjcGlQAFA0kmQ4vKkIPPUC4it1UYxLbg4YvHOWRAJHA=", + "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561", + "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561", + "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5", + "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24", + "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667", + "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8", + "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0", + "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867", + "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4", + "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.1" + constraints = ">= 2.2.0" + hashes = [ + "h1:8iqExjtAvirFTJkpm5YyYD+fC+DGV8NTJzKsE2c70VA=", + "zh:2a0ec154e39911f19c8214acd6241e469157489fc56b6c739f45fbed5896a176", + "zh:57f4e553224a5e849c99131f5e5294be3a7adcabe2d867d8a4fef8d0976e0e52", + "zh:58f09948c608e601bd9d0a9e47dcb78e2b2c13b4bda4d8f097d09152ea9e91c5", + "zh:5c2a297146ed6fb3fe934c800e78380f700f49ff24dbb5fb5463134948e3a65f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7ce41e26f0603e31cdac849085fc99e5cd5b3b73414c6c6d955c0ceb249b593f", + "zh:8c9e8d30c4ef08ee8bcc4294dbf3c2115cd7d9049c6ba21422bd3471d92faf8a", + "zh:93e91be717a7ffbd6410120eb925ebb8658cc8f563de35a8b53804d33c51c8b0", + "zh:982542e921970d727ce10ed64795bf36c4dec77a5db0741d4665230d12250a0d", + "zh:b9d1873f14d6033e216510ef541c891f44d249464f13cc07d3f782d09c7d18de", + "zh:cfe27faa0bc9556391c8803ade135a5856c34a3fe85b9ae3bdd515013c0c87c1", + "zh:e4aabf3184bbb556b89e4b195eab1514c86a2914dd01c23ad9813ec17e863a8a", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.11.1" + constraints = ">= 0.7.0" + hashes = [ + "h1:UyhbtF79Wy4EVNrnvMcOPzmZLVQQyzM2ostfjs2l5PI=", + "zh:19a393db736ec4fd024d098d55aefaef07056c37a448ece3b55b3f5f4c2c7e4a", + "zh:227fa1e221de2907f37be78d40c06ca6a6f7b243a1ec33ade014dfaf6d92cd9c", + "zh:29970fecbf4a3ca23bacbb05d6b90cdd33dd379f90059fe39e08289951502d9f", + "zh:65024596f22f10e7dcb5e0e4a75277f275b529daa0bc0daf34ca7901c678ab88", + "zh:694d080cb5e3bf5ef08c7409208d061c135a4f5f4cdc93ea8607860995264b2e", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b29d15d13e1b3412e6a4e1627d378dbd102659132f7488f64017dd6b6d5216d3", + "zh:bb79f4cae9f8c17c73998edc54aa16c2130a03227f7f4e71fc6ac87e230575ec", + "zh:ceccf80e95929d97f62dcf1bb3c7c7553d5757b2d9e7d222518722fc934f7ad5", + "zh:f40e638336527490e294d9c938ae55919069e6987e85a80506784ba90348792a", + "zh:f99ef33b1629a3b2278201142a3011a8489e66d92da832a5b99e442204de18fb", + "zh:fded14754ea46fdecc62a52cd970126420d4cd190e598cb61190b4724a727edb", + ] +} diff --git a/pipeline/Terraform/container_repositories.tf b/pipeline/Terraform/container_repositories.tf index b51c0e5c..a05f824a 100644 --- a/pipeline/Terraform/container_repositories.tf +++ b/pipeline/Terraform/container_repositories.tf @@ -1,24 +1,5 @@ # Container Build Images -## Python 3.9 - x86 (only) -resource "aws_ecr_repository" "p39build_x86" { - name = "p39build" - image_tag_mutability = "MUTABLE" - force_delete = true - - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ssm_parameter" "p39_build_repo" { - type = "String" - description = "URL for p39 x86 repo" - name = "/${var.app_name}/${local.workspace_full_name}/build/p39/x86/repo" - value = aws_ecr_repository.p39build_x86.repository_url - overwrite = true -} - ## Python 3.10 - x86 resource "aws_ecr_repository" "p310build_x86" { name = "p310build" @@ -95,3 +76,21 @@ resource "aws_ssm_parameter" "p311_arm64_build_repo" { overwrite = true } + +## Python 3.12 builds + +module "python312_x86_build" { + source = "./container_repository" + app_name = var.app_name + workspace_full_name = local.workspace_full_name + python_version = "p312" + architecture = "x86" +} + +module "python312_arm64_build" { + source = "./container_repository" + app_name = var.app_name + workspace_full_name = local.workspace_full_name + python_version = "p312" + architecture = "arm64" +} \ No newline at end of file diff --git a/pipeline/Terraform/oidc_github/outputs.tf b/pipeline/Terraform/oidc_github/outputs.tf index 08389d3a..1c75d560 100644 --- a/pipeline/Terraform/oidc_github/outputs.tf +++ b/pipeline/Terraform/oidc_github/outputs.tf @@ -1,3 +1,7 @@ output "github_role_arn" { value = aws_iam_role.github_role.arn +} + +output "github_oidc_provider_arn" { + value = aws_iam_openid_connect_provider.github.arn } \ No newline at end of file diff --git a/pipeline/Terraform/s3.tf b/pipeline/Terraform/s3.tf index 64cac955..42b567a3 100644 --- a/pipeline/Terraform/s3.tf +++ b/pipeline/Terraform/s3.tf @@ -120,4 +120,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ddb_backup_bucket_config" { } status = "Enabled" } -} \ No newline at end of file +} + + diff --git a/pipeline/Terraform/terraform.tfvars.json b/pipeline/Terraform/terraform.tfvars.json index 5aca1ced..cb04a235 100644 --- a/pipeline/Terraform/terraform.tfvars.json +++ b/pipeline/Terraform/terraform.tfvars.json @@ -20,7 +20,14 @@ "Klayers-devp38": "api-dev.klayers.cloud", "Klayers-prodp38": "api.klayers.cloud" }, + "website_domain_name":{ + "Klayers-defaultp38": "default.klayers.cloud", + "Klayers-devp38": "dev.klayers.cloud", + "Klayers-prodp38": "klayers.cloud" + }, "github_repo": "git@github.com:keithrozario/Klayers.git", + "website_github_repo": "git@github.com:keithrozario/klayers-website.git", + "website_role_name": "github-action-website", "s3bucket_config_parameter_name_suffix": "config_bucket/name", "github_role_name":{ "Klayers-defaultp38": "github_role_default", diff --git a/pipeline/Terraform/variables.tf b/pipeline/Terraform/variables.tf index cfb3c4d7..63faf000 100644 --- a/pipeline/Terraform/variables.tf +++ b/pipeline/Terraform/variables.tf @@ -6,4 +6,7 @@ variable "api_domain_name" { type = map(any) } variable "cron_expression_invoke_pipelines" { type = map(any) } variable "s3bucket_config_parameter_name_suffix" { type = string } variable "github_role_name" { type = map(any) } -variable "config_dir" { type = map(string) } # not used but declared to avoid warning from Terraform \ No newline at end of file +variable "config_dir" { type = map(string) } # not used but declared to avoid warning from Terraform +variable "website_domain_name" { type = map(any) } +variable "website_github_repo" { type = string } +variable "website_role_name" { type = string } diff --git a/pipeline/Terraform/website.tf b/pipeline/Terraform/website.tf new file mode 100644 index 00000000..5af0fb62 --- /dev/null +++ b/pipeline/Terraform/website.tf @@ -0,0 +1,109 @@ +/* +We automated the AWS portion of the website, by creating: + +1. A S3 bucket to host the static assets +2. A CloudFront distribution to serve the content +3. An IAM role for the repo to `s3 sync` to the bucket via the OIDC provider already created. +4. The IAM role will be limited to the github repo under website_github_repo variable in tfvars.json + + +Unfortunately, because we host the domain on Cloudfront, the cert still has to be manually verified. +And the DNS has to manually pointed (via CNAME entry) from Cloudflare to the Cloudfront distribution in (2) +*/ + + +# Creates the bucket and CDN for the website.data +resource "aws_s3_bucket" "website_bucket" { + provider = aws.cloudfront + + bucket_prefix = "klayers-website" + force_destroy = true +} + +resource "aws_ssm_parameter" "website_bucket_name" { + provider = aws.cloudfront + type = "String" + description = "Name of s3 bucket to hold website" + name = "/${var.app_name}/${local.workspace_full_name}/website_bucket/name" + value = aws_s3_bucket.website_bucket.bucket +} + +resource "aws_s3_bucket_public_access_block" "website_bucket" { + provider = aws.cloudfront + bucket = aws_s3_bucket.website_bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +module "cdn" { + providers = { + aws = aws.cloudfront + } + source = "cloudposse/cloudfront-s3-cdn/aws" + version = "0.94.0" + + origin_bucket = aws_s3_bucket.website_bucket.id + aliases = [lookup(var.website_domain_name, local.workspace_full_name)] + acm_certificate_arn = module.website_certificate.cert_arn + depends_on = [module.website_certificate] + cloudfront_access_logging_enabled = false + name = lookup(var.website_domain_name, local.workspace_full_name) +} + + +## This has to be manually set in Cloudflare. +module "website_certificate" { + source = "./certificate_manager" + api_domain_name = lookup(var.website_domain_name, local.workspace_full_name) + app_name = var.app_name + workspace_name = local.workspace_full_name + + providers = { + aws = aws.cloudfront + } + +} + +## Role + +data "aws_iam_policy_document" "github_actions_assume_role_policy" { + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [module.oidc_github.github_oidc_provider_arn] + } + condition { + test = "StringLike" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${var.website_github_repo}:*"] + } + } +} + +resource "aws_iam_role" "website_github_role" { + name = var.website_role_name + assume_role_policy = data.aws_iam_policy_document.github_actions_assume_role_policy.json +} + +data "aws_iam_policy_document" "website_github_role" { + statement { + actions = ["s3:PutObject"] + resources = ["${aws_s3_bucket.website_bucket.arn}/*"] + } +} + +resource "aws_iam_policy" "website_github_policy" { + name = "github-action-website" + path = "/" + description = "Sync to website bucket" + policy = data.aws_iam_policy_document.website_github_role.json +} + +resource "aws_iam_role_policy_attachment" "attach_github_on_push" { + role = aws_iam_role.website_github_role.name + policy_arn = aws_iam_policy.website_github_policy.arn +} \ No newline at end of file diff --git a/pipeline/config/packages_p310.csv b/pipeline/config/packages_p310.csv index 4f9f7fde..b9b9e11d 100644 --- a/pipeline/config/packages_p310.csv +++ b/pipeline/config/packages_p310.csv @@ -6,8 +6,10 @@ beautifulsoup4,MIT,Leonard Richardson boto3,Apache-2.0,AWS cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors dynamodb-encryption-sdk,Apache-2.0, +fastapi, MIT, Sebastián Ramírez idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure.com.au jinja2,BSD,Armin Ronache ; Pallets +mangum, MIT, Jordan Eremieff numpy,https://www.numpy.org/license.html,numpy-discussion@python.org openpyxl,MIT,Charlie Clark pandas,BSD, @@ -22,3 +24,12 @@ mysql-connector-python,GNU GPLv2,Oracle Pillow,https://github.com/python-pillow/Pillow/blob/main/LICENSE,Jeffrey A. Clark aliyun-python-sdk-actiontrail,Apache-2.0,Aliyun tabulate,MIT,s.astanin@gmail.com +paramiko,GNU GPLv2,Jeff Forcier +pydantic,MIT,Samuel Colvin +polars,MIT,Ritchie Vink +s3fs,BSD,Matthew Rocklin +zeep,MIT, Michael van Tellingen +scipy,BSD-3-Clause,scipy-dev@python.org +openai,Apache Software License,support@openai.com +llama-index,MIT,Jerry Liu +praw,https://github.com/praw-dev/praw/blob/master/LICENSE.txt,Bryce Boe diff --git a/pipeline/config/packages_p311-arm64.csv b/pipeline/config/packages_p311-arm64.csv index 0d963796..76bf5426 100644 --- a/pipeline/config/packages_p311-arm64.csv +++ b/pipeline/config/packages_p311-arm64.csv @@ -10,6 +10,7 @@ idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure. jinja2,BSD,Armin Ronache ; Pallets numpy,https://www.numpy.org/license.html,numpy-discussion@python.org pandas,BSD, +Pillow,https://github.com/python-pillow/Pillow/blob/master/LICENSE,Alex Clark pyqldb,Apache-2.0,AWS redshift-connector,Apache License Version 2.0,Amazon Web Services requests,Apache-2.0,Kenneth Reitz diff --git a/pipeline/config/packages_p311.csv b/pipeline/config/packages_p311.csv index 83570158..21c6d65d 100644 --- a/pipeline/config/packages_p311.csv +++ b/pipeline/config/packages_p311.csv @@ -1,4 +1,5 @@ Package_Name,License,Authors/Maintainers +aws-psycopg2,MIT, aws-xray-sdk,Apache-2.0,AWS aws-requests-auth,BSD,davehmuller@gmail.com bcrypt,Apache-2.0,cryptography-dev@python.org @@ -6,8 +7,11 @@ beautifulsoup4,MIT,Leonard Richardson boto3,Apache-2.0,AWS cryptography,Apache-2.0,The Python Cryptographic Authority and individual contributors dynamodb-encryption-sdk,Apache-2.0, +fastapi, MIT, Sebastián Ramírez +google-cloud-bigquery,Apache 2.0,Google LLC idna,https://github.com/kjd/idna/blob/master/LICENSE.rst,Kim Davis kim@cynosure.com.au jinja2,BSD,Armin Ronache ; Pallets +mangum, MIT, Jordan Eremieff matplotlib,PSF,matplotlib-users@python.org mysql-connector-python,GNU GPLv2,Oracle numpy,https://www.numpy.org/license.html,numpy-discussion@python.org @@ -17,3 +21,24 @@ pyqldb,Apache-2.0,AWS redshift-connector,Apache License Version 2.0,Amazon Web Services requests,Apache-2.0,Kenneth Reitz tabulate,MIT,s.astanin@gmail.com +requests-html,MIT,Kenneth Reitz +elasticsearch,Apache-2.0,Honza Kral ; Nick Lang +Pillow,https://github.com/python-pillow/Pillow/blob/master/LICENSE,Alex Clark +psycopg,LGPL-3.0,Daniele Varrazzo +pymysql,MIT,songofacandy@gmail.com +selenium,Apache Software License (Apache 2.0),The Selenium Project +paramiko,GNU GPLv2,Jeff Forcier +pydantic,MIT,Samuel Colvin +polars,MIT,Ritchie Vink +s3fs,BSD,Matthew Rocklin +zeep,MIT, Michael van Tellingen +XlsxWriter, BSD 2-Clause, John McNamara +xlsx2csv, MIT, Dilshod Temirkhdojaev +sqlmodel, MIT, Sebastián Ramírez +python-docx, MIT, Steve Canny +extract-msg, GPL-3.0, Destiny Peterson +polars, MIT, Ritchie Vink +msal, MIT, Microsoft +lxml, BSD, Infrae +reportlab, BSD, ReportLab Europe Ltd. +praw,https://github.com/praw-dev/praw/blob/master/LICENSE.txt,Bryce Boe diff --git a/pipeline/config/packages_p312.csv b/pipeline/config/packages_p312.csv index 7895dee3..46c08807 100644 --- a/pipeline/config/packages_p312.csv +++ b/pipeline/config/packages_p312.csv @@ -15,4 +15,11 @@ redshift-connector,Apache License Version 2.0,Amazon Web Services mysql-connector-python,GNU GPLv2,Oracle aliyun-python-sdk-actiontrail,Apache-2.0,Aliyun -tabulate,MIT,s.astanin@gmail.com \ No newline at end of file +tabulate,MIT,s.astanin@gmail.com +dnspython,ISC License (ISCL),halley@dnspython.org +python-pptx, MIT, python-pptx +memray, Apache 2.0, godlygeek pablogsal +polars, MIT, Ritchie Vink +Pillow,https://github.com/python-pillow/Pillow/blob/master/LICENSE,Alex Clark +psycopg, LGPL-3.0, Daniele Varrazzo +praw,https://github.com/praw-dev/praw/blob/master/LICENSE.txt,Bryce Boe \ No newline at end of file diff --git a/pipeline/container_images/build_images/common/build.py b/pipeline/container_images/build_images/common/build.py index 285d9e43..e7bd09c5 100644 --- a/pipeline/container_images/build_images/common/build.py +++ b/pipeline/container_images/build_images/common/build.py @@ -248,6 +248,19 @@ def dir_size(path="."): return total +def find_test_dirs(start_dir): + """ + List out all diretories named 'tests' in the current directory and its subdirectories + Removing this reduces the size of the final output + """ + test_dirs = [] + for root, dirs, files in os.walk(start_dir): + for dir in dirs: + if dir == 'tests': + test_dirs.append(os.path.join(root, dir)) + return test_dirs + + def install(package, package_dir): """ " Args: @@ -267,6 +280,7 @@ def install(package, package_dir): "-t", package_dir, "--quiet", + "--no-compile", # does not compile __pycache__ or .pyc to save space "--upgrade", "--no-cache-dir", ], @@ -274,6 +288,20 @@ def install(package, package_dir): ) logger.info(output) + tests_dirs = find_test_dirs(package_dir) + logger.info(f"Test Directories: {tests_dirs}") + size_before_tests_deletion = dir_size(package_dir) + for dir in tests_dirs: + shutil.rmtree(dir) # delete tests dirs to save space + size_after_tests_deletion = dir_size(package_dir) + logger.info( + { + "message": "Deleted tests directories", + "size before": size_before_tests_deletion, + "size after": size_after_tests_deletion, + } + ) + return package_dir diff --git a/pipeline/container_images/build_images/common/build_optimized.py b/pipeline/container_images/build_images/common/build_optimized.py new file mode 100644 index 00000000..e589bb98 --- /dev/null +++ b/pipeline/container_images/build_images/common/build_optimized.py @@ -0,0 +1,428 @@ +""" +This is an optimized version of build.py that: + +1. Uses the --no-compile python installation option to save space +2. Deletes all directories in the installation that is named 'tests', to save space. +""" + + +import os +import sys +import shutil +import hashlib +from datetime import datetime + +import boto3 +from botocore.exceptions import ClientError + +from aws_lambda_powertools.logging import Logger + +logger = Logger() + + +def get_pk_sk_latest_build(package: str, python_version: str): + build_v0_prefix = "bldVrsn0#" + package_prefix = "pckg#" + sk = {"S": f"{package_prefix}{package}"} + pk = {"S": f"{build_v0_prefix}{python_version}"} + + return pk, sk + + +def put_requirements_hash( + python_version: str, + package: str, + version: str, + requirements_txt: str, + requirements_hash: str, +): + """ + Args: + package: Package name + python_version: Version of python "p." (e.g. p3.8, p3.9, p3.10) + version: Package version + requirements_hash: SHA256 hash of the requirements.txt file + requirements_txt: requirements txt of the entire build + returns: + None + """ + client = boto3.client("dynamodb") + table_name = os.environ["DB_NAME"] + + pk, sk = get_pk_sk_latest_build(package, python_version) + + # Get latest build version for package + build_version_prefix = "bld#v" + response = client.get_item( + TableName=table_name, + Key={"pk": pk, "sk": sk}, + ProjectionExpression="bltVrsn", + ) + try: + latest_version = int(response["Item"]["bltVrsn"]["N"]) + new_version = latest_version + 1 + except KeyError: + # Version wasn't deployed before, start with v1 + new_version = 1 + + created_date = datetime.utcnow().isoformat() + Item = { + "pk": {"S": f"{build_version_prefix}{new_version}:{python_version}"}, + "sk": sk, + "pckgVrsn": {"S": str(version)}, + "rqrmntsTxt": {"S": requirements_txt}, + "rqrmntsHsh": {"S": requirements_hash}, + "bltVrsn": {"N": str(new_version)}, + "crtdDt": {"S": created_date}, + "pckg": {"S": package}, + "pyVrsn": {"S": python_version}, + } + + # Insert new record + try: + response = client.transact_write_items( + TransactItems=[ + { + "Update": { + "TableName": table_name, + "Key": { + "pk": pk, + "sk": sk, + }, + "UpdateExpression": "set " + "rqrmntsTxt = :rqrmntsTxt, " + "pckgVrsn = :pckgVrsn, " + "rqrmntsHsh = :rqrmntsHsh," + "bltVrsn = :bltVrsn," + "crtdDt = :crtdDt," + "pyVrsn = :pyVrsn", + "ExpressionAttributeValues": { + ":rqrmntsTxt": {"S": requirements_txt}, + ":pckgVrsn": {"S": str(version)}, + ":rqrmntsHsh": {"S": requirements_hash}, + ":bltVrsn": {"N": str(new_version)}, + ":crtdDt": {"S": created_date}, + ":pyVrsn": {"S": python_version}, + }, + "ConditionExpression": "bltVrsn <> :bltVrsn", + } + }, + { + "Put": { + "TableName": table_name, + "Item": Item, + } + }, + ] + ) + logger.info({"message": "Successfully written", "item": Item}) + logger.debug(f"DynamoDB response: {response}") + except ClientError as e: + logger.error( + { + "error_code": e.response["Error"]["Code"], + "error_message": e.response["Error"]["Message"], + "item": Item, + "message": "Failed to Update record for Build Version", + } + ) + exit(1) + + return None + + +def check_requirement_hash(package: str, python_version: str, requirements_hash): + """ + Args: + python_version: Version of python (e.g. p3.8, p3.9, p3.10) + package: Package name + requirements_hash: SHA256 hash of the requirements.txt file + returns: + exists: Boolean value of if the requirements_hash exists in the DB (package was built already) + """ + + client = boto3.client("dynamodb") + table_name = os.environ["DB_NAME"] + pk, sk = get_pk_sk_latest_build(package, python_version) + + response = client.get_item( + TableName=table_name, + Key={"pk": pk, "sk": sk}, + ProjectionExpression="rqrmntsHsh", + ) + + if requirements_hash == response.get("Item", {}).get("rqrmntsHsh", {}).get( + "S", False + ): + hash_match = True + else: + hash_match = False + + return hash_match + + +def freeze_requirements(package, path): + """ + Walks through path, looking for *.dist-info folders. Parses out the package name and versions + returns: package name and version in requirements.txt format as a string + """ + import subprocess + + logger.info("Getting requirements.txt file") + os.environ["PYTHONPATH"] = "/opt/python" + pip_install = subprocess.run( + ["pip", "freeze", "--path", path], shell=False, capture_output=True + ) + requirements_txt = pip_install.stdout.decode("utf-8").strip() + logger.info(f"Requirements txt : \n{requirements_txt}") + requirements_hash = hashlib.sha256(requirements_txt.encode("utf-8")).hexdigest() + + version = None + for line in requirements_txt.split("\n"): + if line[: len(package)].lower() == package.lower(): + version = line.split("==")[1] + logger.info(f"Version of {package} found is {version}") + break + + if version is None: + logger.error( + "Unable to determine version of package....refer to logs for requirements.txt" + ) + exit(1) + + return requirements_txt, requirements_hash, version + + +def upload_to_s3(zip_file, package, uploaded_file_name): + """ + Args: + zip_file: Location of zip file to be uploaded to S3 bucket + package: Name of python package being uploaded + return: + uploaded_file_name: Name of file in S3 bucket + """ + + bucket_name = os.environ["BUCKET_NAME"] + + s3 = boto3.resource("s3") + s3.meta.client.upload_file(zip_file, bucket_name, uploaded_file_name) + + client = boto3.client("s3") + response = client.list_objects_v2(Bucket=bucket_name, Prefix=uploaded_file_name) + logger.info(response) + logger.info( + { + "message": f"Uploaded {package}.zip", + "size": response["Contents"][0]["Size"], + "time": response["Contents"][0]["LastModified"], + "bucket": bucket_name, + } + ) + + return uploaded_file_name + + +def zip_dir(dir_path, package): + zip_file = f"/tmp/{package}" + result = shutil.make_archive( + base_name=zip_file, + format="zip", + base_dir=dir_path.split("/")[-1], + root_dir="/tmp", + ) + logger.info(result) + return f"{zip_file}.zip" + + +def delete_dir(dir): + try: + shutil.rmtree(dir) + logger.debug("Deleted previous version of package directory") + except FileNotFoundError: + logger.debug("No previous installation detected") + return True + + +def dir_size(path="."): + total = 0 + try: + for entry in os.scandir(path): + if entry.is_file(): + total += entry.stat().st_size + elif entry.is_dir(): + total += dir_size(entry.path) + except FileNotFoundError: + total = 0 + return total + + +def find_test_dirs(start_dir): + """ + List out all diretories named 'tests' in the current directory and its subdirectories + Removing this reduces the size of the final output + """ + test_dirs = [] + for root, dirs, files in os.walk(start_dir): + for dir in dirs: + if dir == 'tests': + test_dirs.append(os.path.join(root, dir)) + return test_dirs + + +def install(package, package_dir): + """ " + Args: + package: Name of package to be queried + return: + path to zip file of final package + """ + delete_dir(package_dir) + import subprocess + + os.environ["PYTHONPATH"] = "/opt/python" + output = subprocess.run( + [ + "pip", + "install", + package, + "-t", + package_dir, + "--quiet", + "--no-compile", # does not compile __pycache__ to save space + "--upgrade", + "--no-cache-dir", + ], + capture_output=True, + ) + logger.info(output) + + # Delete all directories named 'tests'. + tests_dirs = find_test_dirs(package_dir) + logger.info(f"Test Directories: {tests_dirs}") + size_before_tests_deletion = dir_size(package_dir) + for dir in tests_dirs: + shutil.rmtree(dir) # delete tests dirs to save space + size_after_tests_deletion = dir_size(package_dir) + logger.info( + { + "message": "Deleted tests directories", + "size before": size_before_tests_deletion, + "size after": size_after_tests_deletion, + } + ) + + return package_dir + + +def check_python_version(python_version: str) -> bool: + """ " + Args: + python_version: Version of python required in form of major.minor + return: + True if matches running version, False otherwise + """ + running_python_version = f"p{sys.version_info.major}.{sys.version_info.minor}" + python_version_supplied = python_version.split("-")[0] # remove trailing arm64 + + if python_version_supplied == running_python_version: + logger.debug( + { + "Python version": python_version, + "Python version supplied": python_version_supplied, + "Python version running": running_python_version, + } + ) + return True + else: + logger.error( + { + "Error": "Python version doesn't match", + "Python version": python_version, + "Python version supplied": python_version_supplied, + "Python version running": running_python_version, + } + ) + return False + + +@logger.inject_lambda_context +def main(event, context): + package = event["package"] + license_info = event["license_info"] + python_version = event["python_version"] + force_build = event["force_build"] + force_deploy = event["force_deploy"] + + if not check_python_version(python_version): + sys.exit(1) + + package_dir = f"/tmp/python" + uploaded_file_name = f"{python_version}/{package}.zip" + build_flag = False + + package_dir = install(package, package_dir=package_dir) + package_size = dir_size(package_dir) + logger.info({"package": package, "size": package_size}) + + requirements_txt, requirements_hash, version = freeze_requirements( + package=package, path=package_dir + ) + logger.info({"message": "Built Package", "requirements_txt": requirements_txt}) + + with open(f"{package_dir}/requirements.txt", "w") as requirements_file: + requirements_file.write(requirements_txt) + zip_file = zip_dir(dir_path=package_dir, package=package) + + if force_build or not check_requirement_hash( + package=package, + requirements_hash=requirements_hash, + python_version=python_version, + ): + logger.info( + { + "requirements_hash": requirements_hash, + "package": package, + "version": version, + "python_version": python_version, + "message": "Uploading to S3", + } + ) + + upload_to_s3( + zip_file=zip_file, package=package, uploaded_file_name=uploaded_file_name + ) + put_requirements_hash( + package=package, + requirements_txt=requirements_txt, + requirements_hash=requirements_hash, + version=version, + python_version=python_version, + ) + + logger.info( + { + "package": package, + "version": version, + "location": f"s3://{os.environ['BUCKET_NAME']}", + "size": os.path.getsize(zip_file), + "requirements_hash": requirements_hash, + } + ) + build_flag = True + + else: + build_flag = False + logger.info( + "Requirements hash previously built, proceeding to check for deployment" + ) + + return { + "zip_file_S3key": uploaded_file_name, + "package": package, + "version": version, + "requirements_hash": requirements_hash, + "license_info": license_info, + "build_flag": build_flag, + "force_deploy": force_deploy, + "python_version": python_version, + } \ No newline at end of file diff --git a/pipeline/container_images/build_images/p312_arm64/Dockerfile b/pipeline/container_images/build_images/p312_arm64/Dockerfile index d6a3486e..b6013827 100644 --- a/pipeline/container_images/build_images/p312_arm64/Dockerfile +++ b/pipeline/container_images/build_images/p312_arm64/Dockerfile @@ -1,6 +1,6 @@ FROM public.ecr.aws/lambda/python:3.12-arm64 -COPY build.py ./ +COPY build_optimized.py ./build.py RUN pip install --upgrade pip RUN pip install boto3 RUN pip install aws-lambda-powertools diff --git a/pipeline/container_images/build_images/p312_x86/Dockerfile b/pipeline/container_images/build_images/p312_x86/Dockerfile index 9ff7dcfb..7ed61063 100644 --- a/pipeline/container_images/build_images/p312_x86/Dockerfile +++ b/pipeline/container_images/build_images/p312_x86/Dockerfile @@ -1,6 +1,7 @@ FROM public.ecr.aws/lambda/python:3.12 COPY build.py ./ +COPY build_optimized.py ./build.py RUN pip install --upgrade pip RUN pip install boto3 RUN pip install aws-lambda-powertools