From cc80ff24b0ecc83afa115266c82ebd6d9d286d7c Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Thu, 21 Sep 2023 16:41:49 +1000 Subject: [PATCH 1/8] [gha] Test Remote Datacube Connection --- .github/workflows/test_rds.yml | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/test_rds.yml diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml new file mode 100644 index 000000000..638895a65 --- /dev/null +++ b/.github/workflows/test_rds.yml @@ -0,0 +1,60 @@ + +name: Test RDS Connection + +on: + push: + +permissions: + id-token: write # Required for OIDC + contents: read +jobs: + test-rds: + runs-on: ubuntu-latest + + # Once we've setup our mamba environment, activate it automatically + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox + aws-region: ap-southeast-2 + + + - name: Get Database Credentials + run: | + username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text) + IFS=: read -r PGUSERNAME PGPASSWORD <<< $username_password + echo "PGUSERNAME=${PGUSERNAME}" >> $GITHUB_ENV + echo "PGPASSWORD=${PGPASSWORD}" >> $GITHUB_ENV + echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV + + - name: Open Port Forward to RDS + run: | + npx basti connect \ + --custom-target-vpc vpc-086904199e505c1f6 \ + --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \ + --custom-target-port 5432 \ + --local-port 5432 & + # Wait until the connection comes up, but, if it doesn't, don't hang forever. + npx wait-on --verbose --timeout 60000 --interval 1000 tcp:127.0.0.1:5432 + echo "PGPORT=5432" >> $GITHUB_ENV + echo "PGHOST=localhost" >> $GITHUB_ENV + + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: '1.5.1' + environment-name: test-env + create-args: >- + python=3.10 + datacube + + - name: Test RDS Connection + run: | + datacube system check + datacube product list From b8ef03d1d1017668fdf2709abf3118556158e45f Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Thu, 21 Sep 2023 17:06:28 +1000 Subject: [PATCH 2/8] Fix Micromamba setup --- .github/workflows/test_rds.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index 638895a65..5843d55d8 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -19,6 +19,13 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: '1.5.1-0' + environment-name: test-env + create-args: >- + python=3.10 + datacube - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -29,9 +36,9 @@ jobs: - name: Get Database Credentials run: | username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text) - IFS=: read -r PGUSERNAME PGPASSWORD <<< $username_password - echo "PGUSERNAME=${PGUSERNAME}" >> $GITHUB_ENV - echo "PGPASSWORD=${PGPASSWORD}" >> $GITHUB_ENV + # IFS=: read -r PGUSERNAME PGPASSWORD <<< $username_password + # echo "PGUSERNAME=${PGUSERNAME}" >> $GITHUB_ENV + # echo "PGPASSWORD=${PGPASSWORD}" >> $GITHUB_ENV echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV - name: Open Port Forward to RDS @@ -46,13 +53,6 @@ jobs: echo "PGPORT=5432" >> $GITHUB_ENV echo "PGHOST=localhost" >> $GITHUB_ENV - - uses: mamba-org/setup-micromamba@v1 - with: - micromamba-version: '1.5.1' - environment-name: test-env - create-args: >- - python=3.10 - datacube - name: Test RDS Connection run: | From f967f79a1fb50f895d833c78886535fa39781293 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 22 Sep 2023 16:22:38 +1000 Subject: [PATCH 3/8] Test S3 Access and RDS Access --- .github/workflows/test_rds.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index 5843d55d8..7a194de89 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -5,13 +5,13 @@ on: push: permissions: - id-token: write # Required for OIDC + id-token: write # Required for OIDC authentication to AWS contents: read jobs: test-rds: runs-on: ubuntu-latest - # Once we've setup our mamba environment, activate it automatically + # Automatically activate our mamba environment when it's ready defaults: run: shell: bash -l {0} @@ -26,16 +26,18 @@ jobs: create-args: >- python=3.10 datacube + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox aws-region: ap-southeast-2 - - name: Get Database Credentials run: | username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption --query Parameter.Value --output text) + # If we need separate username/password they can be split as follows. + # However exporting them to GitHub Environment variables will leave the Password in the Logs, if further steps aren't taken. # IFS=: read -r PGUSERNAME PGPASSWORD <<< $username_password # echo "PGUSERNAME=${PGUSERNAME}" >> $GITHUB_ENV # echo "PGPASSWORD=${PGPASSWORD}" >> $GITHUB_ENV @@ -49,7 +51,7 @@ jobs: --custom-target-port 5432 \ --local-port 5432 & # Wait until the connection comes up, but, if it doesn't, don't hang forever. - npx wait-on --verbose --timeout 60000 --interval 1000 tcp:127.0.0.1:5432 + npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432 echo "PGPORT=5432" >> $GITHUB_ENV echo "PGHOST=localhost" >> $GITHUB_ENV @@ -58,3 +60,9 @@ jobs: run: | datacube system check datacube product list + + - name: Test S3 Access + run: | + aws s3 ls dea-public-data + aws s3 ls dea-public-data-dev + aws s3 cp s3://dea-public-data/index.html . From fc0de11ec7e97c44eed304efa21473c4111ef263 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 22 Sep 2023 22:47:55 +1000 Subject: [PATCH 4/8] attempt to run the whole kit and kaboodle --- .github/workflows/test_rds.yml | 97 +++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index 7a194de89..2a4931ddf 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -18,14 +18,33 @@ jobs: steps: - uses: actions/checkout@v4 + with: + path: dea-notebooks - - uses: mamba-org/setup-micromamba@v1 + # - uses: mamba-org/setup-micromamba@v1 + # with: + # micromamba-version: '1.5.1-0' + # environment-name: test-env + # create-args: >- + # python=3.10 + # datacube + # + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 with: - micromamba-version: '1.5.1-0' - environment-name: test-env - create-args: >- - python=3.10 - datacube + role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly + aws-region: ap-southeast-2 + + - name: Copy tide modelling files with the AWS CLI + run: aws s3 sync s3://dea-non-public-data/tide_models/tide_models tide_models + + - name: Login to Amazon ECR Private + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Pull dea-sandbox image from ECR + run: | + docker pull 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 @@ -56,13 +75,63 @@ jobs: echo "PGHOST=localhost" >> $GITHUB_ENV - - name: Test RDS Connection + - name: Run the Notebook tests run: | - datacube system check - datacube product list + sudo chown -R 1000:100 ./dea-notebooks + cd ./dea-notebooks + docker run \ + --env DATACUBE_DB_URL \ + --env AWS_SESSION_TOKEN \ + --env AWS_REGION \ + --env AWS_ACCESS_KEY_ID \ + --env AWS_SECRET_ACCESS_KEY \ + --env AWS_SESSION_TOKEN \ + --volume ${GITHUB_WORKSPACE}/dea-notebooks:/home/jovyan/dea-notebooks \ + --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \ + --env GDAL_HTTP_MAX_RETRY=3 \ + --entrypoint /bin/bash \ + 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable + ./dea-notebooks/Tests/test_notebooks.sh + + # login-to-amazon-ecr: + # runs-on: ubuntu-latest + # steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly + # aws-region: ap-southeast-2 + # mask-aws-account-id: 'false' + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v1 + # outputs: + # registry: ${{ steps.login-ecr.outputs.registry }} + # docker_username: ${{ steps.login-ecr.outputs.docker_username_538673716275_dkr_ecr_ap_southeast_2_amazonaws_com }} # More information on these outputs can be found below in the 'Docker Credentials' section + # docker_password: ${{ steps.login-ecr.outputs.docker_password_538673716275_dkr_ecr_ap_southeast_2_amazonaws_com }} + # + # run-with-internal-service: + # name: Run tests using dea-sandbox image + # needs: login-to-amazon-ecr + # runs-on: ubuntu-latest + # container: + # image: ${{ needs.login-to-amazon-ecr.outputs.registry }}/geoscienceaustralia/sandbox:stable + # credentials: + # username: ${{ needs.login-to-amazon-ecr.outputs.docker_username }} + # password: ${{ needs.login-to-amazon-ecr.outputs.docker_password }} + # ports: + # - '80:80' + # steps: + # - name: Run steps in container + # run: echo "run steps in container" + # - name: Test RDS Connection + # run: | + # datacube system check + # datacube product list + # + # - name: Test S3 Access + # run: | + # aws s3 ls dea-public-data + # aws s3 ls dea-public-data-dev + # aws s3 cp s3://dea-public-data/index.html . - - name: Test S3 Access - run: | - aws s3 ls dea-public-data - aws s3 ls dea-public-data-dev - aws s3 cp s3://dea-public-data/index.html . From 0d67622bc27542cfc61f1e851f805fe841c5a836 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 22 Sep 2023 22:49:07 +1000 Subject: [PATCH 5/8] Fix GHA Action v numbers --- .github/workflows/test_rds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index 2a4931ddf..49dbf036d 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -30,7 +30,7 @@ jobs: # datacube # - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: arn:aws:iam::538673716275:role/github-actions-role-readonly aws-region: ap-southeast-2 @@ -40,7 +40,7 @@ jobs: - name: Login to Amazon ECR Private id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@v1 - name: Pull dea-sandbox image from ECR run: | From 72c1cb3881bc233b375593ab0e4590a007685e24 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 22 Sep 2023 23:05:41 +1000 Subject: [PATCH 6/8] Whoops, missed a slash --- .github/workflows/test_rds.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index 49dbf036d..984dbb2fa 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -79,7 +79,7 @@ jobs: run: | sudo chown -R 1000:100 ./dea-notebooks cd ./dea-notebooks - docker run \ + docker run --rm \ --env DATACUBE_DB_URL \ --env AWS_SESSION_TOKEN \ --env AWS_REGION \ @@ -90,8 +90,8 @@ jobs: --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \ --env GDAL_HTTP_MAX_RETRY=3 \ --entrypoint /bin/bash \ - 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable - ./dea-notebooks/Tests/test_notebooks.sh + 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com/geoscienceaustralia/sandbox:stable \ + /home/jovyan/dea-notebooks/Tests/test_notebooks.sh # login-to-amazon-ecr: # runs-on: ubuntu-latest From b2423a73b203a50c7272115407613bab63b56253 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Fri, 22 Sep 2023 23:18:35 +1000 Subject: [PATCH 7/8] Magic to access the port outside the docker container --- .github/workflows/test_rds.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index 984dbb2fa..e2534b72d 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -80,7 +80,8 @@ jobs: sudo chown -R 1000:100 ./dea-notebooks cd ./dea-notebooks docker run --rm \ - --env DATACUBE_DB_URL \ + --add-host=host.docker.internal:host-gateway \ + --env "DATACUBE_DB_URL=${DATACUBE_DB_URL/localhost/host.docker.internal}" \ --env AWS_SESSION_TOKEN \ --env AWS_REGION \ --env AWS_ACCESS_KEY_ID \ From d88ba04a02b4540b98c824052c29db33d70b2293 Mon Sep 17 00:00:00 2001 From: Damien Ayers Date: Sat, 23 Sep 2023 21:57:34 +1000 Subject: [PATCH 8/8] Run on host network to access RDS Port Forward --- .github/workflows/test_rds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_rds.yml b/.github/workflows/test_rds.yml index e2534b72d..b8cd17fbd 100644 --- a/.github/workflows/test_rds.yml +++ b/.github/workflows/test_rds.yml @@ -80,8 +80,8 @@ jobs: sudo chown -R 1000:100 ./dea-notebooks cd ./dea-notebooks docker run --rm \ - --add-host=host.docker.internal:host-gateway \ - --env "DATACUBE_DB_URL=${DATACUBE_DB_URL/localhost/host.docker.internal}" \ + --net=host \ + --env DATACUBE_DB_URL \ --env AWS_SESSION_TOKEN \ --env AWS_REGION \ --env AWS_ACCESS_KEY_ID \