From 1e1a9b70387c21bbcd82e958e69f06a7e688c395 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Fri, 27 Oct 2023 09:28:38 -0500 Subject: [PATCH 1/2] Jenkins dockerfiles improvements (#2398) --- .github/workflows/image_build_push.yaml | 8 ++++---- .github/workflows/image_build_push_jenkins.yaml | 11 ++++++----- .github/workflows/image_build_push_squid.yaml | 5 +++-- .secrets.baseline | 4 ++-- Docker/jenkins/Jenkins-CI-Worker/Dockerfile | 7 +++---- Docker/jenkins/Jenkins-Worker/Dockerfile | 6 +----- kube/services/jobs/usersync-job.yaml | 2 +- 7 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml index 51543f0fe..d5bfea351 100644 --- a/.github/workflows/image_build_push.yaml +++ b/.github/workflows/image_build_push.yaml @@ -1,10 +1,10 @@ -name: Build Python Base Images and Push to Quay and ECR +name: Build Python Base Images on: push jobs: python_3-9: - name: Python 3.9 Build and Push + name: Python 3.9 uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile" @@ -17,7 +17,7 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} python_3-10: - name: Python 3.10 Build and Push + name: Python 3.10 uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile" @@ -30,7 +30,7 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} awshelper: - name: AwsHelper Build and Push + name: AwsHelper uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile" diff --git a/.github/workflows/image_build_push_jenkins.yaml b/.github/workflows/image_build_push_jenkins.yaml index 2d85aedf1..094417fe5 100644 --- a/.github/workflows/image_build_push_jenkins.yaml +++ b/.github/workflows/image_build_push_jenkins.yaml @@ -1,13 +1,14 @@ -name: Build Jenkins images and push to Quay +name: Build Jenkins images on: push: paths: + - .github/workflows/image_build_push_jenkins.yaml - Docker/jenkins/** jobs: jenkins: - name: Jenkins Build and Push + name: Jenkins uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins/Dockerfile" @@ -21,7 +22,7 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} jenkins2: - name: Jenkins2 Build and Push + name: Jenkins2 uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins2/Dockerfile" @@ -35,7 +36,7 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} jenkins-ci-worker: - name: Jenkins-CI-Worker Build and Push + name: Jenkins-CI-Worker uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins-CI-Worker/Dockerfile" @@ -49,7 +50,7 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} jenkins-qa-worker: - name: Jenkins-QA-Worker Build and Push + name: Jenkins-QA-Worker uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/jenkins/Jenkins-Worker/Dockerfile" diff --git a/.github/workflows/image_build_push_squid.yaml b/.github/workflows/image_build_push_squid.yaml index 2849f0cc5..ce1761d3c 100644 --- a/.github/workflows/image_build_push_squid.yaml +++ b/.github/workflows/image_build_push_squid.yaml @@ -1,13 +1,14 @@ -name: Build Squid images and push to Quay +name: Build Squid images on: push: paths: + - .github/workflows/image_build_push_squid.yaml - Docker/squid/** jobs: squid: - name: Squid Build and Push + name: Squid image uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/squid/Dockerfile" diff --git a/.secrets.baseline b/.secrets.baseline index 919833990..0a8fe9cc9 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2023-09-18T18:49:22Z", + "generated_at": "2023-10-26T21:32:44Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -79,7 +79,7 @@ "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", "is_secret": false, "is_verified": false, - "line_number": 122, + "line_number": 121, "type": "Secret Keyword" } ], diff --git a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile index 40fd08fa3..f0da68f69 100644 --- a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile @@ -34,11 +34,10 @@ RUN set -xe && apt-get update \ zlib1g-dev \ zsh \ ca-certificates-java \ - openjdk-11-jre-headless \ && ln -s /usr/bin/lua5.3 /usr/local/bin/lua # Use jdk11 -ENV JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64" +ENV JAVA_HOME="/opt/java/openjdk" ENV PATH="$JAVA_HOME/bin:$PATH" COPY ./certfix.sh /certfix.sh @@ -75,7 +74,7 @@ RUN sudo install -m 0755 -d /etc/apt/keyrings \ # install nodejs RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - -RUN apt-get update && apt-get install -y nodejs +RUN apt-get update && apt-get install -y nodejs npm # Install postgres 13 client RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc| gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg && \ @@ -98,7 +97,7 @@ RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release && \ sed -i 's/python3/python3.8/' /usr/bin/add-apt-repository # install aws cli, poetry, pytest, etc. -RUN set -xe && python3.8 -m pip install --upgrade pip && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade +RUN set -xe && python3.8 -m pip install --upgrade pip setuptools && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade # install terraform RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \ diff --git a/Docker/jenkins/Jenkins-Worker/Dockerfile b/Docker/jenkins/Jenkins-Worker/Dockerfile index c31e54923..c824690de 100644 --- a/Docker/jenkins/Jenkins-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-Worker/Dockerfile @@ -8,6 +8,7 @@ RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils build-ess RUN apt-get update \ && apt-get install -y lsb-release \ + git \ apt-transport-https \ r-base \ libffi-dev \ @@ -36,11 +37,6 @@ RUN apt-get update \ # install Ruby. RUN apt-get install -y ruby-full -# install GIT from buster-backports -RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/buster-backports.list \ - && apt-get update \ - && apt-get -t=buster-backports -y install git=1:2.30.* - # # install docker tools: # diff --git a/kube/services/jobs/usersync-job.yaml b/kube/services/jobs/usersync-job.yaml index 8f148a3b0..8a5471a20 100644 --- a/kube/services/jobs/usersync-job.yaml +++ b/kube/services/jobs/usersync-job.yaml @@ -260,7 +260,7 @@ spec: exit 1 fi #----------------- - echo "awshelper downloading ${userYamlS3Path} to /mnt/shared/useryaml"; + echo "awshelper downloading ${userYamlS3Path} to /mnt/shared/user.yaml"; n=0 until [ $n -ge 5 ]; do echo "Download attempt $n" From 3ff37b9775bb8d0d65c4303c4e6824072a9af9be Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Fri, 27 Oct 2023 21:05:42 +0200 Subject: [PATCH 2/2] Feat: add new config for integrating teamproject and arborist check (#2397) * feat: add new config for integrating teamproject and arborist check ...these properties are needed for enabling "teamproject" authorization mode in WebAPI and telling WebAPI where the Arborist endpoint is living for actually getting authorization information. * feat: add ARBORIST_URL as env var for WebAPI * fix: use ARBORIST_URL in config.yaml * feat: add default for ARBORIST_URL * revert default value for ARBORIST_URL ...the service will already default to arborist-service url if not defined --- kube/services/ohdsi-webapi/ohdsi-webapi-config.yaml | 3 +++ kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/kube/services/ohdsi-webapi/ohdsi-webapi-config.yaml b/kube/services/ohdsi-webapi/ohdsi-webapi-config.yaml index 5cd46edd9..8eb01ec08 100644 --- a/kube/services/ohdsi-webapi/ohdsi-webapi-config.yaml +++ b/kube/services/ohdsi-webapi/ohdsi-webapi-config.yaml @@ -55,6 +55,9 @@ stringData: security_oauth_callback_api: https://atlas.$hostname/WebAPI/user/oauth/callback security_oauth_callback_urlResolver: query + security_ohdsi_custom_authorization_mode: teamproject + security_ohdsi_custom_authorization_url: $ARBORIST_URL/auth/mapping + logging_level_root: info logging_level_org_ohdsi: info logging_level_org_apache_shiro: info diff --git a/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml b/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml index 65d6ed38c..258aa8f87 100644 --- a/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml +++ b/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml @@ -59,6 +59,13 @@ spec: containers: - name: ohdsi-webapi GEN3_OHDSI-WEBAPI_IMAGE|-image: quay.io/cdis/ohdsi-webapi:latest-| + env: + - name: ARBORIST_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true livenessProbe: httpGet: path: /WebAPI/info/