Merge pull request #1 from jaytmiller/migrate-rb2

Migrate science platform image building from AWS back to GitHub

.github/workflows/build-test.yml

@@ -1,28 +1,23 @@
 name:  Image Build, Test
 on:
-   push:
-      branches:
-        - main
    pull_request:
       branches:
         - main
    workflow_dispatch:
 jobs:
   docker:
     name: "${{ matrix.DEPLOYMENT_NAME }}  USE_FROZEN=${{ matrix.USE_FROZEN }} ${{ matrix.CAL_VERSION }} -- Image Build, Test"
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     strategy:
-      max-parallel: 4
+      max-parallel: 3
       fail-fast: false
       matrix:
         DEPLOYMENT_NAME: [ jwebbinar, roman, tike]
-        USE_FROZEN: [ 0 ]
-        CAL_VERSION: [ latest ]
-        include:
-        - DEPLOYMENT_NAME: roman
-          USE_FROZEN: 0
-          CAL_VERSION: none
+        USE_FROZEN: [ 1 ]
+        CAL_VERSION: [ none ]
+        FREEZE_CHILL: [ 1 ]
+        OWNER: [ jupyter ]
 
     steps:
       - name: Checkout Code
@@ -51,7 +46,7 @@ jobs:
       - name: Set Up Environment
         shell: bash
         run: |
-           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }}  ${{ matrix.USE_FROZEN }} 1  ${{ matrix.CAL_VERSION }}
+           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }}  --use-frozen ${{ matrix.USE_FROZEN }} --freeze-chill ${{ matrix.FREEZE_CHILL }} --cal-version ${{ matrix.CAL_VERSION }}  --owner ${{ MATRIX.owner }}
            df -h
 
       - name: Image Build
@@ -81,6 +76,6 @@ jobs:
         uses: anchore/scan-action@v3
         with:
           fail-build: false
-          severity-cutoff: medium
+          severity-cutoff: high
           output-format: table
           image: notebook-${{ matrix.DEPLOYMENT_NAME }}

.github/workflows/quality.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   quality-scanning:
     name: "${{ matrix.DEPLOYMENT_NAME }}  -- Code Quality"
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     strategy:
       max-parallel: 5
@@ -30,7 +30,7 @@ jobs:
       - name: Set Up Environment
         shell: bash
         run: |
-           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }}  ${{ matrix.USE_FROZEN }}
+           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }}  --use-frozen ${{ matrix.USE_FROZEN }}
            pip install -r requirements.txt
            df -h
 
@@ -39,7 +39,7 @@ jobs:
         continue-on-error: false
         run: |
            source setup-env
-           scripts/sscan-yaml `find deployments tools -name '*.yaml' -o -name '*.yml'`
+           scripts/sscan-yaml `find deployments .github scripts -name '*.yaml' -o -name '*.yml'`
 
       - name: Flake8 Scan
         shell: bash

.github/workflows/security.yaml → .github/workflows/security.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   security-scanning:
     name: "${{ matrix.DEPLOYMENT_NAME }}  -- Security"
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     strategy:
       max-parallel: 5
@@ -29,7 +29,7 @@ jobs:
       - name: Set Up Environment
         shell: bash
         run: |
-           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }}  ${{ matrix.USE_FROZEN }}
+           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }}  --use-frozen ${{ matrix.USE_FROZEN }}
            pip install -r requirements.txt
            df -h
 
@@ -44,6 +44,6 @@ jobs:
         uses: anchore/scan-action@v3
         with:
           fail-build: true
-          severity-cutoff: critical
+          severity-cutoff: high
           output-format: table
-          path: "."
+          path: deployments/${{ matrix.DEPLOYMENT_NAME }}/env-frozen

.github/workflows/update-frozen.yml

@@ -8,14 +8,17 @@ on:
 jobs:
   docker:
     name: "${{ matrix.DEPLOYMENT_NAME }} -- Image Build, Test"
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
 
     strategy:
       max-parallel: 3
       fail-fast: false
       matrix:
         DEPLOYMENT_NAME: [ tike, jwebbinar, roman ]
-        USE_FROZEN: [ 2 ]
+        USE_FROZEN: [ chilly ]
+        FREEZE_CHILL: [ 1 ]
+        OWNER: [ jupyter ]
+
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
@@ -45,7 +48,7 @@ jobs:
       - name: Set Up Environment
         shell: bash
         run: |
-           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }} ${{ matrix.USE_FROZEN }}
+           scripts/image-configure ${{ matrix.DEPLOYMENT_NAME }} --use-frozen ${{ matrix.USE_FROZEN }} --freeze-chill ${{ matrix.FREEZE_CHILL }} --owner ${{ matrix.OWNER }}
            df -h
 
       - name: Image Build

deployments/common/Dockerfile

@@ -3,7 +3,7 @@
 # If the image (jupyter/scipy-notebook) exists locally it will be used,
 # otherwise the latest from https://hub.docker.com/_/buildpack-deps will be used
 
-ARG BASE_IMAGE=stsci-jh/scipy-notebook
+ARG BASE_IMAGE=spacetelescope/scipy-notebook
 FROM $BASE_IMAGE
 
 # ------------------------------------------------------------------------
@@ -17,15 +17,58 @@ ENV MKL_THREADING_LAYER="GNU"
 ENV CONDA_DIR=/opt/conda
 ENV SHELL=/bin/bash
 ENV PATH=${CONDA_DIR}/bin:${PATH}
-ENV CFLAGS="-fcommon"
+ENV CFLAGS="-fcommon -fpic"
 
 # ------------------------------------------------------------------------
 USER root
 
+ARG TZ='America/New_York'
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG USE_FROZEN=1
+ENV USE_FROZEN=$USE_FROZEN
+
+ARG PIP_SWITCHES="--no-cache-dir"
+ENV PIP_SWITCHES=$PIP_SWITCHES
+
+# Enable easy swap of conda with e.g. mamba
+ARG CONDA_VER=conda
+ENV CONDA_VER=${CONDA_VER}
+
+# Directories used for Docker RUN caching
+ARG CACHE_DIRS=""
+ENV CACHE_DIRS=${CACHE_DIRS}
+
+# ----- # Clearing caches at the end of each RUN saves image space
+# ----- # but seems to mess up dependency debug tools like pipdeptree.
+# ----- # NOT setting this is also a requirement of effective Docker
+# ----- # buildkit cache use.  ATM jupyter/docker-stacks will still wipe
+# ----- # out caches created prior to the common base image regardless.
+# ----- ARG CLEAR_PKG_CACHES="1"
+# ----- ENV CLEAR_PKG_CACHES=${CLEAR_PKG_CACHES}
+
+# These are duplicated in jupyter/docker-stacks so need to be changed
+# in both locations.  Better than hard coding in scripts though.
+ENV NB_UID=1000
+ENV NB_GID=100
+
+RUN mkdir -p /home/jovyan/.cache && chown -R jovyan:users /home/jovyan/.cache
+RUN mkdir -p /home/jovyan/.conda/pkgs && chown -R jovyan:users /home/jovyan/.conda/pkgs
+RUN mkdir -p /opt/conda/pkgs &&  chown -R jovyan:users /opt/conda/pkgs
+
 # Breaking into separate runs will build slower but also defines storage
 # consumption in docker history.
 #
-# COPY common-scripts/apt-install  /opt/common-scripts/apt-install
+COPY common-scripts/apt-install  /opt/common-scripts/apt-install
+RUN /opt/common-scripts/apt-install \
+    apt-utils \
+    tzdata \
+    ca-certificates \
+    openssl \
+    wget \
+    vim \
+    npm \
+    nodejs
 
 # Misc system tools
 RUN /opt/common-scripts/apt-install \
@@ -65,6 +108,9 @@ RUN /opt/common-scripts/apt-install \
         graphviz \
         libopenblas-dev
 
+# Install Rust language
+RUN /opt/common-scripts/apt-install cargo rustc
+
 # Libraries for FITS, DS9, ML
 RUN /opt/common-scripts/apt-install \
         file \
@@ -128,6 +174,8 @@ ENV BASH_ENV=/etc/bash.env
 USER ${NB_UID}
 COPY default-home-contents/   /etc/default-home-contents
 
+ENV  PIP_TOOLS_CACHE_DIR=/home/jovyan/.cache/pip
+
 # ----------------------------------------------------------------------
 # YYYY SEC
 

deployments/common/Dockerfile.trailer

@@ -2,10 +2,6 @@
 
 USER $NB_UID
 
-# Fix JS encoding issue that prevents widgets from displaying properly in notebook mode
-# See https://github.com/jupyter/notebook/issues/6033 for details
-RUN sed -i 's/data-base-url="{{base_url | urlencode}}"/data-base-url="{{base_url}}"/g' `ls -1 ${CONDA_DIR}/lib/python*/site-packages/notebook/templates/notebook.html`
-
 # ----------------------------------------------------------------------
 
 RUN /opt/common-scripts/kernel-setup   # set up Ipython / JupyterLab kernels
@@ -16,7 +12,7 @@ RUN /opt/common-scripts/kernel-setup   # set up Ipython / JupyterLab kernels
 
 USER root
 
-RUN ( find /home/${NB_USER} ! -uid ${NB_UID} | xargs chown ${NB_UID}:${NB_GID} ) || true
+RUN find /home/${NB_USER} ! -uid ${NB_UID} | xargs chown ${NB_UID}:${NB_GID} || true
 
 # remove this step once nbgitpuller enabled; these contents will be in the
 #  jupyterhub-user-content repo.   Install deployment-specific $HOME files.
@@ -50,11 +46,11 @@ USER root
 # for running on AWS,  just common functions needed for both AWS and standalone/CI.
 
 # Enable deletion of non-empty directories from lab GUI
-RUN echo "-------------------------------- STSCI Config Overwrites --------------------------------" \
+RUN echo "# -------------------------------- STSCI Config Overwrites --------------------------------" \
       >> /etc/jupyter/jupyter_notebook_config.py &&\
     echo "c.FileContentsManager.delete_to_trash = False" >> /etc/jupyter/jupyter_notebook_config.py &&\
     echo "c.FileContentsManager.always_delete_dir = True" >> /etc/jupyter/jupyter_notebook_config.py &&\
-    echo "-------------------------------- STSCI Config Overwrites --------------------------------" \
+    echo "# -------------------------------- STSCI Config Overwrites --------------------------------" \
       >> /etc/jupyter/jupyter_server_config.py &&\
     echo "c.FileContentsManager.delete_to_trash = False" >> /etc/jupyter/jupyter_server_config.py &&\
     echo "c.FileContentsManager.always_delete_dir = True" >> /etc/jupyter/jupyter_server_config.py

deployments/common/common-env/jupyter.pip

@@ -28,3 +28,5 @@ git+https://github.com/spacetelescope/nersc-refresh-announcements@octarine-updat
 sidecar
 y-py
 ypy-websocket
+#jupyter_collaboration
+jupyter_bokeh

deployments/common/common-scripts/apt-install

@@ -18,10 +18,5 @@ apt-get update --yes --fix-missing
 
 DEBIAN_FRONTEND=noninteractive apt-get install -f --yes ${PACKAGES}
 
-# Only clear the cache if buildkit caching and dev mode are not being used.
-if [[ "${CLEAR_PKG_CACHES}" == "1" ]]; then
-    apt-get clean
-fi
-
 # These dirs are not cached by buildkit as we configure it
 rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

deployments/common/common-scripts/copy-default-home

@@ -20,3 +20,7 @@ else
     echo "Skipping default config init,  $HOME already initialized."
 fi
 
+# Delete caches owned by root, immutable residue of Docker RUN cache
+# which will block actual user caching.  Fortunately $HOME is writable
+# so root owned caches can be removed to make way for user caching.
+find ${CACHE_DIRS} -user 0 | grep -v /var/cache | xargs rm -rvf

deployments/common/common-scripts/env-compile

@@ -8,9 +8,9 @@ Step 2 of the pip-tools workflow:  resolve pip versions from .pip package list f
 
 Nominal pip-tools workflow:
 
-    1. /opt/common-scripts/env-conda  <env>     # Create minimal conda environment.
+    1. /opt/common-scripts/env-conda   <env>    # Create minimal conda environment.
     2. /opt/common-scripts/env-compile <env>    # Resolve pip dependencies to requirements.txt
-    3. /opt/common-scripts/env-sync  <env>      # Download and install requirements.txt
+    3. /opt/common-scripts/env-sync    <env>    # Install requirements.txt,  remove anything else
     4+ /opt/common-scripts/env-src-install  ... # Build packages with missing binaries from source.
     ...
 
@@ -113,5 +113,3 @@ rm -f ${env_reqs}  # ... hence this delete is REQUIRED
 
 # The real work of determining the version solution...  outputs FROZEN pip reqs
 pip-compile  -v  --resolver=backtracking --output-file ${env_reqs} --pip-args "${PIP_SWITCHES}"  ${all_constraints}
-
-/opt/common-scripts/env-clean

deployments/common/common-scripts/env-conda

@@ -74,7 +74,3 @@ ${CONDA_VER} env export --no-build -n ${env} > ${env_frozen_yml}
 # ${CONDA_VER} env remove -n ${env}
 # echo "===== Creating conda environment ${env} from frozen constraints ${env_frozen_yml}"
 # env_create ${env_frozen_yml}
-
-source /opt/common-scripts/env-activate ${env}
-
-/opt/common-scripts/env-clean

deployments/common/common-scripts/env-src-install

@@ -29,6 +29,6 @@ PKG=$2
 
 source /opt/common-scripts/env-activate $ENV
 
-pip install --verbose --no-deps --no-cache --no-binary :all: --force-reinstall --no-build-isolation ${PKG}
+pip install --verbose --no-deps --no-cache --no-binary ${PKG} --force-reinstall --no-build-isolation ${PKG}
 
 /opt/common-scripts/env-clean

deployments/common/common-scripts/env-update

@@ -48,8 +48,3 @@ for filepath in $*; do
 done
 
 source /opt/common-scripts/env-deactivate ${env}
-
-
-# ----------------------------------------------------------------------
-
-/opt/common-scripts/env-clean

deployments/common/common-scripts/install-common

@@ -10,10 +10,14 @@ env=$1
 source /opt/common-scripts/env-activate ${env}
 
 #  Install pip packages defined by /opt/common/*.conda
-/opt/common-scripts/env-update $env `find /opt/common-env -name '*.conda' | sort`
+/opt/common-scripts/env-update $env `find /opt/common-env -name '*.conda' | sort` `ls /opt/environments/common-hints.conda 2>/dev/null`
 
 #  Install pip packages defined in all of *.pip
-/opt/common-scripts/env-update $env `find /opt/common-env -name '*.pip' | sort` /opt/environments/common-hints.pip
+/opt/common-scripts/env-update $env `find /opt/common-env -name '*.pip' | sort` `ls /opt/environments/common-hints.pip 2>/dev/null`
 
 # Save fully pinned requirements for base,  copy out of image to git later.
 ${CONDA_VER} env export --no-build -n base >/opt/env-frozen/base/requirements.yml
+
+if pip freeze | grep jupyter-server-proxy; then
+    jupyter labextension install @jupyterlab/server-proxy
+fi