Merge pull request #28357 from loganharbour/apptainer_openmpi

Update openmpi container, container cleanup
idaholab · Aug 15, 2024 · d7328bf · d7328bf
2 parents a347719 + ae310dd
commit d7328bf
Show file tree

Hide file tree

Showing 19 changed files with 91 additions and 110 deletions.
diff --git a/apptainer/app.def b/apptainer/app.def
@@ -215,6 +215,13 @@ EOF
 %test
     set -x
 
+    # If we're using openmpi, we need to allow running as root
+    # if we're in a fakeroot environment
+    if [ $(whoami) == root ] && which ompi_info &> /dev/null; then
+        export OMPI_ALLOW_RUN_AS_ROOT=1
+        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+    fi
+
     # Load jinja vars
     export BINARY_NAME={{ BINARY_NAME }}
     export METHOD={{ METHOD }}

diff --git a/apptainer/moose-dev.def b/apptainer/moose-dev.def
@@ -2,7 +2,6 @@
 {#- APPTAINER_BOOTSTRAP: The BootStrap to use (typically localimage or oras)                  -#}
 {#- APPTAINER_FROM: The From to use (path to an image or an oras URI)                         -#}
 {#- MOOSE_DIR: Path on the host to the MOOSE repository                                       -#}
-{#- MOOSE_DIR: Path on the host to the MOOSE repository                                       -#}
 {#- WASP_GIT_SHA: The git SHA to use for WASP                                                 -#}
 {#- WASP_GIT_REMOTE: The git remote to use to get WASP                                        -#}
 
@@ -34,8 +33,6 @@
 {#- The script used to install wasp                                                           -#}
 {%- set WASP_BUILD_SCRIPT = 'update_and_rebuild_wasp.sh' -%}
 
-{%- set SET_PROMPT_SCRIPT = '/.singularity.d/set_prompt.bash' -%}
-
 BootStrap: {{ APPTAINER_BOOTSTRAP }}
 From: {{ APPTAINER_FROM }}
 
@@ -68,15 +65,6 @@ From: {{ APPTAINER_FROM }}
     WASP_GIT_REMOTE={{ WASP_GIT_REMOTE }}
     export MOOSE_JOBS={{ MOOSE_JOBS }}
 
-    # Eventually, this will go in the base container as far down as
-    # we can. But we don't want to do a full rebuild at the moment,
-    # so this is good for now.
-    mv ${ROOT_BUILD_DIR}/set_prompt.bash  ${SINGULARITY_ROOTFS}{{ SET_PROMPT_SCRIPT }}
-    cat <<'EOF' > ${SINGULARITY_ROOTFS}/.singularity.d/env/99-zzz_prompt.sh
-source {{ SET_PROMPT_SCRIPT }}
-set_prompt
-EOF
-
     # Pinned versions
     MINIFORGE_VERSION=23.3.1-1
     PYTHON_VERSION=3.10
@@ -154,7 +142,6 @@ EOF
     rm -rf ${ROOT_BUILD_DIR}
 
 %files
-    {{ MOOSE_DIR }}/apptainer/set_prompt.bash {{ ROOT_BUILD_DIR }}/set_prompt.bash
     {{ MOOSE_DIR }}/scripts/{{ WASP_BUILD_SCRIPT }} {{ ROOT_BUILD_DIR }}/{{ WASP_BUILD_SCRIPT }}
     {{ MOOSE_DIR }}/scripts/configure_wasp.sh {{ ROOT_BUILD_DIR }}/configure_wasp.sh
 {%- if WITH_LIBTORCH %}

diff --git a/apptainer/mpi.def b/apptainer/mpi.def
@@ -1,5 +1,5 @@
 {#- Required jinja arguments                                                                  -#}
-{#- ARCH: The machine arch                                                                    -#}
+{#- MOOSE_DIR: Path on the host to the MOOSE repository                                       -#}
 
 {#- MOOSE_JOBS: Number of jobs to pass to the builds                                          -#}
 {#- Optional jinja arguments                                                                  -#}
@@ -8,6 +8,12 @@
 {#- The minimum version of LLVM/Clang                                                         -#}
 {%- set MIN_CLANG_VERSION = '10.0.1' -%}
 
+{#- The within-container build directory to use                                               -#}
+{%- set ROOT_BUILD_DIR = '/root/build' -%}
+
+{#- The location for the bash prompt script in the container                                  -#}
+{%- set SET_PROMPT_SCRIPT = '/.singularity.d/set_prompt.bash' -%}
+
 {#- The system GCC version based on ALTERNATE_FROM clang or clang_min                         -#}
 {#- Note: harmless if not set                                                                 -#}
 {%- if ALTERNATE_FROM == "clang" %}
@@ -20,10 +26,10 @@
 
 {%- if ALTERNATE_FROM == "gcc_min" or ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang_min" %}
 Bootstrap: docker
-From: rockylinux:8.8.20230518
+From: rockylinux:8.9.20231119
 {%- else %}
 BootStrap: oras
-From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky-x86_64:8.8-0
+From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky-x86_64:8.9-0
 {%- endif %}
 
 %environment
@@ -40,7 +46,32 @@ From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky-x86_64:8.8-0
     export LD_LIBRARY_PATH=/opt/llvm/lib:${LD_LIBRARY_PATH}
 {%- endif %}
 
+    # Load a custom environment if it exists
+    if [ -e /init_env ]; then
+        source /init_env
+    fi
+
 %post
+    umask 022
+
+    # Load jinja vars
+    ROOT_BUILD_DIR={{ ROOT_BUILD_DIR }}
+
+    # Setup the build directory
+    mkdir ${ROOT_BUILD_DIR}
+
+    # Setup the custom prompt. The apptainer generator script will append
+    # extra variables to the %environment step that identify what container
+    # is currently being used
+    # Eventually, this will go in the base container as far down as
+    # we can. But we don't want to do a full rebuild at the moment,
+    # so this is good for now.
+    cat <<'EOF' > ${SINGULARITY_ROOTFS}/.singularity.d/env/99-zzz_prompt.sh
+source {{ SET_PROMPT_SCRIPT }}
+set_prompt
+EOF
+    chmod 755 ${SINGULARITY_ROOTFS}{{ SET_PROMPT_SCRIPT }}
+
     export MOOSE_JOBS={{ MOOSE_JOBS or "1" }}
 
     # Install prefix for mpich
@@ -59,10 +90,6 @@ From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky-x86_64:8.8-0
     export MIN_CLANG_VERSION={{ MIN_CLANG_VERSION }}
 {%- endif %}
 
-    # Prepare a temp directory
-    TEMP_LOC=/root/build
-    mkdir ${TEMP_LOC}
-
     # Enable power tools
     dnf install -y dnf-plugins-core
     dnf config-manager --set-enabled powertools
@@ -107,54 +134,17 @@ From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky-x86_64:8.8-0
     alternatives --set python /usr/bin/python3.9
     alternatives --set python3 /usr/bin/python3.9
 
-    # Allow a custom initialization routine if the following files exist at /
-    touch ${SINGULARITY_ROOTFS}/none
-    cat <<'EOF' >> ${SINGULARITY_ROOTFS}/.singularity.d/env/90-environment.sh
-# If discovered, use these files as initialization routines
-if [ -f /init_env ]; then
-  file="/init_env"
-else
-  file="/none"
-fi
-action="${0##*/}"
-case "${action}" in
-shell)
-    if [ "${SINGULARITY_SHELL:-}" = "/bin/bash" ]; then
-        set -- --noprofile --rcfile $file
-    elif test -z "${SINGULARITY_SHELL:-}"; then
-        export SINGULARITY_SHELL=/bin/bash
-        set -- --noprofile --rcfile $file
-    fi
-    ;;
-exec)
-    export BASH_ENV="$file"
-    set -- /bin/bash --noprofile --rcfile $file -c "$*"
-    ;;
-run)
-    set -- /bin/bash --noprofile --rcfile $file
-esac
-EOF
-
-    # Create a pretty prompt (ensure last item sourced)
-    cat <<'EOF' > ${SINGULARITY_ROOTFS}/.singularity.d/env/99-zzz_prompt.sh
-if [ -n "${CUSTOM_PROMPT}" ]; then
-    PS1=${CUSTOM_PROMPT}
-else
-    PS1="\[\033[1;34m\][`basename ${APPTAINER_NAME:-$SINGULARITY_NAME} .sif`]\[\033[1;32m\][\w]\[\033[0m\]> "
-fi
-EOF
-
     # We need git lfs
-    mkdir ${TEMP_LOC}/gitlfs
-    cd ${TEMP_LOC}/gitlfs
+    mkdir ${ROOT_BUILD_DIR}/gitlfs
+    cd ${ROOT_BUILD_DIR}/gitlfs
     curl -L -O https://github.com/git-lfs/git-lfs/releases/download/v3.2.0/git-lfs-linux-amd64-v3.2.0.tar.gz
     tar -xf git-lfs-linux-amd64-v3.2.0.tar.gz
     cd git-lfs-3.2.0
     ./install.sh
 
 {%- if ALTERNATE_FROM == "clang_min" %}
     # Build minimum Clang {{ MIN_CLANG_VERSION }} from source
-    cd ${TEMP_LOC}
+    cd ${ROOT_BUILD_DIR}
     git clone --depth 1 --branch llvmorg-${MIN_CLANG_VERSION} https://github.com/llvm/llvm-project
     cd llvm-project
     mkdir llvm-build
@@ -180,8 +170,8 @@ EOF
 
 {%- if ALTERNATE_FROM == "gcc_min" or ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang_min" %}
     # Build and install MPICH
-    mkdir ${TEMP_LOC}/mpich
-    cd ${TEMP_LOC}/mpich
+    mkdir ${ROOT_BUILD_DIR}/mpich
+    cd ${ROOT_BUILD_DIR}/mpich
     curl -L -O http://www.mpich.org/static/downloads/4.1.2/mpich-4.1.2.tar.gz
     tar -xf mpich-4.1.2.tar.gz
     mkdir mpich-4.1.2/build
@@ -232,5 +222,8 @@ EOF
 {%- endif %}
 
     # Clean Up
-    rm -rf ${TEMP_LOC}
+    rm -rf ${ROOT_BUILD_DIR}
     dnf clean all
+
+%files
+    {{ MOOSE_DIR }}/apptainer/set_prompt.bash {{ SET_PROMPT_SCRIPT }}
diff --git a/apptainer/petsc.def b/apptainer/petsc.def
@@ -5,7 +5,6 @@
 {#- PETSC_GIT_REMOTE: The git remote to use to get PETSc                                      -#}
 
 {#- Optional jinja arguments                                                                  -#}
-{#- PETSC_ALT: Set to anything to use the alternate version of PETSc                          -#}
 {#- PETSC_OPTIONS: Options to pass to the PETSc build script                                  -#}
 {#- MOOSE_JOBS: Number of jobs to pass to the PETSc build script                              -#}
 {#- MPI_FLAVOUR: The flavour of MPI to use (options: mpich, openmpi; default: mpich)          -#}
@@ -20,12 +19,6 @@
 {%- set MPI_FLAVOUR = 'mpich' -%}
 {%- endif %}
 
-{%- if PETSC_ALT is defined %}
-{%- set PETSC_BUILD_SCRIPT = 'update_and_rebuild_petsc_alt.sh' -%}
-{%- else -%}
-{%- set PETSC_BUILD_SCRIPT = 'update_and_rebuild_petsc.sh' -%}
-{%- endif %}
-
 BootStrap: {{ APPTAINER_BOOTSTRAP }}
 From: {{ APPTAINER_FROM }}
 
@@ -41,11 +34,15 @@ From: {{ APPTAINER_FROM }}
     export PETSC_DIR={{ PETSC_DIR }}
 
 %post
+    umask 022
+
     # Set the MPI environment
 {%- if MPI_FLAVOUR == "mpich" %}
     source /opt/mpi/use-mpich
+    rm -rf /opt/openmpi /opt/mpi/use-openmpi
 {%- elif MPI_FLAVOUR == "openmpi" %}
     source /opt/mpi/use-openmpi
+    rm -rf /opt/mpich /opt/mpi/use-mpich
 {%- endif %}
 
     # Load jinja vars
@@ -61,28 +58,23 @@ From: {{ APPTAINER_FROM }}
     cd ${PETSC_SRC_DIR}
     git checkout ${PETSC_GIT_SHA}
 
-    PETSC_OPTIONS+=" --skip-submodule-update"
-{%- if PETSC_ALT is defined %}
-    PETSC_BUILD_SCRIPT=${ROOT_BUILD_DIR}/scripts/update_and_rebuild_petsc_alt.sh
-{%- else %}
-    # Need a newer cmake for current petsc
-    PETSC_OPTIONS+=" --download-cmake"
-    PETSC_BUILD_SCRIPT=${ROOT_BUILD_DIR}/scripts/update_and_rebuild_petsc.sh
-{%- endif %}
-
     # Build PETSc
     umask 022
     cd ${ROOT_BUILD_DIR}
-    MOOSE_JOBS={{ MOOSE_JOBS }} PETSC_PREFIX=${PETSC_DIR} PETSC_SRC_DIR=${PETSC_SRC_DIR} ${PETSC_BUILD_SCRIPT} ${PETSC_OPTIONS}
+    MOOSE_JOBS={{ MOOSE_JOBS }} PETSC_PREFIX=${PETSC_DIR} PETSC_SRC_DIR=${PETSC_SRC_DIR} ${ROOT_BUILD_DIR}/scripts/update_and_rebuild_petsc.sh --skip-submodule-update${PETSC_OPTIONS}
+
+{%- if MPI_FLAVOUR == "openmpi" %}
+    # If we're using openmpi, we need to allow running as root
+    # if we're in a fakeroot environment
+    if [ $(whoami) == root ]; then
+        export OMPI_ALLOW_RUN_AS_ROOT=1
+        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+    fi
+{%- endif %}
 
-{%- if MPI_FLAVOUR != "openmpi" %}
     # Test PETSc
     cd petsc
     make SLEPC_DIR=${PETSC_DIR} PETSC_DIR=${PETSC_DIR} PETSC_ARCH= check
-{%- endif %}
-
-    # Fix possibly bad permissions
-    chmod -R o=u-w,g=u-w ${PETSC_DIR}
 
     # Clean Up
     rm -rf ${ROOT_BUILD_DIR}
@@ -91,6 +83,3 @@ From: {{ APPTAINER_FROM }}
     {{ MOOSE_DIR }}/scripts/configure_petsc.sh {{ ROOT_BUILD_DIR }}/scripts/configure_petsc.sh
     {{ MOOSE_DIR }}/scripts/update_and_rebuild_petsc.sh {{ ROOT_BUILD_DIR }}/scripts/update_and_rebuild_petsc.sh
     {{ MOOSE_DIR }}/scripts/tao_restore_viewer.patch {{ ROOT_BUILD_DIR }}/scripts/tao_restore_viewer.patch
-{%- if PETSC_ALT is defined %}
-    {{ MOOSE_DIR }}/scripts/update_and_rebuild_petsc_alt.sh {{ ROOT_BUILD_DIR }}/scripts/update_and_rebuild_petsc_alt.sh
-{%- endif %}
diff --git a/conda/libmesh-vtk/conda_build_config.yaml b/conda/libmesh-vtk/conda_build_config.yaml
@@ -3,7 +3,7 @@ mpi:
   - openmpi
 
 moose_mpi:
-  - moose-mpi 2024.08.05
+  - moose-mpi 2024.08.12
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/conda/libmesh-vtk/meta.yaml b/conda/libmesh-vtk/meta.yaml
@@ -3,7 +3,7 @@
 #   libmesh/*
 #
 # As well as any directions pertaining to modifying those files.
-{% set build = 1 %}
+{% set build = 2 %}
 {% set vtk_version = "9.3.0" %}
 {% set vtk_friendly_version = "9.3" %}
 {% set sha256 = "fdc7b9295225b34e4fdddc49cd06e66e94260cb00efee456e0f66568c9681be9" %}

diff --git a/conda/libmesh/conda_build_config.yaml b/conda/libmesh/conda_build_config.yaml
@@ -3,12 +3,12 @@ mpi:
   - openmpi
 
 moose_petsc:
-  - moose-petsc 3.21.4 mpich_0
-  - moose-petsc 3.21.4 openmpi_0
+  - moose-petsc 3.21.4 mpich_1
+  - moose-petsc 3.21.4 openmpi_1
 
 moose_libmesh_vtk:
-  - moose-libmesh-vtk 9.3.0 mpich_1
-  - moose-libmesh-vtk 9.3.0 openmpi_1
+  - moose-libmesh-vtk 9.3.0 mpich_2
+  - moose-libmesh-vtk 9.3.0 openmpi_2
 
 zip_keys:
   - mpi

diff --git a/conda/libmesh/meta.yaml b/conda/libmesh/meta.yaml
@@ -4,7 +4,7 @@
 #
 # As well as any directions pertaining to modifying those files.
 # ALSO: Follow the directions in scripts/tests/versioner_hashes.yaml
-{% set build = 1 %}
+{% set build = 2 %}
 {% set version = "2024.07.27" %}
 
 package:

diff --git a/conda/moose-dev/conda_build_config.yaml b/conda/moose-dev/conda_build_config.yaml
@@ -3,8 +3,8 @@ mpi:
   - openmpi
 
 moose_libmesh:
-  - moose-libmesh 2024.07.27 mpich_1
-  - moose-libmesh 2024.07.27 openmpi_1
+  - moose-libmesh 2024.07.27 mpich_2
+  - moose-libmesh 2024.07.27 openmpi_2
 
 moose_wasp:
   - moose-wasp 2024.05.08
@@ -13,7 +13,7 @@ moose_tools:
   - moose-tools 2024.07.19
 
 moose_peacock:
-  - moose-peacock 2024.08.05
+  - moose-peacock 2024.08.12
 
 zip_keys:
   - mpi

diff --git a/conda/moose-dev/meta.yaml b/conda/moose-dev/meta.yaml
@@ -6,7 +6,7 @@
 #   framework/doc/packages_config.yml
 #
 # As well as any directions pertaining to modifying those files.
-{% set version = "2024.08.05" %}
+{% set version = "2024.08.12" %}
 
 package:
   name: moose-dev

diff --git a/conda/moose/conda_build_config.yaml b/conda/moose/conda_build_config.yaml
@@ -3,7 +3,7 @@ mpi:
   - openmpi
 
 moose_dev:
-  - moose-dev 2024.08.05
+  - moose-dev 2024.08.12
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/conda/mpi/meta.yaml b/conda/mpi/meta.yaml
@@ -6,7 +6,7 @@
 #
 # As well as any directions pertaining to modifying those files.
 # ALSO: Follow the directions in scripts/tests/versioner_hashes.yaml
-{% set version = "2024.08.05" %}
+{% set version = "2024.08.12" %}
 
 # HDF5 Version
 {% set hdf5_version = "1.14.3" %}

diff --git a/conda/peacock/conda_build_config.yaml b/conda/peacock/conda_build_config.yaml
@@ -3,7 +3,7 @@ mpi:
   - openmpi
 
 moose_mpi:
-  - moose-mpi 2024.08.05
+  - moose-mpi 2024.08.12
 
 # Pesky packages that break internal CI
 # Note: Modifying/Updating this will require changes to conda/mpich!