From 0739c2f03da6c659a796a46307980ffab65e2270 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 29 May 2024 19:45:55 -0600 Subject: [PATCH 01/28] Add Cardinal submodule Refs #38 --- .gitmodules | 3 +++ cardinal | 1 + 2 files changed, 4 insertions(+) create mode 160000 cardinal diff --git a/.gitmodules b/.gitmodules index 422c4c68..32858488 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "tmap8"] path = tmap8 url = ../../idaholab/tmap8.git +[submodule "cardinal"] + path = cardinal + url = ../../neams-th-coe/cardinal.git diff --git a/cardinal b/cardinal new file mode 160000 index 00000000..f5dc4046 --- /dev/null +++ b/cardinal @@ -0,0 +1 @@ +Subproject commit f5dc40461febecf38e1a95c146baebafcbf46ab7 From 475fc17ad5659a7a2b01cffe42993297ed5481fc Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Thu, 27 Jun 2024 11:23:29 -0600 Subject: [PATCH 02/28] Add Cardinal to Makefile This enables linking to the executable, but object registration is not done yet --- Makefile | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f527e11b..f1c6db6f 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,16 @@ # Optional Environment variables # MOOSE_DIR - Root directory of the MOOSE project # TMAP8_DIR - Root directory of the TMAP8 project +# CARDINAL_DIR - Root directory of the Cardinal project # ############################################################################### # Use the MOOSE submodule if it exists and MOOSE_DIR is not set -MOOSE_SUBMODULE := $(CURDIR)/moose +# If it doesn't exist, and MOOSE_DIR is not set, then look for it adjacent to the application +MOOSE_SUBMODULE := $(CURDIR)/moose ifneq ($(wildcard $(MOOSE_SUBMODULE)/framework/Makefile),) - MOOSE_DIR ?= $(MOOSE_SUBMODULE) + MOOSE_DIR ?= $(MOOSE_SUBMODULE) else - MOOSE_DIR ?= $(shell dirname `pwd`)/moose + MOOSE_DIR ?= $(shell dirname `pwd`)/moose endif # Use the TMAP8 submodule if it exists and TMAP8_DIR is not set @@ -24,6 +26,18 @@ else TMAP8_DIR ?= $(shell dirname `pwd`)/tmap8 endif +# Use the Cardinal submodule if it exists and CARDINAL_DIR is not set +# If it doesn't exist, and CARDINAL_DIR is not set, then look for it adjacent to the application +CARDINAL_SUBMODULE := $(CURDIR)/cardinal +ifneq ($(wildcard $(CARDINAL_SUBMODULE)/Makefile),) + CARDINAL_DIR ?= $(CARDINAL_SUBMODULE) +else + CARDINAL_DIR ?= $(shell dirname `pwd`)/cardinal +endif +CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib + +OPENMC_DIR ?= ${CARDINAL_CONTRIB_DIR}/openmc + # framework FRAMEWORK_DIR := $(MOOSE_DIR)/framework include $(FRAMEWORK_DIR)/build.mk @@ -73,6 +87,13 @@ BUILD_EXEC := no GEN_REVISION := no include $(FRAMEWORK_DIR)/app.mk +# Cardinal +APPLICATION_DIR := $(CARDINAL_DIR) +APPLICATION_NAME := cardinal +BUILD_EXEC := no +GEN_REVISION := no +include $(FRAMEWORK_DIR)/app.mk + # dep apps APPLICATION_DIR := $(CURDIR) APPLICATION_NAME := fenix From 288ee680788bc52fd097cf13e343c196e90fe6f9 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Thu, 27 Jun 2024 11:23:43 -0600 Subject: [PATCH 03/28] Update Cardinal submodule --- cardinal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cardinal b/cardinal index f5dc4046..935473ff 160000 --- a/cardinal +++ b/cardinal @@ -1 +1 @@ -Subproject commit f5dc40461febecf38e1a95c146baebafcbf46ab7 +Subproject commit 935473ff564b34ef20ab9be118ab6bec4328ec18 From ad7d1f6fdb954063d07ffb32af0d2dff6a8b360f Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 3 Jul 2024 16:42:28 -0600 Subject: [PATCH 04/28] Get Cardinal and contribs (OpenMC, DAGMC, MOAB) fully linking to FENIX Still need to somewhat automate build process for FENIX end users (via a script). --- Makefile | 17 +++++++++++++++-- src/base/FenixApp.C | 3 +++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f1c6db6f..e2540825 100644 --- a/Makefile +++ b/Makefile @@ -34,9 +34,12 @@ ifneq ($(wildcard $(CARDINAL_SUBMODULE)/Makefile),) else CARDINAL_DIR ?= $(shell dirname `pwd`)/cardinal endif -CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib -OPENMC_DIR ?= ${CARDINAL_CONTRIB_DIR}/openmc +# Cardinal contrib variables +CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib +CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install +OPENMC_INSTALL_DIR := $(CONTRIB_INSTALL_DIR) +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib # framework FRAMEWORK_DIR := $(MOOSE_DIR)/framework @@ -99,7 +102,17 @@ APPLICATION_DIR := $(CURDIR) APPLICATION_NAME := fenix BUILD_EXEC := yes GEN_REVISION := no + +# Cardinal dependency libraries +ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB + include $(FRAMEWORK_DIR)/app.mk ############################################################################### # Additional special case targets should be added here + +CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib -L$(OPENMC_LIBDIR) \ + -Wl,-rpath,$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB + +# EXTERNAL_FLAGS is used in rules for app.mk +$(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) diff --git a/src/base/FenixApp.C b/src/base/FenixApp.C index c03015f2..3ba1c04c 100644 --- a/src/base/FenixApp.C +++ b/src/base/FenixApp.C @@ -21,6 +21,7 @@ // contribs #include "TMAP8App.h" +#include "CardinalApp.h" InputParameters FenixApp::validParams() @@ -46,6 +47,7 @@ FenixApp::registerAll(Factory & f, ActionFactory & af, Syntax & s) Registry::registerActionsTo(af, {"FenixApp"}); TMAP8App::registerAll(f, af, s); + CardinalApp::registerAll(f, af, s); /* register custom execute flags, action syntax, etc. here */ } @@ -56,6 +58,7 @@ FenixApp::registerApps() registerApp(FenixApp); ModulesApp::registerApps(); TMAP8App::registerApps(); + CardinalApp::registerApps(); } /*************************************************************************************************** From 9a675553d9ac5e963b02e87c65e91266397f2872 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 3 Jul 2024 16:46:23 -0600 Subject: [PATCH 05/28] Add OpenMC download location in FENIX to gitignore file --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5f8cae72..daa60eb8 100644 --- a/.gitignore +++ b/.gitignore @@ -325,3 +325,6 @@ fenix-unit.yaml /modules/fsi/unit/fsi-unit.yaml /tutorials/tutorial01_app_development/*/babbler.yaml /tutorials/darcy_thermo_mech/*/darcy_thermo_mech.yaml + +# Cardinal +cross_sections From 40b90db307606c62c0bd27fca33fb5e69dce8bbc Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Fri, 5 Jul 2024 11:10:02 -0600 Subject: [PATCH 06/28] Add initial cardinal build script --- scripts/update_and_rebuild_cardinal.sh | 139 +++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100755 scripts/update_and_rebuild_cardinal.sh diff --git a/scripts/update_and_rebuild_cardinal.sh b/scripts/update_and_rebuild_cardinal.sh new file mode 100755 index 00000000..a83f5a7e --- /dev/null +++ b/scripts/update_and_rebuild_cardinal.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +#* This file is part of FENIX: Fusion ENergy Integrated multiphys-X, +#* A multiphysics application for modeling plasma facing components +#* https://github.com/idaholab/fenix +#* https://mooseframework.inl.gov/fenix +#* +#* FENIX is powered by the MOOSE Framework +#* https://www.mooseframework.inl.gov +#* +#* Licensed under LGPL 2.1, please see LICENSE for details +#* https://www.gnu.org/licenses/lgpl-2.1.html +#* +#* Copyright 2024, Battelle Energy Alliance, LLC +#* ALL RIGHTS RESERVED +#* + +# Colors for bash output +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Set various flags based on argument input (fast, help, skip update, etc.) +for i in "$@" +do + shift + if [ "$i" == "--fast" ]; then + go_fast=1; + fi + + if [[ "$i" == "-h" || "$i" == "--help" ]]; then + help=1; + fi + + if [ "$i" == "--skip-submodule-update" ]; then + skip_sub_update=1; + else # Remove the skip submodule update argument before passing further + set -- "$@" "$i" + fi +done + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo "Running ${SCRIPT_DIR}/update_and_rebuild_fenix.sh..." +FENIX_DIR=$(realpath $SCRIPT_DIR/..) + +if [ -n "$MOOSE_DIR" ]; then + echo "INFO: MOOSE_DIR set - ${MOOSE_DIR} will be used for both FENIX and Cardinal..." +else + export MOOSE_DIR=${FENIX_DIR}/moose +fi + +if [ -n "$CARDINAL_DIR" ]; then + echo "INFO: CARDINAL_DIR set - using ${CARDINAL_DIR} and skipping submodule update..." + skip_sub_update=1 +else + CARDINAL_DIR=${FENIX_DIR}/cardinal +fi + +# Checking for HDF5 using HDF5_DIR (set via conda automatically). If this is not set, +# throw an error code and request that the user set it! +if [ -n "$HDF5_DIR" ]; then + export HDF5_ROOT=$HDF5_DIR +else + echo -e "${RED}ERROR: HDF5 not found; please set HDF5_DIR in your environment!${NC}" + exit 1 +fi + + +# Display help if requested +if [[ -n "$help" ]]; then + echo "Usage: $0 [-h | --help | --fast | --skip-submodule-update ]" + echo + echo "-h | --help Display this message" + echo "--fast Run Cardinal 'make' only, do NOT update Cardinal contribs" + echo "--skip-submodule-update Do not update the Cardinal submodule, use the current version" + echo "*************************************************************************************" + echo "" + + exit 0 +fi + +# Print information to the screen if fast is used with any other arguments other than skip-submodule-update +if [[ -n "$go_fast" && $# != 1 ]]; then + echo -e "${RED}ERROR: --fast can only be used by itself or with --skip-submodule-update." + echo -e "Try again, removing either --fast or all other conflicting arguments!${NC}" + exit 1; +fi + +# Update Cardinal submodule if we're not skipping +if [ -z "$skip_sub_update" ]; then + cd ${FENIX_DIR} + git_dir=`git rev-parse --show-cdup 2>/dev/null` + if [[ -z "$go_fast" && $? == 0 && "x$git_dir" == "x" ]]; then + git submodule update --init cardinal + if [[ $? != 0 ]]; then + echo "git submodule command failed, are your proxy settings correct?" + exit 1 + fi + fi +fi + +cd ${CARDINAL_DIR} + +# Enable/Disable Cardinal contribs +export ENABLE_NEK=no +export ENABLE_DAGMC=yes # Also enables MOAB + +# If we're not going fast, update all dependencies +if [ -z "$go_fast" ]; then + echo "INFO: Updating OpenMC, DAGMC, and MOAB submodules..." + git submodule update --init --recursive contrib/openmc + git submodule update --init contrib/DAGMC + git submodule update --init contrib/moab +fi + +if [[ -z "${OPENMC_CROSS_SECTIONS}" ]]; then + echo -e "${YELLOW}" + echo "If you are using OpenMC, remember that you need to set OPENMC_CROSS_SECTIONS to point to a cross_sections.xml file!" + echo "To get the ENDF/b-7.1 dataset, please run:" + echo "" + echo "$CARDINAL_DIR/scripts/download-openmc-cross-sections.sh" + echo -e "${NC}" +fi + +# If using conda, set LD_LIBRARY_PATH to workaround linking issue for some systems. +# Cardinal dev team is working on a more elegant fix, and this should be removed +# when that occurs. +if [ -n "$CONDA_PREFIX" ]; then + export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH +fi + +# Run make, letting the number of jobs be either MOOSE_JOBS or 1 if MOOSE_JOBS is not set. +# MAKEFLAGS is used to make sure that CMake within Cardinal also uses the appropriate +# number of jobs. +make -j ${MOOSE_JOBS:-1} MAKEFLAGS=-j${MOOSE_JOBS:-1} + +# Remove cardinal executable, not required for FENIX build. +echo -e "${YELLOW}WARNING: Removing cardinal-${METHOD:-opt} since FENIX will be built." +echo -e "If a Cardinal executable is desired, please navigate to ${CARDINAL_DIR} and 'make' again.${NC}" +rm cardinal-"${METHOD:-opt}" From 049a7102e3b74391044f207c13eb3d2bbf468097 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Fri, 5 Jul 2024 12:08:47 -0600 Subject: [PATCH 07/28] Add Cardinal to installation instructions Refs #38 --- doc/content/getting_started/installation.md | 25 +++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index dd68bfb7..b73ed696 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -67,14 +67,35 @@ git submodule update --init moose git submodule update --init tmap8 ``` +Finally, Cardinal and its dependencies must be downloaded and pre-built. To perform this, first run +the build script from the FENIX clone directory: + +```bash +scripts/update_and_rebuild_cardinal.sh +``` + +Then, to download OpenMC cross sections needed for OpenMC-based Cardinal runs, run: + +```bash +cd ~/projects/FENIX/cardinal +scripts/download-openmc-cross-sections.sh +``` + +and subsequently set the location of those cross sections in your environment: + +```bash +export OPENMC_CROSS_SECTIONS=~/projects/FENIX/cross_sections/cross_sections.xml +``` + !alert! note -The copies of MOOSE and TMAP8 provided with FENIX have been fully tested against the current +The copies of MOOSE, TMAP8, and Cardinal provided with FENIX have been fully tested against the current FENIX version, and is guaranteed to work with all current FENIX tests. !alert-end! Once all dependencies have been downloaded, FENIX can be compiled and tested: ```bash +cd ~/projects/FENIX make -j8 ./run_tests -j8 ``` @@ -90,7 +111,7 @@ FENIX is ready to be used and further developed. ## Update FENIX -FENIX (and its underlying dependencies MOOSE and TMAP8) is under heavy development and is +FENIX (and its underlying dependencies MOOSE, TMAP8, and Cardinal) is under heavy development and is updated on a continuous basis. Therefore, it is important that the local copy of FENIX be periodically updated to obtain new capabilities, improvements, and bugfixes. Weekly updates are recommended as, at minimum, the MOOSE submodule within FENIX is updated up to several times a week. From d35dc40af00323107b1683b6680dbe7b88a51e7c Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Sat, 6 Jul 2024 12:31:55 -0600 Subject: [PATCH 08/28] Address initial review comments --- doc/content/getting_started/installation.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index b73ed696..3a87bf78 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -71,6 +71,7 @@ Finally, Cardinal and its dependencies must be downloaded and pre-built. To perf the build script from the FENIX clone directory: ```bash +cd ~/projects/FENIX scripts/update_and_rebuild_cardinal.sh ``` @@ -87,6 +88,12 @@ and subsequently set the location of those cross sections in your environment: export OPENMC_CROSS_SECTIONS=~/projects/FENIX/cross_sections/cross_sections.xml ``` +!alert! warning +This variable +must+ be set in your environment anytime you wish to run FENIX input files that +utilize the OpenMC functionality! This can be done either using `export` on the command line, +or placing this command within a shell config file (`.bashrc`, `.zshrc`, etc.). +!alert-end! + !alert! note The copies of MOOSE, TMAP8, and Cardinal provided with FENIX have been fully tested against the current FENIX version, and is guaranteed to work with all current FENIX tests. From db482b6efcc8591219201d5b8c4da4e65dedd24e Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Sat, 6 Jul 2024 14:41:03 -0600 Subject: [PATCH 09/28] Add new content to documentation - Adds Cardinal content - Excludes Nek-related content - Adds landing page for BulkEnergyConservationICAction - Adds landing page for VolumetricHeatSourceICAction - Removes excess syntax information for both IC actions - Adds landing page for Cardinal/ICs custom syntax - Adds module content - Add SQA dependencies and SQA requirement content for modules and apps Refs #38 and #52 --- doc/cardinal_nek_exclude.yml | 35 +++++++++++++++++++ doc/config.yml | 9 +++++ .../ICs/BulkEnergyConservation/index.md | 1 + .../ICs/VolumetricHeatSource/index.md | 1 + doc/content/syntax/Cardinal/ICs/index.md | 1 + doc/content/syntax/Cardinal/index.md | 5 +++ doc/sqa_fenix.yml | 1 + doc/sqa_reports.yml | 2 ++ 8 files changed, 55 insertions(+) create mode 100644 doc/cardinal_nek_exclude.yml create mode 100644 doc/content/syntax/Cardinal/ICs/BulkEnergyConservation/index.md create mode 100644 doc/content/syntax/Cardinal/ICs/VolumetricHeatSource/index.md create mode 100644 doc/content/syntax/Cardinal/ICs/index.md create mode 100644 doc/content/syntax/Cardinal/index.md diff --git a/doc/cardinal_nek_exclude.yml b/doc/cardinal_nek_exclude.yml new file mode 100644 index 00000000..eb62a902 --- /dev/null +++ b/doc/cardinal_nek_exclude.yml @@ -0,0 +1,35 @@ +- source/** +- media/** +- ~source/auxkernels/NekSpatialBinComponentAux.md +- ~source/timesteppers/NekTimeStepper.md +- ~source/mesh/NekRSMesh.md +- ~source/postprocessors/boundary_specs.md +- ~source/postprocessors/NekHeatFluxIntegral.md +- ~source/postprocessors/NekNumRanks.md +- ~source/postprocessors/NekMassFluxWeightedSideAverage.md +- ~source/postprocessors/NekMassFluxWeightedSideIntegral.md +- ~source/postprocessors/NekPressureSurfaceForce.md +- ~source/postprocessors/NekScalarValuePostprocessor.md +- ~source/postprocessors/NekSideAverage.md +- ~source/postprocessors/NekSideExtremeValue.md +- ~source/postprocessors/NekSideIntegral.md +- ~source/postprocessors/NekUsrWrkBoundaryIntegral.md +- ~source/postprocessors/NekVolumeAverage.md +- ~source/postprocessors/NekVolumeExtremeValue.md +- ~source/postprocessors/NekVolumeIntegral.md +- ~source/postprocessors/PecletNumber.md +- ~source/postprocessors/ReynoldsNumber.md +- ~source/problems/NekRSProblem.md +- ~source/problems/NekRSSeparateDomainProblem.md +- ~source/problems/nek_classes.md +- ~source/problems/nondimensional_problem.md +- ~source/problems/output_solution.md +- ~source/problems/NekRSStandaloneProblem.md +- ~source/userobjects/NekBinnedPlaneAverage.md +- ~source/userobjects/NekBinnedPlaneIntegral.md +- ~source/userobjects/NekBinnedSideAverage.md +- ~source/userobjects/NekBinnedSideIntegral.md +- ~source/userobjects/NekBinnedVolumeAverage.md +- ~source/userobjects/NekBinnedVolumeIntegral.md +- ~source/userobjects/NekScalarValue.md +- ~source/userobjects/velocity_component.md diff --git a/doc/config.yml b/doc/config.yml index e33214a5..19c23a22 100644 --- a/doc/config.yml +++ b/doc/config.yml @@ -54,6 +54,9 @@ Content: - python/TestHarness.md tmap8: root_dir: ${ROOT_DIR}/tmap8/doc/content + cardinal: + root_dir: ${ROOT_DIR}/cardinal/doc/content + content: !include ${ROOT_DIR}/doc/cardinal_nek_exclude.yml Renderer: type: MooseDocs.base.MaterializeRenderer @@ -86,11 +89,16 @@ Extensions: executable: ${ROOT_DIR} remove: framework: !include ${MOOSE_DIR}/framework/doc/remove.yml + cardinal: + - /ICs/BulkEnergyConservationIC + - /Cardinal/ICs/BulkEnergyConservation/BulkEnergyConservationICAction + - /Cardinal/ICs/VolumetricHeatSource/VolumetricHeatSourceICAction MooseDocs.extensions.common: shortcuts: !include ${MOOSE_DIR}/framework/doc/globals.yml MooseDocs.extensions.acronym: acronyms: framework: !include ${MOOSE_DIR}/framework/doc/acronyms.yml + cardinal: !include ${ROOT_DIR}/cardinal/doc/acronyms.yml fenix: FENIX: Fusion ENergy Integrated multiphys-X PIC: particle-in-cell @@ -117,6 +125,7 @@ Extensions: stochastic_tools: !include ${MOOSE_DIR}/modules/stochastic_tools/doc/sqa_stochastic_tools.yml thermal_hydraulics: !include ${MOOSE_DIR}/modules/thermal_hydraulics/doc/sqa_thermal_hydraulics.yml tmap8: !include ${ROOT_DIR}/tmap8/doc/sqa_tmap8.yml + #cardinal: !include ${ROOT_DIR}/cardinal/doc/sqa_cardinal.yml reports: !include ${ROOT_DIR}/doc/sqa_reports.yml repos: default: https://github.com/idaholab/fenix diff --git a/doc/content/syntax/Cardinal/ICs/BulkEnergyConservation/index.md b/doc/content/syntax/Cardinal/ICs/BulkEnergyConservation/index.md new file mode 100644 index 00000000..a659921e --- /dev/null +++ b/doc/content/syntax/Cardinal/ICs/BulkEnergyConservation/index.md @@ -0,0 +1 @@ +!include BulkEnergyConservationIC.md diff --git a/doc/content/syntax/Cardinal/ICs/VolumetricHeatSource/index.md b/doc/content/syntax/Cardinal/ICs/VolumetricHeatSource/index.md new file mode 100644 index 00000000..520f9640 --- /dev/null +++ b/doc/content/syntax/Cardinal/ICs/VolumetricHeatSource/index.md @@ -0,0 +1 @@ +!include VolumetricHeatSourceICAction.md diff --git a/doc/content/syntax/Cardinal/ICs/index.md b/doc/content/syntax/Cardinal/ICs/index.md new file mode 100644 index 00000000..7c20d2fa --- /dev/null +++ b/doc/content/syntax/Cardinal/ICs/index.md @@ -0,0 +1 @@ +!include syntax/Cardinal/index.md diff --git a/doc/content/syntax/Cardinal/index.md b/doc/content/syntax/Cardinal/index.md new file mode 100644 index 00000000..9ba3f9c1 --- /dev/null +++ b/doc/content/syntax/Cardinal/index.md @@ -0,0 +1,5 @@ +# Custom Cardinal Syntax + +Custom block syntax is used for both of the [VolumetricHeatSourceICAction.md] and +[BulkEnergyConservationIC.md] objects. Examples of the usage of this syntax can +be found in either of the linked pages. diff --git a/doc/sqa_fenix.yml b/doc/sqa_fenix.yml index aabc574f..3c6ea811 100644 --- a/doc/sqa_fenix.yml +++ b/doc/sqa_fenix.yml @@ -20,4 +20,5 @@ dependencies: # commented-out categories are not available - stochastic_tools - thermal_hydraulics - tmap8 + #- cardinal reports: !include ${ROOT_DIR}/doc/sqa_reports.yml diff --git a/doc/sqa_reports.yml b/doc/sqa_reports.yml index 69de022c..87e38782 100644 --- a/doc/sqa_reports.yml +++ b/doc/sqa_reports.yml @@ -48,9 +48,11 @@ Requirements: - ${MOOSE_DIR}/modules/ray_tracing/doc/content - ${MOOSE_DIR}/modules/scalar_transport/doc/content - ${MOOSE_DIR}/modules/solid_mechanics/doc/content + - ${MOOSE_DIR}/modules/solid_properties/doc/content - ${MOOSE_DIR}/modules/stochastic_tools/doc/content - ${MOOSE_DIR}/modules/thermal_hydraulics/doc/content - ${ROOT_DIR}/tmap8/doc/content + - ${ROOT_DIR}/cardinal/doc/content directories: - ${ROOT_DIR}/test show_warning: false From 0d08962fa93ec7742ac82ce8db1b40c9270564e7 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 10 Jul 2024 12:13:24 -0600 Subject: [PATCH 10/28] Transition to running the cardinal build in the prebuild Makefile rule --- Makefile | 7 ++ scripts/build_cardinal.sh | 80 ++++++++++++++ scripts/update_and_rebuild_cardinal.sh | 139 ------------------------- 3 files changed, 87 insertions(+), 139 deletions(-) create mode 100755 scripts/build_cardinal.sh delete mode 100755 scripts/update_and_rebuild_cardinal.sh diff --git a/Makefile b/Makefile index e2540825..61811251 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,13 @@ include $(FRAMEWORK_DIR)/app.mk ############################################################################### # Additional special case targets should be added here +# Used in prebuild +export MOOSE_JOBS=$(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS))) + +# Adding to rule in moose.mk to prebuild cardinal and dependencies +prebuild:: + +@-bash $(CURDIR)/scripts/build_cardinal.sh + CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib -L$(OPENMC_LIBDIR) \ -Wl,-rpath,$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB diff --git a/scripts/build_cardinal.sh b/scripts/build_cardinal.sh new file mode 100755 index 00000000..37f0ae79 --- /dev/null +++ b/scripts/build_cardinal.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +#* This file is part of FENIX: Fusion ENergy Integrated multiphys-X, +#* A multiphysics application for modeling plasma facing components +#* https://github.com/idaholab/fenix +#* https://mooseframework.inl.gov/fenix +#* +#* FENIX is powered by the MOOSE Framework +#* https://www.mooseframework.inl.gov +#* +#* Licensed under LGPL 2.1, please see LICENSE for details +#* https://www.gnu.org/licenses/lgpl-2.1.html +#* +#* Copyright 2024, Battelle Energy Alliance, LLC +#* ALL RIGHTS RESERVED +#* + +# Colors for bash output +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo "Running ${SCRIPT_DIR}/build_cardinal.sh..." +FENIX_DIR=$(realpath $SCRIPT_DIR/..) + +if [ -n "$MOOSE_DIR" ]; then + echo "INFO: MOOSE_DIR set - ${MOOSE_DIR} will be used for both FENIX and Cardinal..." +else + export MOOSE_DIR=${FENIX_DIR}/moose +fi + +if [ -n "$CARDINAL_DIR" ]; then + echo "INFO: CARDINAL_DIR set - using ${CARDINAL_DIR}..." +else + CARDINAL_DIR=${FENIX_DIR}/cardinal +fi + +# Checking for HDF5 using HDF5_DIR (set via conda automatically). If this is not set, +# throw an error code and request that the user set it! +if [ -n "$HDF5_DIR" ]; then + export HDF5_ROOT=$HDF5_DIR +else + echo -e "${RED}ERROR: HDF5 not found; please set HDF5_DIR in your environment!${NC}" + exit 1 +fi + +cd ${CARDINAL_DIR} + +# Enable/Disable Cardinal contribs +export ENABLE_NEK=no +export ENABLE_DAGMC=yes # Also enables MOAB + +# Download all cardinal dependencies +echo "INFO: Updating OpenMC, DAGMC, and MOAB submodules..." +git submodule update --init --recursive contrib/openmc +git submodule update --init contrib/DAGMC +git submodule update --init contrib/moab + +if [[ -z "${OPENMC_CROSS_SECTIONS}" ]]; then + echo -e "${YELLOW}" + echo "If you are using OpenMC, remember that you need to set OPENMC_CROSS_SECTIONS to point to a cross_sections.xml file!" + echo "To get the ENDF/b-7.1 dataset, please run:" + echo "" + echo "$CARDINAL_DIR/scripts/download-openmc-cross-sections.sh" + echo -e "${NC}" +fi + +# If using conda, set LD_LIBRARY_PATH to workaround linking issue for some systems. +# Cardinal dev team is working on a more elegant fix, and this should be removed +# when that occurs. +if [ -n "$CONDA_PREFIX" ]; then + export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH +fi + +# Run make, letting the number of jobs be either MOOSE_JOBS or 1 if MOOSE_JOBS is not set. +# MAKEFLAGS is used to make sure that CMake within Cardinal also uses the appropriate +# number of jobs. 'sh -c' is used to invoke make since this script is used within a +# Makefile, where it would be treated as a "submake". We want to de-couple this make +# invokation with the parent process. +sh -c make -j ${MOOSE_JOBS:-1} MAKEFLAGS=-j${MOOSE_JOBS:-1} diff --git a/scripts/update_and_rebuild_cardinal.sh b/scripts/update_and_rebuild_cardinal.sh deleted file mode 100755 index a83f5a7e..00000000 --- a/scripts/update_and_rebuild_cardinal.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash -#* This file is part of FENIX: Fusion ENergy Integrated multiphys-X, -#* A multiphysics application for modeling plasma facing components -#* https://github.com/idaholab/fenix -#* https://mooseframework.inl.gov/fenix -#* -#* FENIX is powered by the MOOSE Framework -#* https://www.mooseframework.inl.gov -#* -#* Licensed under LGPL 2.1, please see LICENSE for details -#* https://www.gnu.org/licenses/lgpl-2.1.html -#* -#* Copyright 2024, Battelle Energy Alliance, LLC -#* ALL RIGHTS RESERVED -#* - -# Colors for bash output -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Set various flags based on argument input (fast, help, skip update, etc.) -for i in "$@" -do - shift - if [ "$i" == "--fast" ]; then - go_fast=1; - fi - - if [[ "$i" == "-h" || "$i" == "--help" ]]; then - help=1; - fi - - if [ "$i" == "--skip-submodule-update" ]; then - skip_sub_update=1; - else # Remove the skip submodule update argument before passing further - set -- "$@" "$i" - fi -done - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "Running ${SCRIPT_DIR}/update_and_rebuild_fenix.sh..." -FENIX_DIR=$(realpath $SCRIPT_DIR/..) - -if [ -n "$MOOSE_DIR" ]; then - echo "INFO: MOOSE_DIR set - ${MOOSE_DIR} will be used for both FENIX and Cardinal..." -else - export MOOSE_DIR=${FENIX_DIR}/moose -fi - -if [ -n "$CARDINAL_DIR" ]; then - echo "INFO: CARDINAL_DIR set - using ${CARDINAL_DIR} and skipping submodule update..." - skip_sub_update=1 -else - CARDINAL_DIR=${FENIX_DIR}/cardinal -fi - -# Checking for HDF5 using HDF5_DIR (set via conda automatically). If this is not set, -# throw an error code and request that the user set it! -if [ -n "$HDF5_DIR" ]; then - export HDF5_ROOT=$HDF5_DIR -else - echo -e "${RED}ERROR: HDF5 not found; please set HDF5_DIR in your environment!${NC}" - exit 1 -fi - - -# Display help if requested -if [[ -n "$help" ]]; then - echo "Usage: $0 [-h | --help | --fast | --skip-submodule-update ]" - echo - echo "-h | --help Display this message" - echo "--fast Run Cardinal 'make' only, do NOT update Cardinal contribs" - echo "--skip-submodule-update Do not update the Cardinal submodule, use the current version" - echo "*************************************************************************************" - echo "" - - exit 0 -fi - -# Print information to the screen if fast is used with any other arguments other than skip-submodule-update -if [[ -n "$go_fast" && $# != 1 ]]; then - echo -e "${RED}ERROR: --fast can only be used by itself or with --skip-submodule-update." - echo -e "Try again, removing either --fast or all other conflicting arguments!${NC}" - exit 1; -fi - -# Update Cardinal submodule if we're not skipping -if [ -z "$skip_sub_update" ]; then - cd ${FENIX_DIR} - git_dir=`git rev-parse --show-cdup 2>/dev/null` - if [[ -z "$go_fast" && $? == 0 && "x$git_dir" == "x" ]]; then - git submodule update --init cardinal - if [[ $? != 0 ]]; then - echo "git submodule command failed, are your proxy settings correct?" - exit 1 - fi - fi -fi - -cd ${CARDINAL_DIR} - -# Enable/Disable Cardinal contribs -export ENABLE_NEK=no -export ENABLE_DAGMC=yes # Also enables MOAB - -# If we're not going fast, update all dependencies -if [ -z "$go_fast" ]; then - echo "INFO: Updating OpenMC, DAGMC, and MOAB submodules..." - git submodule update --init --recursive contrib/openmc - git submodule update --init contrib/DAGMC - git submodule update --init contrib/moab -fi - -if [[ -z "${OPENMC_CROSS_SECTIONS}" ]]; then - echo -e "${YELLOW}" - echo "If you are using OpenMC, remember that you need to set OPENMC_CROSS_SECTIONS to point to a cross_sections.xml file!" - echo "To get the ENDF/b-7.1 dataset, please run:" - echo "" - echo "$CARDINAL_DIR/scripts/download-openmc-cross-sections.sh" - echo -e "${NC}" -fi - -# If using conda, set LD_LIBRARY_PATH to workaround linking issue for some systems. -# Cardinal dev team is working on a more elegant fix, and this should be removed -# when that occurs. -if [ -n "$CONDA_PREFIX" ]; then - export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH -fi - -# Run make, letting the number of jobs be either MOOSE_JOBS or 1 if MOOSE_JOBS is not set. -# MAKEFLAGS is used to make sure that CMake within Cardinal also uses the appropriate -# number of jobs. -make -j ${MOOSE_JOBS:-1} MAKEFLAGS=-j${MOOSE_JOBS:-1} - -# Remove cardinal executable, not required for FENIX build. -echo -e "${YELLOW}WARNING: Removing cardinal-${METHOD:-opt} since FENIX will be built." -echo -e "If a Cardinal executable is desired, please navigate to ${CARDINAL_DIR} and 'make' again.${NC}" -rm cardinal-"${METHOD:-opt}" From ad2a1377737ac4b5cbf8006e4e6e7b4202468b65 Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Wed, 17 Jul 2024 07:48:01 -0600 Subject: [PATCH 11/28] Fixup build Cardinal script for case when PETSc is pre-installed and HDF5 needs to be found in PETSc location --- scripts/build_cardinal.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/build_cardinal.sh b/scripts/build_cardinal.sh index 37f0ae79..680211a5 100755 --- a/scripts/build_cardinal.sh +++ b/scripts/build_cardinal.sh @@ -36,9 +36,13 @@ else fi # Checking for HDF5 using HDF5_DIR (set via conda automatically). If this is not set, +# check if PETSC_DIR is defined (indicates that PETSc is pre-installed, and HDF5 is +# installed via the PETSc script if not available using conda). If all else fails, # throw an error code and request that the user set it! if [ -n "$HDF5_DIR" ]; then export HDF5_ROOT=$HDF5_DIR +elif [ -n "$PETSC_DIR" ]; then + export HDF5_ROOT=$PETSC_DIR else echo -e "${RED}ERROR: HDF5 not found; please set HDF5_DIR in your environment!${NC}" exit 1 From db1bbd77a84d9ce6a0acfe04186ad41d3f1efa3a Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Wed, 17 Jul 2024 09:56:23 -0400 Subject: [PATCH 12/28] Fixup installation documentation given Makefile changes --- doc/content/getting_started/installation.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index 3a87bf78..39be748c 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -65,15 +65,9 @@ Then navigate to the FENIX clone directory and download the MOOSE and TMAP8 subm cd ~/projects/FENIX git submodule update --init moose git submodule update --init tmap8 +git submodule update --init cardinal ``` -Finally, Cardinal and its dependencies must be downloaded and pre-built. To perform this, first run -the build script from the FENIX clone directory: - -```bash -cd ~/projects/FENIX -scripts/update_and_rebuild_cardinal.sh -``` Then, to download OpenMC cross sections needed for OpenMC-based Cardinal runs, run: From fe46bb3ef6340daa6e9094bcc044a714ff55ccea Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Thu, 1 Aug 2024 16:26:35 -0600 Subject: [PATCH 13/28] Address review comment Co-authored-by: Pierre-Clement Simon --- doc/content/getting_started/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index 39be748c..1bdb0d22 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -59,7 +59,7 @@ To compile FENIX, first make sure that the conda MOOSE environment is activated conda activate moose ``` -Then navigate to the FENIX clone directory and download the MOOSE and TMAP8 submodules: +Then navigate to the FENIX clone directory and download the MOOSE, TMAP8, and Cardinal submodules: ```bash cd ~/projects/FENIX From 6f6f757d8007f8c0dd9181573e2f9763213f7713 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Thu, 1 Aug 2024 16:28:33 -0600 Subject: [PATCH 14/28] Move submodule updates to getting started instructions --- doc/content/getting_started/installation.md | 10 +++++++++- scripts/build_cardinal.sh | 6 ------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index 1bdb0d22..db19fb1d 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -68,8 +68,16 @@ git submodule update --init tmap8 git submodule update --init cardinal ``` +Next, some Cardinal dependencies need to be downloaded: -Then, to download OpenMC cross sections needed for OpenMC-based Cardinal runs, run: +```bash +cd ~/projects/FENIX/cardinal +git submodule update --init --recursive contrib/openmc +git submodule update --init contrib/DAGMC +git submodule update --init contrib/moab +``` + +To download OpenMC cross sections needed for OpenMC-based Cardinal runs, run: ```bash cd ~/projects/FENIX/cardinal diff --git a/scripts/build_cardinal.sh b/scripts/build_cardinal.sh index 680211a5..9e5568a6 100755 --- a/scripts/build_cardinal.sh +++ b/scripts/build_cardinal.sh @@ -54,12 +54,6 @@ cd ${CARDINAL_DIR} export ENABLE_NEK=no export ENABLE_DAGMC=yes # Also enables MOAB -# Download all cardinal dependencies -echo "INFO: Updating OpenMC, DAGMC, and MOAB submodules..." -git submodule update --init --recursive contrib/openmc -git submodule update --init contrib/DAGMC -git submodule update --init contrib/moab - if [[ -z "${OPENMC_CROSS_SECTIONS}" ]]; then echo -e "${YELLOW}" echo "If you are using OpenMC, remember that you need to set OPENMC_CROSS_SECTIONS to point to a cross_sections.xml file!" From 380c77255f2d376429b8c1ddb610a887209e4583 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Tue, 13 Aug 2024 18:06:25 -0600 Subject: [PATCH 15/28] Building Cardinal contribs directly in the FENIX makefile This borrows heavily from the workflow in "cardinal/Makefile" and uses "cardinal/config" *.mk files to do the contrib builds and linking directly in the FENIX makefile, generating a single executable --- Makefile | 150 +++++++++++++++++++++++++++++++------- scripts/build_cardinal.sh | 78 -------------------- 2 files changed, 125 insertions(+), 103 deletions(-) delete mode 100755 scripts/build_cardinal.sh diff --git a/Makefile b/Makefile index 61811251..3f84586e 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,29 @@ -############################################################################### -################### MOOSE Application Standard Makefile ####################### -############################################################################### +#################################################################################### +################################## FENIX Makefile ################################## +#################################################################################### # -# Optional Environment variables -# MOOSE_DIR - Root directory of the MOOSE project -# TMAP8_DIR - Root directory of the TMAP8 project -# CARDINAL_DIR - Root directory of the Cardinal project +# Optional Environment variables: # -############################################################################### +# MOOSE_DIR - Root directory of MOOSE +# TMAP8_DIR - Root directory of TMAP8 +# CARDINAL_DIR - Root directory of Cardinal +# OPENMC_DIR - Root directory of OpenMC +# DAGMC_DIR - Root directory of DagMC +# MOAB_DIR - Root directory of MOAB +# +# EIGEN3_DIR - Root directory of Eigen3 (should contain FindEigen3.cmake). +# This is needed for DagMC. +# +# OpenMC uses HDF5; below are influential environment variables for that +# installation. None of these need to be set if HDF5 is being pulled from +# a manual PETSc installation, and HDF5_DIR is provided directly from conda. +# +# HDF5_DIR - Root directory of HDF5 +# HDF5_INCLUDE_DIR - Root directory for HDF5 headers (default: $(HDF5_DIR)/include) +# HDF5_LIBDIR - Root directory for HDF5 libraries (default: $(HDF5_DIR)/lib) +# PETSC_DIR - Root directory for PETSc (default: $(MOOSE_DIR)/petsc) +# PETSC_ARCH - PETSc architecture (default: arch-moose) +#################################################################################### # Use the MOOSE submodule if it exists and MOOSE_DIR is not set # If it doesn't exist, and MOOSE_DIR is not set, then look for it adjacent to the application MOOSE_SUBMODULE := $(CURDIR)/moose @@ -36,10 +52,69 @@ else endif # Cardinal contrib variables -CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib -CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install -OPENMC_INSTALL_DIR := $(CONTRIB_INSTALL_DIR) -OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib +CARDINAL_CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install +OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +OPENMC_BUILDDIR := $(CARDINAL_DIR)/build/openmc +OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) +OPENMC_INCLUDES := -I$(OPENMC_INSTALL_DIR)/include +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +OPENMC_LIB := $(OPENMC_LIBDIR)/libopenmc.so + +PETSC_DIR ?= $(MOOSE_DIR)/petsc +PETSC_ARCH ?= arch-moose +LIBMESH_DIR ?= $(MOOSE_DIR)/libmesh/installed/ + +# This is the Eigen3 location on CIVET. If you are using MOOSE's conda environment, +# you don't need to set these variables, because conda sets them for you. The only +# scenario where you might need to manually set these is if you're not using the +# conda environment. You will get a compile error about FindEigen3.cmake if you +# do indeed need to set these. +EIGEN3_DIR ?= $(LIBMESH_DIR)/include +Eigen3_DIR ?= $(EIGEN3_DIR) + +# If HDF5_DIR is set, use those settings to link HDF5 to OpenMC. +# Otherwise, use where PETSc will put HDF5 if downloading it. +ifeq ($(HDF5_DIR),) + HDF5_DIR := $(PETSC_DIR)/$(PETSC_ARCH) + export HDF5_DIR +endif + +# HDF5_ROOT is used in makefiles for the Cardinal third-party dependencies. +# Set it using HDF5_DIR +HDF5_ROOT := $(HDF5_DIR) +export HDF5_ROOT + +HDF5_INCLUDE_DIR ?= $(HDF5_DIR)/include +HDF5_LIBDIR ?= $(HDF5_DIR)/lib +# This is used in $(FRAMEWORK_DIR)/build.mk +HDF5_INCLUDES := -I$(HDF5_INCLUDE_DIR) -I$(HDF5_DIR)/include + +# BUILD_TYPE will be passed to CMake via CMAKE_BUILD_TYPE +ifeq ($(METHOD),dbg) + BUILD_TYPE := Debug +else + BUILD_TYPE := Release +endif + +DAGMC_BUILDDIR := $(CARDINAL_DIR)/build/DAGMC +DAGMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) + +MOAB_BUILDDIR := $(CARDINAL_DIR)/build/moab +MOAB_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) + +# Set default values for all third party dependencies +OPENMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/openmc +DAGMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/DAGMC +MOAB_DIR ?= $(CARDINAL_CONTRIB_DIR)/moab + +# Use compiler info discovered by PETSC +ifeq ($(PETSC_ARCH),) + include $(PETSC_DIR)/$(PETSC_ARCH)/lib/petsc/conf/petscvariables +else + include $(PETSC_DIR)/lib/petsc/conf/petscvariables +endif # framework FRAMEWORK_DIR := $(MOOSE_DIR)/framework @@ -83,6 +158,31 @@ XFEM := no include $(MOOSE_DIR)/modules/modules.mk ############################################################################### +# Add OpenMC flags +ADDITIONAL_CPPFLAGS += $(HDF5_INCLUDES) $(OPENMC_INCLUDES) +libmesh_CXXFLAGS += -DENABLE_OPENMC_COUPLING + +# Add DagMC flags (-DDAGMC is used in OpenMC) +libmesh_CXXFLAGS += -DENABLE_DAGMC -DDAGMC + +# Configure and build MOAB, DagMC, and then OpenMC +include $(CARDINAL_DIR)/config/moab.mk +include $(CARDINAL_DIR)/config/dagmc.mk + +# autoconf-archive puts some arguments (e.g. -std=c++17) into the compiler +# variable rather than the compiler flags variable. +# +# cmake allows this, but wants any compiler arguments to be +# semicolon-separated, not space-separated +# libmesh_CC, etc., were defined in build.mk +space := $(subst ,, ) +LIBMESH_CC_LIST := $(subst $(space),;,$(libmesh_CC)) +LIBMESH_CXX_LIST := $(subst $(space),;,$(libmesh_CXX)) +LIBMESH_F90_LIST := $(subst $(space),;,$(libmesh_F90)) + +ENABLE_DAGMC := ON +include $(CARDINAL_DIR)/config/openmc.mk + # TMAP8 APPLICATION_DIR := $(TMAP8_DIR) APPLICATION_NAME := tmap8 @@ -97,29 +197,29 @@ BUILD_EXEC := no GEN_REVISION := no include $(FRAMEWORK_DIR)/app.mk -# dep apps +# app_objects are defined in moose.mk and built according to the rules in build.mk +# We need to build these first so we get include dirs +$(app_objects): build_moab build_dagmc build_openmc +$(test_objects): build_moab build_dagmc build_openmc + +# FENIX APPLICATION_DIR := $(CURDIR) APPLICATION_NAME := fenix BUILD_EXEC := yes GEN_REVISION := no # Cardinal dependency libraries -ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB +ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ + -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \ + $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) include $(FRAMEWORK_DIR)/app.mk -############################################################################### -# Additional special case targets should be added here - -# Used in prebuild -export MOOSE_JOBS=$(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS))) - -# Adding to rule in moose.mk to prebuild cardinal and dependencies -prebuild:: - +@-bash $(CURDIR)/scripts/build_cardinal.sh -CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib -L$(OPENMC_LIBDIR) \ - -Wl,-rpath,$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB +# Cardinal contrib flags used in app.mk targets +CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib $(BLASLAPACK_LIB) $(PETSC_EXTERNAL_LIB_BASIC) -L$(OPENMC_LIBDIR) -L$(HDF5_LIBDIR) -lopenmc -ldagmc -lMOAB $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) $(CC_LINKER_SLFLAG)$(HDF5_LIBDIR) # EXTERNAL_FLAGS is used in rules for app.mk +$(app_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) +$(app_test_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) $(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) diff --git a/scripts/build_cardinal.sh b/scripts/build_cardinal.sh deleted file mode 100755 index 9e5568a6..00000000 --- a/scripts/build_cardinal.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -#* This file is part of FENIX: Fusion ENergy Integrated multiphys-X, -#* A multiphysics application for modeling plasma facing components -#* https://github.com/idaholab/fenix -#* https://mooseframework.inl.gov/fenix -#* -#* FENIX is powered by the MOOSE Framework -#* https://www.mooseframework.inl.gov -#* -#* Licensed under LGPL 2.1, please see LICENSE for details -#* https://www.gnu.org/licenses/lgpl-2.1.html -#* -#* Copyright 2024, Battelle Energy Alliance, LLC -#* ALL RIGHTS RESERVED -#* - -# Colors for bash output -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "Running ${SCRIPT_DIR}/build_cardinal.sh..." -FENIX_DIR=$(realpath $SCRIPT_DIR/..) - -if [ -n "$MOOSE_DIR" ]; then - echo "INFO: MOOSE_DIR set - ${MOOSE_DIR} will be used for both FENIX and Cardinal..." -else - export MOOSE_DIR=${FENIX_DIR}/moose -fi - -if [ -n "$CARDINAL_DIR" ]; then - echo "INFO: CARDINAL_DIR set - using ${CARDINAL_DIR}..." -else - CARDINAL_DIR=${FENIX_DIR}/cardinal -fi - -# Checking for HDF5 using HDF5_DIR (set via conda automatically). If this is not set, -# check if PETSC_DIR is defined (indicates that PETSc is pre-installed, and HDF5 is -# installed via the PETSc script if not available using conda). If all else fails, -# throw an error code and request that the user set it! -if [ -n "$HDF5_DIR" ]; then - export HDF5_ROOT=$HDF5_DIR -elif [ -n "$PETSC_DIR" ]; then - export HDF5_ROOT=$PETSC_DIR -else - echo -e "${RED}ERROR: HDF5 not found; please set HDF5_DIR in your environment!${NC}" - exit 1 -fi - -cd ${CARDINAL_DIR} - -# Enable/Disable Cardinal contribs -export ENABLE_NEK=no -export ENABLE_DAGMC=yes # Also enables MOAB - -if [[ -z "${OPENMC_CROSS_SECTIONS}" ]]; then - echo -e "${YELLOW}" - echo "If you are using OpenMC, remember that you need to set OPENMC_CROSS_SECTIONS to point to a cross_sections.xml file!" - echo "To get the ENDF/b-7.1 dataset, please run:" - echo "" - echo "$CARDINAL_DIR/scripts/download-openmc-cross-sections.sh" - echo -e "${NC}" -fi - -# If using conda, set LD_LIBRARY_PATH to workaround linking issue for some systems. -# Cardinal dev team is working on a more elegant fix, and this should be removed -# when that occurs. -if [ -n "$CONDA_PREFIX" ]; then - export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH -fi - -# Run make, letting the number of jobs be either MOOSE_JOBS or 1 if MOOSE_JOBS is not set. -# MAKEFLAGS is used to make sure that CMake within Cardinal also uses the appropriate -# number of jobs. 'sh -c' is used to invoke make since this script is used within a -# Makefile, where it would be treated as a "submake". We want to de-couple this make -# invokation with the parent process. -sh -c make -j ${MOOSE_JOBS:-1} MAKEFLAGS=-j${MOOSE_JOBS:-1} From 5c02119ecf607953c6eaa7222cddaf4ce665df05 Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Tue, 13 Aug 2024 19:04:31 -0600 Subject: [PATCH 16/28] Update cardinal submodule --- cardinal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cardinal b/cardinal index 935473ff..ff1bb58b 160000 --- a/cardinal +++ b/cardinal @@ -1 +1 @@ -Subproject commit 935473ff564b34ef20ab9be118ab6bec4328ec18 +Subproject commit ff1bb58ba321cb76af93051d4e040ef0452c2fa4 From 3abe93c17c2d4b3cf451cceb9b30c7a61bf07a96 Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Tue, 13 Aug 2024 19:48:41 -0600 Subject: [PATCH 17/28] Fixup breakages with updated cardinal - Generate CardinalRevision.h due to installation changes, turn on FenixRevision.h as well (not required yet) - Add dummy redirect file for AddTallyAction for use with SQA --- Makefile | 4 ++-- doc/content/syntax/Problem/Tallies/index.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 doc/content/syntax/Problem/Tallies/index.md diff --git a/Makefile b/Makefile index 3f84586e..5da30ea4 100644 --- a/Makefile +++ b/Makefile @@ -194,7 +194,7 @@ include $(FRAMEWORK_DIR)/app.mk APPLICATION_DIR := $(CARDINAL_DIR) APPLICATION_NAME := cardinal BUILD_EXEC := no -GEN_REVISION := no +GEN_REVISION := yes include $(FRAMEWORK_DIR)/app.mk # app_objects are defined in moose.mk and built according to the rules in build.mk @@ -206,7 +206,7 @@ $(test_objects): build_moab build_dagmc build_openmc APPLICATION_DIR := $(CURDIR) APPLICATION_NAME := fenix BUILD_EXEC := yes -GEN_REVISION := no +GEN_REVISION := yes # Cardinal dependency libraries ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ diff --git a/doc/content/syntax/Problem/Tallies/index.md b/doc/content/syntax/Problem/Tallies/index.md new file mode 100644 index 00000000..5b69f860 --- /dev/null +++ b/doc/content/syntax/Problem/Tallies/index.md @@ -0,0 +1 @@ +!include source/actions/AddTallyAction.md From c9d7173528b13cc291bfeda93f33dd40cfce94b8 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 14 Aug 2024 11:28:44 -0600 Subject: [PATCH 18/28] Adjust HDF5_DIR in the case that PETSc has built HDF5 This will hopefully robustify this for the case when PETSc is installed in a container as well as locally (within an arch-* location) --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5da30ea4..f73289ac 100644 --- a/Makefile +++ b/Makefile @@ -75,9 +75,9 @@ EIGEN3_DIR ?= $(LIBMESH_DIR)/include Eigen3_DIR ?= $(EIGEN3_DIR) # If HDF5_DIR is set, use those settings to link HDF5 to OpenMC. -# Otherwise, use where PETSc will put HDF5 if downloading it. +# Otherwise, use the PETSc location, as the self-built HDF5 libraries will be housed within. ifeq ($(HDF5_DIR),) - HDF5_DIR := $(PETSC_DIR)/$(PETSC_ARCH) + HDF5_DIR := $(PETSC_DIR) export HDF5_DIR endif From 3b9dca59a25bc191f72eb620797589330b496b18 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 14 Aug 2024 15:09:34 -0600 Subject: [PATCH 19/28] Add cardinal and dependencies to FENIX unit Makefile --- unit/Makefile | 169 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 153 insertions(+), 16 deletions(-) diff --git a/unit/Makefile b/unit/Makefile index 197b0a1f..0a59a997 100644 --- a/unit/Makefile +++ b/unit/Makefile @@ -1,16 +1,31 @@ -############################################################################### -################### MOOSE Application Standard Makefile ####################### -############################################################################### +#################################################################################### +################################## FENIX Makefile ################################## +#################################################################################### # -# Required Environment variables (one of the following) -# PACKAGES_DIR - Location of the MOOSE redistributable package +# Optional Environment variables: # -# Optional Environment variables -# MOOSE_DIR - Root directory of the MOOSE project -# FRAMEWORK_DIR - Location of the MOOSE framework +# MOOSE_DIR - Root directory of MOOSE +# TMAP8_DIR - Root directory of TMAP8 +# CARDINAL_DIR - Root directory of Cardinal +# OPENMC_DIR - Root directory of OpenMC +# DAGMC_DIR - Root directory of DagMC +# MOAB_DIR - Root directory of MOAB # -############################################################################### +# EIGEN3_DIR - Root directory of Eigen3 (should contain FindEigen3.cmake). +# This is needed for DagMC. +# +# OpenMC uses HDF5; below are influential environment variables for that +# installation. None of these need to be set if HDF5 is being pulled from +# a manual PETSc installation, and HDF5_DIR is provided directly from conda. +# +# HDF5_DIR - Root directory of HDF5 +# HDF5_INCLUDE_DIR - Root directory for HDF5 headers (default: $(HDF5_DIR)/include) +# HDF5_LIBDIR - Root directory for HDF5 libraries (default: $(HDF5_DIR)/lib) +# PETSC_DIR - Root directory for PETSc (default: $(MOOSE_DIR)/petsc) +# PETSC_ARCH - PETSc architecture (default: arch-moose) +#################################################################################### # Use the MOOSE submodule if it exists and MOOSE_DIR is not set +# If it doesn't exist, and MOOSE_DIR is not set, then look for it adjacent to the application MOOSE_SUBMODULE := $(CURDIR)/../moose ifneq ($(wildcard $(MOOSE_SUBMODULE)/framework/Makefile),) MOOSE_DIR ?= $(MOOSE_SUBMODULE) @@ -22,11 +37,85 @@ FRAMEWORK_DIR ?= $(MOOSE_DIR)/framework # Use the TMAP8 submodule if it exists and TMAP8_DIR is not set # If it doesn't exist, and TMAP8_DIR is not set, then look for it adjacent to the application -TMAP8_SUBMODULE := $(CURDIR)/tmap8 +TMAP8_SUBMODULE := $(CURDIR)/../tmap8 ifneq ($(wildcard $(TMAP8_SUBMODULE)/Makefile),) TMAP8_DIR ?= $(TMAP8_SUBMODULE) else - TMAP8_DIR ?= $(shell dirname `pwd`)/tmap8 + TMAP8_DIR ?= $(shell dirname `pwd`)/../tmap8 +endif + +# Use the Cardinal submodule if it exists and CARDINAL_DIR is not set +# If it doesn't exist, and CARDINAL_DIR is not set, then look for it adjacent to the application +CARDINAL_SUBMODULE := $(CURDIR)/../cardinal +ifneq ($(wildcard $(CARDINAL_SUBMODULE)/Makefile),) + CARDINAL_DIR ?= $(CARDINAL_SUBMODULE) +else + CARDINAL_DIR ?= $(shell dirname `pwd`)/../cardinal +endif + +# Cardinal contrib variables +CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib +CARDINAL_CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install +OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +OPENMC_BUILDDIR := $(CARDINAL_DIR)/build/openmc +OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) +OPENMC_INCLUDES := -I$(OPENMC_INSTALL_DIR)/include +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +OPENMC_LIB := $(OPENMC_LIBDIR)/libopenmc.so + +PETSC_DIR ?= $(MOOSE_DIR)/petsc +PETSC_ARCH ?= arch-moose +LIBMESH_DIR ?= $(MOOSE_DIR)/libmesh/installed/ + +# This is the Eigen3 location on CIVET. If you are using MOOSE's conda environment, +# you don't need to set these variables, because conda sets them for you. The only +# scenario where you might need to manually set these is if you're not using the +# conda environment. You will get a compile error about FindEigen3.cmake if you +# do indeed need to set these. +EIGEN3_DIR ?= $(LIBMESH_DIR)/include +Eigen3_DIR ?= $(EIGEN3_DIR) + +# If HDF5_DIR is set, use those settings to link HDF5 to OpenMC. +# Otherwise, use the PETSc location, as the self-built HDF5 libraries will be housed within. +ifeq ($(HDF5_DIR),) + HDF5_DIR := $(PETSC_DIR) + export HDF5_DIR +endif + +# HDF5_ROOT is used in makefiles for the Cardinal third-party dependencies. +# Set it using HDF5_DIR +HDF5_ROOT := $(HDF5_DIR) +export HDF5_ROOT + +HDF5_INCLUDE_DIR ?= $(HDF5_DIR)/include +HDF5_LIBDIR ?= $(HDF5_DIR)/lib +# This is used in $(FRAMEWORK_DIR)/build.mk +HDF5_INCLUDES := -I$(HDF5_INCLUDE_DIR) -I$(HDF5_DIR)/include + +# BUILD_TYPE will be passed to CMake via CMAKE_BUILD_TYPE +ifeq ($(METHOD),dbg) + BUILD_TYPE := Debug +else + BUILD_TYPE := Release +endif + +DAGMC_BUILDDIR := $(CARDINAL_DIR)/build/DAGMC +DAGMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) + +MOAB_BUILDDIR := $(CARDINAL_DIR)/build/moab +MOAB_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) + +# Set default values for all third party dependencies +OPENMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/openmc +DAGMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/DAGMC +MOAB_DIR ?= $(CARDINAL_CONTRIB_DIR)/moab + +# Use compiler info discovered by PETSC +ifeq ($(PETSC_ARCH),) + include $(PETSC_DIR)/$(PETSC_ARCH)/lib/petsc/conf/petscvariables +else + include $(PETSC_DIR)/lib/petsc/conf/petscvariables endif # framework @@ -74,6 +163,31 @@ include $(MOOSE_DIR)/modules/modules.mk ADDITIONAL_INCLUDES := -I$(FRAMEWORK_DIR)/contrib/gtest ADDITIONAL_LIBS := $(FRAMEWORK_DIR)/contrib/gtest/libgtest.la +# Add OpenMC flags +ADDITIONAL_CPPFLAGS += $(HDF5_INCLUDES) $(OPENMC_INCLUDES) +libmesh_CXXFLAGS += -DENABLE_OPENMC_COUPLING + +# Add DagMC flags (-DDAGMC is used in OpenMC) +libmesh_CXXFLAGS += -DENABLE_DAGMC -DDAGMC + +# Configure and build MOAB, DagMC, and then OpenMC +include $(CARDINAL_DIR)/config/moab.mk +include $(CARDINAL_DIR)/config/dagmc.mk + +# autoconf-archive puts some arguments (e.g. -std=c++17) into the compiler +# variable rather than the compiler flags variable. +# +# cmake allows this, but wants any compiler arguments to be +# semicolon-separated, not space-separated +# libmesh_CC, etc., were defined in build.mk +space := $(subst ,, ) +LIBMESH_CC_LIST := $(subst $(space),;,$(libmesh_CC)) +LIBMESH_CXX_LIST := $(subst $(space),;,$(libmesh_CXX)) +LIBMESH_F90_LIST := $(subst $(space),;,$(libmesh_F90)) + +ENABLE_DAGMC := ON +include $(CARDINAL_DIR)/config/openmc.mk + # TMAP8 APPLICATION_DIR := $(TMAP8_DIR) APPLICATION_NAME := tmap8 @@ -81,23 +195,46 @@ BUILD_EXEC := no GEN_REVISION := no include $(FRAMEWORK_DIR)/app.mk -# dep apps +# Cardinal +APPLICATION_DIR := $(CARDINAL_DIR) +APPLICATION_NAME := cardinal +BUILD_EXEC := no +GEN_REVISION := yes +include $(FRAMEWORK_DIR)/app.mk + +# app_objects are defined in moose.mk and built according to the rules in build.mk +# We need to build these first so we get include dirs +$(app_objects): build_moab build_dagmc build_openmc +$(test_objects): build_moab build_dagmc build_openmc + +# FENIX CURRENT_DIR := $(shell pwd) APPLICATION_DIR := $(CURRENT_DIR)/.. APPLICATION_NAME := fenix +GEN_REVISION := yes include $(FRAMEWORK_DIR)/app.mk APPLICATION_DIR := $(CURRENT_DIR) APPLICATION_NAME := fenix-unit BUILD_EXEC := yes - DEP_APPS ?= $(shell $(FRAMEWORK_DIR)/scripts/find_dep_apps.py $(APPLICATION_NAME)) + +# Cardinal dependency libraries +ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ + -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \ + $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) + include $(FRAMEWORK_DIR)/app.mk +# Cardinal contrib flags used in app.mk targets +CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib $(BLASLAPACK_LIB) $(PETSC_EXTERNAL_LIB_BASIC) -L$(OPENMC_LIBDIR) -L$(HDF5_LIBDIR) -lopenmc -ldagmc -lMOAB $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) $(CC_LINKER_SLFLAG)$(HDF5_LIBDIR) + +# EXTERNAL_FLAGS is used in rules for app.mk +$(app_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) +$(app_test_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) +$(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) + # Find all the Fenix unit test source files and include their dependencies. fenix_unit_srcfiles := $(shell find $(CURRENT_DIR)/src -name "*.C") fenix_unit_deps := $(patsubst %.C, %.$(obj-suffix).d, $(fenix_unit_srcfiles)) -include $(fenix_unit_deps) - -############################################################################### -# Additional special case targets should be added here From 90e7eefb6014eb8c794703ef1f88116e8009a12a Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Thu, 15 Aug 2024 13:57:43 -0600 Subject: [PATCH 20/28] Add apptainer context for generating an apptainer image of FENIX that includes Cardinal contribs and cross sections --- apptainer/README.md | 7 +++++++ apptainer/app_environment | 1 + apptainer/app_post_post_install | 7 +++++++ apptainer/app_post_pre_make | 5 +++++ 4 files changed, 20 insertions(+) create mode 100644 apptainer/README.md create mode 100644 apptainer/app_environment create mode 100644 apptainer/app_post_post_install create mode 100644 apptainer/app_post_pre_make diff --git a/apptainer/README.md b/apptainer/README.md new file mode 100644 index 00000000..7d990d4d --- /dev/null +++ b/apptainer/README.md @@ -0,0 +1,7 @@ +Contents in this directory are used for the generation of apptainer images using the `ApptainerGenerator` script found in MOOSE at `scripts/apptainer_generator.py`. + +In particular: + +- `app_envrionment`: Appends to the environment in the container to add `OPENMC_CROSS_SECTIONS` +- `app_post_post_install`: Copies the cross section download script for use in testing +- `app_post_pre_make`: Sets extra variables for the build and builds the extra dependencies (openmc, dagmc) before the main build so that the `MAKEFLAGS` variable can be set for parallel builds. diff --git a/apptainer/app_environment b/apptainer/app_environment new file mode 100644 index 00000000..8a03cea5 --- /dev/null +++ b/apptainer/app_environment @@ -0,0 +1 @@ +export OPENMC_CROSS_SECTIONS=/opt/fenix/share/fenix/endfb-vii.1-hdf5/cross_sections.xml diff --git a/apptainer/app_post_post_install b/apptainer/app_post_post_install new file mode 100644 index 00000000..fb9548ed --- /dev/null +++ b/apptainer/app_post_post_install @@ -0,0 +1,7 @@ +# Download cross sections +umask 022 +FROM_XS_SCRIPT=${APPLICATION_DIR}/cardinal/scripts/download-openmc-cross-sections.sh +XS_SCRIPT=${MOOSE_PREFIX}/share/${BINARY_NAME}/download-openmc-cross-sections.sh +cp ${FROM_XS_SCRIPT} ${XS_SCRIPT} +chmod +x ${XS_SCRIPT} +${XS_SCRIPT} ${MOOSE_PREFIX}/share/fenix diff --git a/apptainer/app_post_pre_make b/apptainer/app_post_pre_make new file mode 100644 index 00000000..39085a2c --- /dev/null +++ b/apptainer/app_post_pre_make @@ -0,0 +1,5 @@ +export CARDINAL_CONTRIB_INSTALL_DIR=/opt/fenix/contrib +export HDF5_ROOT=${PETSC_DIR} +export ENABLE_DAGMC=ON +make build_openmc MAKEFLAGS=-j${MOOSE_JOBS} +make build_dagmc MAKEFLAGS=-j${MOOSE_JOBS} From 687c57ef79225775ae45e5482cc98f2f8a4edd57 Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Fri, 16 Aug 2024 16:35:24 -0600 Subject: [PATCH 21/28] Add tip in documentation about speeding up Cardinal contrib builds --- doc/content/getting_started/installation.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index db19fb1d..91fc24c3 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -115,6 +115,19 @@ build the code and run the tests. The number in that flag can be changed to the number of physical and virtual cores on the workstation being used to build FENIX. !alert-end! +!alert! note title=Speeding up OpenMC/DAGMC/MOAB builds +If the build of Cardinal third-party libraries OpenMC, DAGMC, and MOAB is too slow on your machine, +you can attempt to pass the number of `make` cores using the `MAKEFLAGS` variable. This can do a +better job enabling proper parallelism in nested `make` builds (as is the case with Cardinal +dependencies). To use this variable, simply perform: + +```bash +make MAKEFLAGS=-j8 +``` + +instead of the `make` commands highlighted above. +!alert-end! + If FENIX is working correctly, all active tests will pass. This indicates that FENIX is ready to be used and further developed. From ffa88f907dc1672ac3510595e86b52339982be23 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Tue, 20 Aug 2024 09:22:28 -0600 Subject: [PATCH 22/28] Address review comments --- doc/content/getting_started/installation.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index 91fc24c3..74e5a9d5 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -73,7 +73,7 @@ Next, some Cardinal dependencies need to be downloaded: ```bash cd ~/projects/FENIX/cardinal git submodule update --init --recursive contrib/openmc -git submodule update --init contrib/DAGMC +git submodule update --init --recursive contrib/DAGMC git submodule update --init contrib/moab ``` @@ -158,6 +158,12 @@ git checkout main git fetch upstream git rebase upstream/main git submodule update moose +git submodule update tmap8 +git submodule update cardinal +cd cardinal +git submodule update --recursive contrib/openmc +git submodule update --recursive contrib/DAGMC +git submodule update contrib/moab ``` To update your FENIX repository as a FENIX developer who regularly makes modifications to the code, use the following commands, @@ -169,6 +175,12 @@ git checkout devel git fetch upstream git rebase upstream/devel git submodule update moose +git submodule update tmap8 +git submodule update cardinal +cd cardinal +git submodule update --recursive contrib/openmc +git submodule update --recursive contrib/DAGMC +git submodule update contrib/moab ``` Both sets of instructions assume that your copy of FENIX is stored in `~/projects` and that the [idaholab/FENIX](https://github.com/idaholab/FENIX) From a134ecb1f862d78c24eef735aa8ca0e57e9f13cb Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Mon, 26 Aug 2024 15:25:55 -0600 Subject: [PATCH 23/28] Make Cardinal an optional dependency of FENIX --- Makefile | 139 +++++------------------------- config/build_cardinal.mk | 37 ++++++++ config/check_deps.mk | 18 ++++ config/configure_cardinal.mk | 64 ++++++++++++++ config/external_cardinal_flags.mk | 9 ++ src/base/FenixApp.C | 8 ++ unit/Makefile | 139 +++++------------------------- 7 files changed, 177 insertions(+), 237 deletions(-) create mode 100644 config/build_cardinal.mk create mode 100644 config/check_deps.mk create mode 100644 config/configure_cardinal.mk create mode 100644 config/external_cardinal_flags.mk diff --git a/Makefile b/Makefile index f73289ac..0ed3ff5f 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ ifneq ($(wildcard $(MOOSE_SUBMODULE)/framework/Makefile),) else MOOSE_DIR ?= $(shell dirname `pwd`)/moose endif +FRAMEWORK_DIR := $(MOOSE_DIR)/framework # Use the TMAP8 submodule if it exists and TMAP8_DIR is not set # If it doesn't exist, and TMAP8_DIR is not set, then look for it adjacent to the application @@ -42,82 +43,16 @@ else TMAP8_DIR ?= $(shell dirname `pwd`)/tmap8 endif -# Use the Cardinal submodule if it exists and CARDINAL_DIR is not set -# If it doesn't exist, and CARDINAL_DIR is not set, then look for it adjacent to the application -CARDINAL_SUBMODULE := $(CURDIR)/cardinal -ifneq ($(wildcard $(CARDINAL_SUBMODULE)/Makefile),) - CARDINAL_DIR ?= $(CARDINAL_SUBMODULE) -else - CARDINAL_DIR ?= $(shell dirname `pwd`)/cardinal -endif - -# Cardinal contrib variables -CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib -CARDINAL_CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install -OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) -OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib -OPENMC_BUILDDIR := $(CARDINAL_DIR)/build/openmc -OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) -OPENMC_INCLUDES := -I$(OPENMC_INSTALL_DIR)/include -OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib -OPENMC_LIB := $(OPENMC_LIBDIR)/libopenmc.so - -PETSC_DIR ?= $(MOOSE_DIR)/petsc -PETSC_ARCH ?= arch-moose -LIBMESH_DIR ?= $(MOOSE_DIR)/libmesh/installed/ - -# This is the Eigen3 location on CIVET. If you are using MOOSE's conda environment, -# you don't need to set these variables, because conda sets them for you. The only -# scenario where you might need to manually set these is if you're not using the -# conda environment. You will get a compile error about FindEigen3.cmake if you -# do indeed need to set these. -EIGEN3_DIR ?= $(LIBMESH_DIR)/include -Eigen3_DIR ?= $(EIGEN3_DIR) - -# If HDF5_DIR is set, use those settings to link HDF5 to OpenMC. -# Otherwise, use the PETSc location, as the self-built HDF5 libraries will be housed within. -ifeq ($(HDF5_DIR),) - HDF5_DIR := $(PETSC_DIR) - export HDF5_DIR -endif - -# HDF5_ROOT is used in makefiles for the Cardinal third-party dependencies. -# Set it using HDF5_DIR -HDF5_ROOT := $(HDF5_DIR) -export HDF5_ROOT +# Check for optional dependencies and, if found, configure for building. +# Set default for ENABLE_CARDINAL. +ENABLE_CARDINAL := no +include config/check_deps.mk -HDF5_INCLUDE_DIR ?= $(HDF5_DIR)/include -HDF5_LIBDIR ?= $(HDF5_DIR)/lib -# This is used in $(FRAMEWORK_DIR)/build.mk -HDF5_INCLUDES := -I$(HDF5_INCLUDE_DIR) -I$(HDF5_DIR)/include - -# BUILD_TYPE will be passed to CMake via CMAKE_BUILD_TYPE -ifeq ($(METHOD),dbg) - BUILD_TYPE := Debug -else - BUILD_TYPE := Release -endif - -DAGMC_BUILDDIR := $(CARDINAL_DIR)/build/DAGMC -DAGMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) - -MOAB_BUILDDIR := $(CARDINAL_DIR)/build/moab -MOAB_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) - -# Set default values for all third party dependencies -OPENMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/openmc -DAGMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/DAGMC -MOAB_DIR ?= $(CARDINAL_CONTRIB_DIR)/moab - -# Use compiler info discovered by PETSC -ifeq ($(PETSC_ARCH),) - include $(PETSC_DIR)/$(PETSC_ARCH)/lib/petsc/conf/petscvariables -else - include $(PETSC_DIR)/lib/petsc/conf/petscvariables +ifeq ($(ENABLE_CARDINAL),yes) + include config/configure_cardinal.mk endif # framework -FRAMEWORK_DIR := $(MOOSE_DIR)/framework include $(FRAMEWORK_DIR)/build.mk include $(FRAMEWORK_DIR)/moose.mk @@ -158,31 +93,6 @@ XFEM := no include $(MOOSE_DIR)/modules/modules.mk ############################################################################### -# Add OpenMC flags -ADDITIONAL_CPPFLAGS += $(HDF5_INCLUDES) $(OPENMC_INCLUDES) -libmesh_CXXFLAGS += -DENABLE_OPENMC_COUPLING - -# Add DagMC flags (-DDAGMC is used in OpenMC) -libmesh_CXXFLAGS += -DENABLE_DAGMC -DDAGMC - -# Configure and build MOAB, DagMC, and then OpenMC -include $(CARDINAL_DIR)/config/moab.mk -include $(CARDINAL_DIR)/config/dagmc.mk - -# autoconf-archive puts some arguments (e.g. -std=c++17) into the compiler -# variable rather than the compiler flags variable. -# -# cmake allows this, but wants any compiler arguments to be -# semicolon-separated, not space-separated -# libmesh_CC, etc., were defined in build.mk -space := $(subst ,, ) -LIBMESH_CC_LIST := $(subst $(space),;,$(libmesh_CC)) -LIBMESH_CXX_LIST := $(subst $(space),;,$(libmesh_CXX)) -LIBMESH_F90_LIST := $(subst $(space),;,$(libmesh_F90)) - -ENABLE_DAGMC := ON -include $(CARDINAL_DIR)/config/openmc.mk - # TMAP8 APPLICATION_DIR := $(TMAP8_DIR) APPLICATION_NAME := tmap8 @@ -191,16 +101,9 @@ GEN_REVISION := no include $(FRAMEWORK_DIR)/app.mk # Cardinal -APPLICATION_DIR := $(CARDINAL_DIR) -APPLICATION_NAME := cardinal -BUILD_EXEC := no -GEN_REVISION := yes -include $(FRAMEWORK_DIR)/app.mk - -# app_objects are defined in moose.mk and built according to the rules in build.mk -# We need to build these first so we get include dirs -$(app_objects): build_moab build_dagmc build_openmc -$(test_objects): build_moab build_dagmc build_openmc +ifeq ($(ENABLE_CARDINAL),yes) + include config/build_cardinal.mk +endif # FENIX APPLICATION_DIR := $(CURDIR) @@ -208,18 +111,16 @@ APPLICATION_NAME := fenix BUILD_EXEC := yes GEN_REVISION := yes -# Cardinal dependency libraries -ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ - -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \ - $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) +# Cardinal dependency libraries needed for FENIX linking +ifeq ($(ENABLE_CARDINAL),yes) + ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ + -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \ + $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) +endif include $(FRAMEWORK_DIR)/app.mk - -# Cardinal contrib flags used in app.mk targets -CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib $(BLASLAPACK_LIB) $(PETSC_EXTERNAL_LIB_BASIC) -L$(OPENMC_LIBDIR) -L$(HDF5_LIBDIR) -lopenmc -ldagmc -lMOAB $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) $(CC_LINKER_SLFLAG)$(HDF5_LIBDIR) - -# EXTERNAL_FLAGS is used in rules for app.mk -$(app_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) -$(app_test_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) -$(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) +# External flags +ifeq ($(ENABLE_CARDINAL),yes) + include config/external_cardinal_flags.mk +endif diff --git a/config/build_cardinal.mk b/config/build_cardinal.mk new file mode 100644 index 00000000..26064ad3 --- /dev/null +++ b/config/build_cardinal.mk @@ -0,0 +1,37 @@ +# Add OpenMC flags +ADDITIONAL_CPPFLAGS += $(HDF5_INCLUDES) $(OPENMC_INCLUDES) +libmesh_CXXFLAGS += -DENABLE_OPENMC_COUPLING + +# Add DagMC flags (-DDAGMC is used in OpenMC) +libmesh_CXXFLAGS += -DENABLE_DAGMC -DDAGMC + +# Configure and build MOAB, DagMC, and then OpenMC +include $(CARDINAL_DIR)/config/moab.mk +include $(CARDINAL_DIR)/config/dagmc.mk + +# autoconf-archive puts some arguments (e.g. -std=c++17) into the compiler +# variable rather than the compiler flags variable. +# +# cmake allows this, but wants any compiler arguments to be +# semicolon-separated, not space-separated +# libmesh_CC, etc., were defined in build.mk +space := $(subst ,, ) +LIBMESH_CC_LIST := $(subst $(space),;,$(libmesh_CC)) +LIBMESH_CXX_LIST := $(subst $(space),;,$(libmesh_CXX)) +LIBMESH_F90_LIST := $(subst $(space),;,$(libmesh_F90)) + +ENABLE_DAGMC := ON +include $(CARDINAL_DIR)/config/openmc.mk + +# Cardinal +libmesh_CXXFLAGS += -DENABLE_CARDINAL +APPLICATION_DIR := $(CARDINAL_DIR) +APPLICATION_NAME := cardinal +BUILD_EXEC := no +GEN_REVISION := yes +include $(FRAMEWORK_DIR)/app.mk + +# app_objects are defined in moose.mk and built according to the rules in build.mk +# We need to build these first so we get include dirs +$(app_objects): build_moab build_dagmc build_openmc +$(test_objects): build_moab build_dagmc build_openmc diff --git a/config/check_deps.mk b/config/check_deps.mk new file mode 100644 index 00000000..8f33440a --- /dev/null +++ b/config/check_deps.mk @@ -0,0 +1,18 @@ +define n + + +endef + +# Set default value for Cardinal +CARDINAL_DIR ?= $(CURDIR)/cardinal + +# Check for CARDINAL_CONTENT within CARDINAL_DIR. This will cause Cardinal (and contribs) +# to either be built or skipped within the configure_cardinal.mk file. +CARDINAL_CONTENT := $(shell ls $(CARDINAL_DIR) 2> /dev/null) + +ifeq ($(CARDINAL_CONTENT),) + $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source.") +else + $(info FENIX is using Cardinal from $(CARDINAL_DIR)) + ENABLE_CARDINAL := yes +endif diff --git a/config/configure_cardinal.mk b/config/configure_cardinal.mk new file mode 100644 index 00000000..623461f3 --- /dev/null +++ b/config/configure_cardinal.mk @@ -0,0 +1,64 @@ +# Cardinal contrib variables +CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib +CARDINAL_CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install +OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +OPENMC_BUILDDIR := $(CARDINAL_DIR)/build/openmc +OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) +OPENMC_INCLUDES := -I$(OPENMC_INSTALL_DIR)/include +OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib +OPENMC_LIB := $(OPENMC_LIBDIR)/libopenmc.so + +PETSC_DIR ?= $(MOOSE_DIR)/petsc +PETSC_ARCH ?= arch-moose +LIBMESH_DIR ?= $(MOOSE_DIR)/libmesh/installed/ + +# This is the Eigen3 location on CIVET. If you are using MOOSE's conda environment, +# you don't need to set these variables, because conda sets them for you. The only +# scenario where you might need to manually set these is if you're not using the +# conda environment. You will get a compile error about FindEigen3.cmake if you +# do indeed need to set these. +EIGEN3_DIR ?= $(LIBMESH_DIR)/include +Eigen3_DIR ?= $(EIGEN3_DIR) + +# If HDF5_DIR is set, use those settings to link HDF5 to OpenMC. +# Otherwise, use the PETSc location, as the self-built HDF5 libraries will be housed within. +ifeq ($(HDF5_DIR),) + HDF5_DIR := $(PETSC_DIR) + export HDF5_DIR +endif + +# HDF5_ROOT is used in makefiles for the Cardinal third-party dependencies. +# Set it using HDF5_DIR +HDF5_ROOT := $(HDF5_DIR) +export HDF5_ROOT + +HDF5_INCLUDE_DIR ?= $(HDF5_DIR)/include +HDF5_LIBDIR ?= $(HDF5_DIR)/lib +# This is used in $(FRAMEWORK_DIR)/build.mk +HDF5_INCLUDES := -I$(HDF5_INCLUDE_DIR) -I$(HDF5_DIR)/include + +# BUILD_TYPE will be passed to CMake via CMAKE_BUILD_TYPE +ifeq ($(METHOD),dbg) + BUILD_TYPE := Debug +else + BUILD_TYPE := Release +endif + +DAGMC_BUILDDIR := $(CARDINAL_DIR)/build/DAGMC +DAGMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) + +MOAB_BUILDDIR := $(CARDINAL_DIR)/build/moab +MOAB_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) + +# Set default values for all third party dependencies +OPENMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/openmc +DAGMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/DAGMC +MOAB_DIR ?= $(CARDINAL_CONTRIB_DIR)/moab + +# Use compiler info discovered by PETSC +ifeq ($(PETSC_ARCH),) + include $(PETSC_DIR)/$(PETSC_ARCH)/lib/petsc/conf/petscvariables +else + include $(PETSC_DIR)/lib/petsc/conf/petscvariables +endif diff --git a/config/external_cardinal_flags.mk b/config/external_cardinal_flags.mk new file mode 100644 index 00000000..a1bfbcca --- /dev/null +++ b/config/external_cardinal_flags.mk @@ -0,0 +1,9 @@ +ifeq ($(ENABLE_CARDINAL),yes) + # Cardinal contrib flags used in app.mk targets + CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib $(BLASLAPACK_LIB) $(PETSC_EXTERNAL_LIB_BASIC) -L$(OPENMC_LIBDIR) -L$(HDF5_LIBDIR) -lopenmc -ldagmc -lMOAB $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) $(CC_LINKER_SLFLAG)$(HDF5_LIBDIR) + + # EXTERNAL_FLAGS is used in rules for app.mk + $(app_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) + $(app_test_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) + $(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) +endif diff --git a/src/base/FenixApp.C b/src/base/FenixApp.C index 3ba1c04c..f1a8c296 100644 --- a/src/base/FenixApp.C +++ b/src/base/FenixApp.C @@ -21,7 +21,9 @@ // contribs #include "TMAP8App.h" +#ifdef ENABLE_CARDINAL #include "CardinalApp.h" +#endif InputParameters FenixApp::validParams() @@ -47,7 +49,10 @@ FenixApp::registerAll(Factory & f, ActionFactory & af, Syntax & s) Registry::registerActionsTo(af, {"FenixApp"}); TMAP8App::registerAll(f, af, s); + +#ifdef ENABLE_CARDINAL CardinalApp::registerAll(f, af, s); +#endif /* register custom execute flags, action syntax, etc. here */ } @@ -58,7 +63,10 @@ FenixApp::registerApps() registerApp(FenixApp); ModulesApp::registerApps(); TMAP8App::registerApps(); + +#ifdef ENABLE_CARDINAL CardinalApp::registerApps(); +#endif } /*************************************************************************************************** diff --git a/unit/Makefile b/unit/Makefile index 0a59a997..197e4c93 100644 --- a/unit/Makefile +++ b/unit/Makefile @@ -44,78 +44,14 @@ else TMAP8_DIR ?= $(shell dirname `pwd`)/../tmap8 endif -# Use the Cardinal submodule if it exists and CARDINAL_DIR is not set -# If it doesn't exist, and CARDINAL_DIR is not set, then look for it adjacent to the application -CARDINAL_SUBMODULE := $(CURDIR)/../cardinal -ifneq ($(wildcard $(CARDINAL_SUBMODULE)/Makefile),) - CARDINAL_DIR ?= $(CARDINAL_SUBMODULE) -else - CARDINAL_DIR ?= $(shell dirname `pwd`)/../cardinal -endif - -# Cardinal contrib variables -CARDINAL_CONTRIB_DIR := $(CARDINAL_DIR)/contrib -CARDINAL_CONTRIB_INSTALL_DIR ?= $(CARDINAL_DIR)/install -OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) -OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib -OPENMC_BUILDDIR := $(CARDINAL_DIR)/build/openmc -OPENMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) -OPENMC_INCLUDES := -I$(OPENMC_INSTALL_DIR)/include -OPENMC_LIBDIR := $(OPENMC_INSTALL_DIR)/lib -OPENMC_LIB := $(OPENMC_LIBDIR)/libopenmc.so - -PETSC_DIR ?= $(MOOSE_DIR)/petsc -PETSC_ARCH ?= arch-moose -LIBMESH_DIR ?= $(MOOSE_DIR)/libmesh/installed/ - -# This is the Eigen3 location on CIVET. If you are using MOOSE's conda environment, -# you don't need to set these variables, because conda sets them for you. The only -# scenario where you might need to manually set these is if you're not using the -# conda environment. You will get a compile error about FindEigen3.cmake if you -# do indeed need to set these. -EIGEN3_DIR ?= $(LIBMESH_DIR)/include -Eigen3_DIR ?= $(EIGEN3_DIR) - -# If HDF5_DIR is set, use those settings to link HDF5 to OpenMC. -# Otherwise, use the PETSc location, as the self-built HDF5 libraries will be housed within. -ifeq ($(HDF5_DIR),) - HDF5_DIR := $(PETSC_DIR) - export HDF5_DIR -endif - -# HDF5_ROOT is used in makefiles for the Cardinal third-party dependencies. -# Set it using HDF5_DIR -HDF5_ROOT := $(HDF5_DIR) -export HDF5_ROOT - -HDF5_INCLUDE_DIR ?= $(HDF5_DIR)/include -HDF5_LIBDIR ?= $(HDF5_DIR)/lib -# This is used in $(FRAMEWORK_DIR)/build.mk -HDF5_INCLUDES := -I$(HDF5_INCLUDE_DIR) -I$(HDF5_DIR)/include - -# BUILD_TYPE will be passed to CMake via CMAKE_BUILD_TYPE -ifeq ($(METHOD),dbg) - BUILD_TYPE := Debug -else - BUILD_TYPE := Release -endif - -DAGMC_BUILDDIR := $(CARDINAL_DIR)/build/DAGMC -DAGMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) - -MOAB_BUILDDIR := $(CARDINAL_DIR)/build/moab -MOAB_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR) - -# Set default values for all third party dependencies -OPENMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/openmc -DAGMC_DIR ?= $(CARDINAL_CONTRIB_DIR)/DAGMC -MOAB_DIR ?= $(CARDINAL_CONTRIB_DIR)/moab - -# Use compiler info discovered by PETSC -ifeq ($(PETSC_ARCH),) - include $(PETSC_DIR)/$(PETSC_ARCH)/lib/petsc/conf/petscvariables -else - include $(PETSC_DIR)/lib/petsc/conf/petscvariables +# Check for optional dependencies and, if found, configure for building. +# Set default for ENABLE_CARDINAL, and override default CARDINAL_DIR for unit build. +ENABLE_CARDINAL := no +CARDINAL_DIR ?= $(CURDIR)/../cardinal +include $(CURDIR)/../config/check_deps.mk + +ifeq ($(ENABLE_CARDINAL),yes) + include $(CURDIR)/../config/configure_cardinal.mk endif # framework @@ -163,31 +99,6 @@ include $(MOOSE_DIR)/modules/modules.mk ADDITIONAL_INCLUDES := -I$(FRAMEWORK_DIR)/contrib/gtest ADDITIONAL_LIBS := $(FRAMEWORK_DIR)/contrib/gtest/libgtest.la -# Add OpenMC flags -ADDITIONAL_CPPFLAGS += $(HDF5_INCLUDES) $(OPENMC_INCLUDES) -libmesh_CXXFLAGS += -DENABLE_OPENMC_COUPLING - -# Add DagMC flags (-DDAGMC is used in OpenMC) -libmesh_CXXFLAGS += -DENABLE_DAGMC -DDAGMC - -# Configure and build MOAB, DagMC, and then OpenMC -include $(CARDINAL_DIR)/config/moab.mk -include $(CARDINAL_DIR)/config/dagmc.mk - -# autoconf-archive puts some arguments (e.g. -std=c++17) into the compiler -# variable rather than the compiler flags variable. -# -# cmake allows this, but wants any compiler arguments to be -# semicolon-separated, not space-separated -# libmesh_CC, etc., were defined in build.mk -space := $(subst ,, ) -LIBMESH_CC_LIST := $(subst $(space),;,$(libmesh_CC)) -LIBMESH_CXX_LIST := $(subst $(space),;,$(libmesh_CXX)) -LIBMESH_F90_LIST := $(subst $(space),;,$(libmesh_F90)) - -ENABLE_DAGMC := ON -include $(CARDINAL_DIR)/config/openmc.mk - # TMAP8 APPLICATION_DIR := $(TMAP8_DIR) APPLICATION_NAME := tmap8 @@ -196,16 +107,9 @@ GEN_REVISION := no include $(FRAMEWORK_DIR)/app.mk # Cardinal -APPLICATION_DIR := $(CARDINAL_DIR) -APPLICATION_NAME := cardinal -BUILD_EXEC := no -GEN_REVISION := yes -include $(FRAMEWORK_DIR)/app.mk - -# app_objects are defined in moose.mk and built according to the rules in build.mk -# We need to build these first so we get include dirs -$(app_objects): build_moab build_dagmc build_openmc -$(test_objects): build_moab build_dagmc build_openmc +ifeq ($(ENABLE_CARDINAL),yes) + include $(CURDIR)/../config/build_cardinal.mk +endif # FENIX CURRENT_DIR := $(shell pwd) @@ -219,20 +123,19 @@ APPLICATION_NAME := fenix-unit BUILD_EXEC := yes DEP_APPS ?= $(shell $(FRAMEWORK_DIR)/scripts/find_dep_apps.py $(APPLICATION_NAME)) -# Cardinal dependency libraries -ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ - -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \ - $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) +# Cardinal dependency libraries needed for FENIX linking +ifeq ($(ENABLE_CARDINAL),yes) + ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \ + -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \ + $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) +endif include $(FRAMEWORK_DIR)/app.mk -# Cardinal contrib flags used in app.mk targets -CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib $(BLASLAPACK_LIB) $(PETSC_EXTERNAL_LIB_BASIC) -L$(OPENMC_LIBDIR) -L$(HDF5_LIBDIR) -lopenmc -ldagmc -lMOAB $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) $(CC_LINKER_SLFLAG)$(HDF5_LIBDIR) - -# EXTERNAL_FLAGS is used in rules for app.mk -$(app_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) -$(app_test_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) -$(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS) +# External flags +ifeq ($(ENABLE_CARDINAL),yes) + include $(CURRENT_DIR)/../config/external_cardinal_flags.mk +endif # Find all the Fenix unit test source files and include their dependencies. fenix_unit_srcfiles := $(shell find $(CURRENT_DIR)/src -name "*.C") From 93cb186acaa8f4c1375e219b998bea0f347479a3 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Tue, 27 Aug 2024 16:50:17 -0600 Subject: [PATCH 24/28] Address review comment Default for Cardinal is changed to 'yes' instead of 'no' --- Makefile | 2 -- config/check_deps.mk | 8 ++++---- unit/Makefile | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 0ed3ff5f..d463ecd4 100644 --- a/Makefile +++ b/Makefile @@ -44,8 +44,6 @@ else endif # Check for optional dependencies and, if found, configure for building. -# Set default for ENABLE_CARDINAL. -ENABLE_CARDINAL := no include config/check_deps.mk ifeq ($(ENABLE_CARDINAL),yes) diff --git a/config/check_deps.mk b/config/check_deps.mk index 8f33440a..4406c6ed 100644 --- a/config/check_deps.mk +++ b/config/check_deps.mk @@ -3,16 +3,16 @@ define n endef -# Set default value for Cardinal +# Set default values for Cardinal location and ENABLE_CARDINAL. CARDINAL_DIR ?= $(CURDIR)/cardinal +ENABLE_CARDINAL := yes -# Check for CARDINAL_CONTENT within CARDINAL_DIR. This will cause Cardinal (and contribs) -# to either be built or skipped within the configure_cardinal.mk file. +# Check for CARDINAL_CONTENT within CARDINAL_DIR. CARDINAL_CONTENT := $(shell ls $(CARDINAL_DIR) 2> /dev/null) ifeq ($(CARDINAL_CONTENT),) $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source.") + ENABLE_CARDINAL := no else $(info FENIX is using Cardinal from $(CARDINAL_DIR)) - ENABLE_CARDINAL := yes endif diff --git a/unit/Makefile b/unit/Makefile index 197e4c93..d7cc872d 100644 --- a/unit/Makefile +++ b/unit/Makefile @@ -45,8 +45,7 @@ else endif # Check for optional dependencies and, if found, configure for building. -# Set default for ENABLE_CARDINAL, and override default CARDINAL_DIR for unit build. -ENABLE_CARDINAL := no +# Override default CARDINAL_DIR for unit build. CARDINAL_DIR ?= $(CURDIR)/../cardinal include $(CURDIR)/../config/check_deps.mk From 9fd6515981bd861a80b711868f818d3bfd3d2906 Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Wed, 28 Aug 2024 08:48:59 -0600 Subject: [PATCH 25/28] Apply suggestions from code review Co-authored-by: Pierre-Clement Simon --- Makefile | 1 + config/check_deps.mk | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d463ecd4..29def370 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ endif # Check for optional dependencies and, if found, configure for building. include config/check_deps.mk +# ENABLE_CARDINAL = yes by default, but is set to "no" automatically if Cardinal is not found via CARDINAL_DIR. ifeq ($(ENABLE_CARDINAL),yes) include config/configure_cardinal.mk endif diff --git a/config/check_deps.mk b/config/check_deps.mk index 4406c6ed..8e8318d8 100644 --- a/config/check_deps.mk +++ b/config/check_deps.mk @@ -11,7 +11,7 @@ ENABLE_CARDINAL := yes CARDINAL_CONTENT := $(shell ls $(CARDINAL_DIR) 2> /dev/null) ifeq ($(CARDINAL_CONTENT),) - $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source.") + $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source. $n In the meantime, FENIX will be built without Cardinal.") ENABLE_CARDINAL := no else $(info FENIX is using Cardinal from $(CARDINAL_DIR)) From f4de00e31ca583d277ba5fd514272385a3768af7 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 28 Aug 2024 10:38:44 -0600 Subject: [PATCH 26/28] Fixup intermittent documentation failure related to a vague link --- doc/content/verification_validation_examples/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/content/verification_validation_examples/index.md b/doc/content/verification_validation_examples/index.md index 5934b3e1..5801078b 100644 --- a/doc/content/verification_validation_examples/index.md +++ b/doc/content/verification_validation_examples/index.md @@ -1,6 +1,6 @@ # FENIX Verification, Validation, and Example Cases -For [software quality assurance (SQA)](sqa/index.md) purposes, FENIX undergoes verification, validation, and benchmarking. The FENIX development team defines these terms as: +For [software quality assurance (SQA)](sqa/index.md exact=True) purposes, FENIX undergoes verification, validation, and benchmarking. The FENIX development team defines these terms as: - +Verification+: Comparing FENIX predictions against analytical solutions in different conditions, which are often simple cases. - +Validation+: Comparing FENIX predictions against experimental data. From 4505a6686622416b1727e92eff7b53ef55740086 Mon Sep 17 00:00:00 2001 From: "Icenhour, Casey Tyler" Date: Wed, 28 Aug 2024 10:39:47 -0600 Subject: [PATCH 27/28] Add tip to installation instructions regarding disabling Cardinal This is also reflected in the HPC instructions. Finally, improve spacing for warning message when Cardinal is disabled. --- config/check_deps.mk | 2 +- doc/content/getting_started/installation.md | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/check_deps.mk b/config/check_deps.mk index 8e8318d8..a117d86a 100644 --- a/config/check_deps.mk +++ b/config/check_deps.mk @@ -11,7 +11,7 @@ ENABLE_CARDINAL := yes CARDINAL_CONTENT := $(shell ls $(CARDINAL_DIR) 2> /dev/null) ifeq ($(CARDINAL_CONTENT),) - $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source. $n In the meantime, FENIX will be built without Cardinal.") + $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source. $n$nIn the meantime, FENIX will be built without Cardinal.") ENABLE_CARDINAL := no else $(info FENIX is using Cardinal from $(CARDINAL_DIR)) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index 74e5a9d5..27243068 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -92,8 +92,8 @@ export OPENMC_CROSS_SECTIONS=~/projects/FENIX/cross_sections/cross_sections.xml !alert! warning This variable +must+ be set in your environment anytime you wish to run FENIX input files that -utilize the OpenMC functionality! This can be done either using `export` on the command line, -or placing this command within a shell config file (`.bashrc`, `.zshrc`, etc.). +utilize the OpenMC functionality within Cardinal! This can be done either using `export` on the +command line, or placing this command within a shell config file (`.bashrc`, `.zshrc`, etc.). !alert-end! !alert! note @@ -101,6 +101,13 @@ The copies of MOOSE, TMAP8, and Cardinal provided with FENIX have been fully tes FENIX version, and is guaranteed to work with all current FENIX tests. !alert-end! +!alert! tip title=Disabling Cardinal +Cardinal is an optional dependency of FENIX and can be disabled to speed up build times, if Cardinal +capabilities are not needed. This can be done by simply not downloading the Cardinal submodule +(i.e., not performing the `git submodule update --init cardinal` command as well as the Cardinal +dependency download commands above.) +!alert-end! + Once all dependencies have been downloaded, FENIX can be compiled and tested: ```bash From 8473f788a5b306ea7124659d7fe25eda773e529d Mon Sep 17 00:00:00 2001 From: Casey Icenhour Date: Wed, 28 Aug 2024 12:15:59 -0600 Subject: [PATCH 28/28] Apply suggestion from code review Co-authored-by: Pierre-Clement Simon --- doc/content/getting_started/installation.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/content/getting_started/installation.md b/doc/content/getting_started/installation.md index 27243068..3bd01517 100644 --- a/doc/content/getting_started/installation.md +++ b/doc/content/getting_started/installation.md @@ -190,6 +190,13 @@ git submodule update --recursive contrib/DAGMC git submodule update contrib/moab ``` +!alert! tip title=Disabling Cardinal +Cardinal is an optional dependency of FENIX and can be disabled to speed up build times, if Cardinal +capabilities are not needed. This can be done by simply not downloading the Cardinal submodule +(i.e., not performing the `git submodule update cardinal` command as well as the following Cardinal +dependency update commands above.) +!alert-end! + Both sets of instructions assume that your copy of FENIX is stored in `~/projects` and that the [idaholab/FENIX](https://github.com/idaholab/FENIX) git remote is labeled `upstream`. Use `git remote -v` in the FENIX repository location to check for this and change the commands as necessary. Finally, FENIX can be re-compiled and re-tested.