idaholab · cticenhour · Aug 29, 2024 · May 30, 2024 · Jun 27, 2024 · Jun 27, 2024
diff --git a/.gitignore b/.gitignore
@@ -325,3 +325,6 @@ fenix-unit.yaml
 /modules/fsi/unit/fsi-unit.yaml
 /tutorials/tutorial01_app_development/*/babbler.yaml
 /tutorials/darcy_thermo_mech/*/darcy_thermo_mech.yaml
+
+# Cardinal
+cross_sections
diff --git a/.gitmodules b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "tmap8"]
 	path = tmap8
 	url = ../../idaholab/tmap8.git
+[submodule "cardinal"]
+	path = cardinal
+	url = ../../neams-th-coe/cardinal.git
diff --git a/Makefile b/Makefile
@@ -1,19 +1,38 @@
-###############################################################################
-################### MOOSE Application Standard Makefile #######################
-###############################################################################
+####################################################################################
+################################## FENIX Makefile ##################################
+####################################################################################
 #
-# Optional Environment variables
-# MOOSE_DIR        - Root directory of the MOOSE project
-# TMAP8_DIR        - Root directory of the TMAP8 project
+# Optional Environment variables:
 #
-###############################################################################
+# MOOSE_DIR        - Root directory of MOOSE
+# TMAP8_DIR        - Root directory of TMAP8
+# CARDINAL_DIR     - Root directory of Cardinal
+# OPENMC_DIR       - Root directory of OpenMC
+# DAGMC_DIR        - Root directory of DagMC
+# MOAB_DIR         - Root directory of MOAB
+#
+# EIGEN3_DIR       - Root directory of Eigen3 (should contain FindEigen3.cmake).
+#                    This is needed for DagMC.
+#
+# OpenMC uses HDF5; below are influential environment variables for that
+# installation. None of these need to be set if HDF5 is being pulled from
+# a manual PETSc installation, and HDF5_DIR is provided directly from conda.
+#
+# HDF5_DIR         - Root directory of HDF5
+# HDF5_INCLUDE_DIR - Root directory for HDF5 headers (default: $(HDF5_DIR)/include)
+# HDF5_LIBDIR      - Root directory for HDF5 libraries (default: $(HDF5_DIR)/lib)
+# PETSC_DIR        - Root directory for PETSc (default: $(MOOSE_DIR)/petsc)
+# PETSC_ARCH       - PETSc architecture (default: arch-moose)
+####################################################################################
 # Use the MOOSE submodule if it exists and MOOSE_DIR is not set
-MOOSE_SUBMODULE    := $(CURDIR)/moose
+# If it doesn't exist, and MOOSE_DIR is not set, then look for it adjacent to the application
+MOOSE_SUBMODULE       := $(CURDIR)/moose
 ifneq ($(wildcard $(MOOSE_SUBMODULE)/framework/Makefile),)
-  MOOSE_DIR        ?= $(MOOSE_SUBMODULE)
+  MOOSE_DIR           ?= $(MOOSE_SUBMODULE)
 else
-  MOOSE_DIR        ?= $(shell dirname `pwd`)/moose
+  MOOSE_DIR           ?= $(shell dirname `pwd`)/moose
 endif
+FRAMEWORK_DIR      := $(MOOSE_DIR)/framework
 
 # Use the TMAP8 submodule if it exists and TMAP8_DIR is not set
 # If it doesn't exist, and TMAP8_DIR is not set, then look for it adjacent to the application
@@ -24,8 +43,15 @@ else
   TMAP8_DIR        ?= $(shell dirname `pwd`)/tmap8
 endif
 
+# Check for optional dependencies and, if found, configure for building.
+include config/check_deps.mk
+
+# ENABLE_CARDINAL = yes by default, but is set to "no" automatically if Cardinal is not found via CARDINAL_DIR. 
+ifeq ($(ENABLE_CARDINAL),yes)
+  include config/configure_cardinal.mk
+endif
+
 # framework
-FRAMEWORK_DIR      := $(MOOSE_DIR)/framework
 include $(FRAMEWORK_DIR)/build.mk
 include $(FRAMEWORK_DIR)/moose.mk
 
@@ -73,12 +99,27 @@ BUILD_EXEC         := no
 GEN_REVISION       := no
 include            $(FRAMEWORK_DIR)/app.mk
 
-# dep apps
+# Cardinal
+ifeq ($(ENABLE_CARDINAL),yes)
+  include config/build_cardinal.mk
+endif
+
+# FENIX
 APPLICATION_DIR    := $(CURDIR)
 APPLICATION_NAME   := fenix
 BUILD_EXEC         := yes
-GEN_REVISION       := no
+GEN_REVISION       := yes
+
+# Cardinal dependency libraries needed for FENIX linking
+ifeq ($(ENABLE_CARDINAL),yes)
+  ADDITIONAL_LIBS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib \
+                     -L$(OPENMC_LIBDIR) -lopenmc -lhdf5_hl -ldagmc -lMOAB \
+                     $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR)
+endif
+
 include            $(FRAMEWORK_DIR)/app.mk
 
-###############################################################################
-# Additional special case targets should be added here
+# External flags
+ifeq ($(ENABLE_CARDINAL),yes)
+  include config/external_cardinal_flags.mk
+endif
diff --git a/apptainer/README.md b/apptainer/README.md
@@ -0,0 +1,7 @@
+Contents in this directory are used for the generation of apptainer images using the `ApptainerGenerator` script found in MOOSE at `scripts/apptainer_generator.py`.
+
+In particular:
+
+- `app_envrionment`: Appends to the environment in the container to add `OPENMC_CROSS_SECTIONS`
+- `app_post_post_install`: Copies the cross section download script for use in testing
+- `app_post_pre_make`: Sets extra variables for the build and builds the extra dependencies (openmc, dagmc) before the main build so that the `MAKEFLAGS` variable can be set for parallel builds.
diff --git a/apptainer/app_environment b/apptainer/app_environment
@@ -0,0 +1 @@
+export OPENMC_CROSS_SECTIONS=/opt/fenix/share/fenix/endfb-vii.1-hdf5/cross_sections.xml
diff --git a/apptainer/app_post_post_install b/apptainer/app_post_post_install
@@ -0,0 +1,7 @@
+# Download cross sections
+umask 022
+FROM_XS_SCRIPT=${APPLICATION_DIR}/cardinal/scripts/download-openmc-cross-sections.sh
+XS_SCRIPT=${MOOSE_PREFIX}/share/${BINARY_NAME}/download-openmc-cross-sections.sh
+cp ${FROM_XS_SCRIPT} ${XS_SCRIPT}
+chmod +x ${XS_SCRIPT}
+${XS_SCRIPT} ${MOOSE_PREFIX}/share/fenix
diff --git a/apptainer/app_post_pre_make b/apptainer/app_post_pre_make
@@ -0,0 +1,5 @@
+export CARDINAL_CONTRIB_INSTALL_DIR=/opt/fenix/contrib
+export HDF5_ROOT=${PETSC_DIR}
+export ENABLE_DAGMC=ON
+make build_openmc MAKEFLAGS=-j${MOOSE_JOBS}
+make build_dagmc MAKEFLAGS=-j${MOOSE_JOBS}
diff --git a/cardinal b/cardinal
diff --git a/config/build_cardinal.mk b/config/build_cardinal.mk
@@ -0,0 +1,37 @@
+# Add OpenMC flags
+ADDITIONAL_CPPFLAGS += $(HDF5_INCLUDES) $(OPENMC_INCLUDES)
+libmesh_CXXFLAGS    += -DENABLE_OPENMC_COUPLING
+
+# Add DagMC flags (-DDAGMC is used in OpenMC)
+libmesh_CXXFLAGS    += -DENABLE_DAGMC -DDAGMC
+
+# Configure and build MOAB, DagMC, and then OpenMC
+include          $(CARDINAL_DIR)/config/moab.mk
+include          $(CARDINAL_DIR)/config/dagmc.mk
+
+# autoconf-archive puts some arguments (e.g. -std=c++17) into the compiler
+# variable rather than the compiler flags variable.
+#
+# cmake allows this, but wants any compiler arguments to be
+# semicolon-separated, not space-separated
+# libmesh_CC, etc., were defined in build.mk
+space := $(subst ,, )
+LIBMESH_CC_LIST := $(subst $(space),;,$(libmesh_CC))
+LIBMESH_CXX_LIST := $(subst $(space),;,$(libmesh_CXX))
+LIBMESH_F90_LIST := $(subst $(space),;,$(libmesh_F90))
+
+ENABLE_DAGMC     := ON
+include            $(CARDINAL_DIR)/config/openmc.mk
+
+# Cardinal
+libmesh_CXXFLAGS   += -DENABLE_CARDINAL
+APPLICATION_DIR    := $(CARDINAL_DIR)
+APPLICATION_NAME   := cardinal
+BUILD_EXEC         := no
+GEN_REVISION       := yes
+include            $(FRAMEWORK_DIR)/app.mk
+
+# app_objects are defined in moose.mk and built according to the rules in build.mk
+# We need to build these first so we get include dirs
+$(app_objects): build_moab build_dagmc build_openmc
+$(test_objects): build_moab build_dagmc build_openmc
diff --git a/config/check_deps.mk b/config/check_deps.mk
@@ -0,0 +1,18 @@
+define n
+
+
+endef
+
+# Set default values for Cardinal location and ENABLE_CARDINAL.
+CARDINAL_DIR         ?= $(CURDIR)/cardinal
+ENABLE_CARDINAL      := yes
+
+# Check for CARDINAL_CONTENT within CARDINAL_DIR.
+CARDINAL_CONTENT     := $(shell ls $(CARDINAL_DIR) 2> /dev/null)
+
+ifeq ($(CARDINAL_CONTENT),)
+  $(warning $n"Cardinal does not seem to be available. If usage of Cardinal is desired within FENIX, make sure that either the submodule is checked out$nor that CARDINAL_DIR points to a location with the Cardinal source. $n$nIn the meantime, FENIX will be built without Cardinal.")
+  ENABLE_CARDINAL    := no
+else
+  $(info FENIX is using Cardinal from     $(CARDINAL_DIR))
+endif
diff --git a/config/configure_cardinal.mk b/config/configure_cardinal.mk
@@ -0,0 +1,64 @@
+# Cardinal contrib variables
+CARDINAL_CONTRIB_DIR           := $(CARDINAL_DIR)/contrib
+CARDINAL_CONTRIB_INSTALL_DIR   ?= $(CARDINAL_DIR)/install
+OPENMC_INSTALL_DIR             := $(CARDINAL_CONTRIB_INSTALL_DIR)
+OPENMC_LIBDIR                  := $(OPENMC_INSTALL_DIR)/lib
+OPENMC_BUILDDIR                := $(CARDINAL_DIR)/build/openmc
+OPENMC_INSTALL_DIR             := $(CARDINAL_CONTRIB_INSTALL_DIR)
+OPENMC_INCLUDES                := -I$(OPENMC_INSTALL_DIR)/include
+OPENMC_LIBDIR                  := $(OPENMC_INSTALL_DIR)/lib
+OPENMC_LIB                     := $(OPENMC_LIBDIR)/libopenmc.so
+
+PETSC_DIR           ?= $(MOOSE_DIR)/petsc
+PETSC_ARCH          ?= arch-moose
+LIBMESH_DIR         ?= $(MOOSE_DIR)/libmesh/installed/
+
+# This is the Eigen3 location on CIVET. If you are using MOOSE's conda environment,
+# you don't need to set these variables, because conda sets them for you. The only
+# scenario where you might need to manually set these is if you're not using the
+# conda environment. You will get a compile error about FindEigen3.cmake if you
+# do indeed need to set these.
+EIGEN3_DIR          ?= $(LIBMESH_DIR)/include
+Eigen3_DIR          ?= $(EIGEN3_DIR)
+
+# If HDF5_DIR is set, use those settings to link HDF5 to OpenMC.
+# Otherwise, use the PETSc location, as the self-built HDF5 libraries will be housed within.
+ifeq ($(HDF5_DIR),)
+  HDF5_DIR          := $(PETSC_DIR)
+  export HDF5_DIR
+endif
+
+# HDF5_ROOT is used in makefiles for the Cardinal third-party dependencies.
+# Set it using HDF5_DIR
+HDF5_ROOT          := $(HDF5_DIR)
+export HDF5_ROOT
+
+HDF5_INCLUDE_DIR    ?= $(HDF5_DIR)/include
+HDF5_LIBDIR         ?= $(HDF5_DIR)/lib
+# This is used in $(FRAMEWORK_DIR)/build.mk
+HDF5_INCLUDES       := -I$(HDF5_INCLUDE_DIR) -I$(HDF5_DIR)/include
+
+# BUILD_TYPE will be passed to CMake via CMAKE_BUILD_TYPE
+ifeq ($(METHOD),dbg)
+  BUILD_TYPE := Debug
+else
+  BUILD_TYPE := Release
+endif
+
+DAGMC_BUILDDIR := $(CARDINAL_DIR)/build/DAGMC
+DAGMC_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR)
+
+MOAB_BUILDDIR := $(CARDINAL_DIR)/build/moab
+MOAB_INSTALL_DIR := $(CARDINAL_CONTRIB_INSTALL_DIR)
+
+# Set default values for all third party dependencies
+OPENMC_DIR          ?= $(CARDINAL_CONTRIB_DIR)/openmc
+DAGMC_DIR           ?= $(CARDINAL_CONTRIB_DIR)/DAGMC
+MOAB_DIR            ?= $(CARDINAL_CONTRIB_DIR)/moab
+
+# Use compiler info discovered by PETSC
+ifeq ($(PETSC_ARCH),)
+  include $(PETSC_DIR)/$(PETSC_ARCH)/lib/petsc/conf/petscvariables
+else
+  include $(PETSC_DIR)/lib/petsc/conf/petscvariables
+endif
diff --git a/config/external_cardinal_flags.mk b/config/external_cardinal_flags.mk
@@ -0,0 +1,9 @@
+ifeq ($(ENABLE_CARDINAL),yes)
+  # Cardinal contrib flags used in app.mk targets
+  CARDINAL_EXTERNAL_FLAGS := -L$(CARDINAL_DIR)/lib $(CC_LINKER_SLFLAG)$(CARDINAL_DIR)/lib $(BLASLAPACK_LIB) $(PETSC_EXTERNAL_LIB_BASIC) -L$(OPENMC_LIBDIR)  -L$(HDF5_LIBDIR) -lopenmc -ldagmc -lMOAB $(CC_LINKER_SLFLAG)$(OPENMC_LIBDIR) $(CC_LINKER_SLFLAG)$(HDF5_LIBDIR)
+
+  # EXTERNAL_FLAGS is used in rules for app.mk
+  $(app_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS)
+  $(app_test_LIB): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS)
+  $(app_EXEC): EXTERNAL_FLAGS := $(CARDINAL_EXTERNAL_FLAGS)
+endif
diff --git a/doc/cardinal_nek_exclude.yml b/doc/cardinal_nek_exclude.yml
@@ -0,0 +1,35 @@
+- source/**
+- media/**
+- ~source/auxkernels/NekSpatialBinComponentAux.md
+- ~source/timesteppers/NekTimeStepper.md
+- ~source/mesh/NekRSMesh.md
+- ~source/postprocessors/boundary_specs.md
+- ~source/postprocessors/NekHeatFluxIntegral.md
+- ~source/postprocessors/NekNumRanks.md
+- ~source/postprocessors/NekMassFluxWeightedSideAverage.md
+- ~source/postprocessors/NekMassFluxWeightedSideIntegral.md
+- ~source/postprocessors/NekPressureSurfaceForce.md
+- ~source/postprocessors/NekScalarValuePostprocessor.md
+- ~source/postprocessors/NekSideAverage.md
+- ~source/postprocessors/NekSideExtremeValue.md
+- ~source/postprocessors/NekSideIntegral.md
+- ~source/postprocessors/NekUsrWrkBoundaryIntegral.md
+- ~source/postprocessors/NekVolumeAverage.md
+- ~source/postprocessors/NekVolumeExtremeValue.md
+- ~source/postprocessors/NekVolumeIntegral.md
+- ~source/postprocessors/PecletNumber.md
+- ~source/postprocessors/ReynoldsNumber.md
+- ~source/problems/NekRSProblem.md
+- ~source/problems/NekRSSeparateDomainProblem.md
+- ~source/problems/nek_classes.md
+- ~source/problems/nondimensional_problem.md
+- ~source/problems/output_solution.md
+- ~source/problems/NekRSStandaloneProblem.md
+- ~source/userobjects/NekBinnedPlaneAverage.md
+- ~source/userobjects/NekBinnedPlaneIntegral.md
+- ~source/userobjects/NekBinnedSideAverage.md
+- ~source/userobjects/NekBinnedSideIntegral.md
+- ~source/userobjects/NekBinnedVolumeAverage.md
+- ~source/userobjects/NekBinnedVolumeIntegral.md
+- ~source/userobjects/NekScalarValue.md
+- ~source/userobjects/velocity_component.md
diff --git a/doc/config.yml b/doc/config.yml
@@ -54,6 +54,9 @@ Content:
             - python/TestHarness.md
     tmap8:
         root_dir: ${ROOT_DIR}/tmap8/doc/content
+    cardinal:
+        root_dir: ${ROOT_DIR}/cardinal/doc/content
+        content: !include ${ROOT_DIR}/doc/cardinal_nek_exclude.yml
 
 Renderer:
     type: MooseDocs.base.MaterializeRenderer
@@ -86,11 +89,16 @@ Extensions:
         executable: ${ROOT_DIR}
         remove:
             framework: !include ${MOOSE_DIR}/framework/doc/remove.yml
+            cardinal:
+                - /ICs/BulkEnergyConservationIC
+                - /Cardinal/ICs/BulkEnergyConservation/BulkEnergyConservationICAction
+                - /Cardinal/ICs/VolumetricHeatSource/VolumetricHeatSourceICAction
     MooseDocs.extensions.common:
         shortcuts: !include ${MOOSE_DIR}/framework/doc/globals.yml
     MooseDocs.extensions.acronym:
         acronyms:
             framework: !include ${MOOSE_DIR}/framework/doc/acronyms.yml
+            cardinal: !include ${ROOT_DIR}/cardinal/doc/acronyms.yml
             fenix:
                 FENIX: Fusion ENergy Integrated multiphys-X
                 PIC: particle-in-cell
@@ -117,6 +125,7 @@ Extensions:
             stochastic_tools: !include ${MOOSE_DIR}/modules/stochastic_tools/doc/sqa_stochastic_tools.yml
             thermal_hydraulics: !include ${MOOSE_DIR}/modules/thermal_hydraulics/doc/sqa_thermal_hydraulics.yml
             tmap8: !include ${ROOT_DIR}/tmap8/doc/sqa_tmap8.yml
+            #cardinal: !include ${ROOT_DIR}/cardinal/doc/sqa_cardinal.yml
         reports: !include ${ROOT_DIR}/doc/sqa_reports.yml
         repos:
             default: https://github.com/idaholab/fenix