initial work

nci_environment/build_environment_module.py

@@ -9,11 +9,11 @@
  - (opt) Conda environment to create
  - (opt) Pip style requirements.txt to install to a directory
 
-It requires python 3.8+ and pyyaml.
+It requires python 3.9+ and pyyaml.
 Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI:
 New DEA-Env Module
   $ module use /g/data/v10/public/modules/modulefiles/
-  $ module load python3/3.8.5
+  $ module load python3/3.9.2
   # if pyyaml is not installed in gadi
   $ pip install PyYAML --user
 
@@ -22,7 +22,7 @@
 
 New DEA Module
   $ module use /g/data/v10/public/modules/modulefiles/
-  $ module load python3/3.8.5
+  $ module load python3/3.9.2
 
   $ # Building a new DEA Module
   $ python3 build_environment_module.py dea/modulespec.yaml
@@ -262,8 +262,8 @@ def copy_and_fill_templates(template_tasks, variables):
 
         src = Path(task["src"])
         dest = Path(task["dest"])
-        LOG.info("Copy and fill dea-env modulefile %s in %s", src, dest)
-        # Write the module file template to modulefiles/dea-env directory
+        LOG.info("Copy and fill dea modulefile %s in %s", src, dest)
+        # Write the module file template to modulefiles/dea directory
         write_template(src, variables, dest)
 
         if "chmod" in task:
@@ -361,18 +361,18 @@ def run_final_commands_on_module(commands, module_path):
         run_command(cmd)
 
 
-def include_stable_module_dep_versions(config):
-    """
-    Include stable module dependency versions
+# def include_stable_module_dep_versions(config):
+#     """
+#     Include stable module dependency versions
 
-    :param config: Dictionary of configuration variables
-    :return: None
-    """
-    stable_module_deps = config.get("stable_module_deps", [])
-    for dep in stable_module_deps:
-        default_version = find_default_version(dep)
-        dep = dep.replace("-", "_")
-        config["variables"][f"fixed_{dep}"] = default_version
+#     :param config: Dictionary of configuration variables
+#     :return: None
+#     """
+#     stable_module_deps = config.get("stable_module_deps", [])
+#     for dep in stable_module_deps:
+#         default_version = find_default_version(dep)
+#         dep = dep.replace("-", "_")
+#         config["variables"][f"fixed_{dep}"] = default_version
 
 
 def main(config_path):
@@ -394,7 +394,7 @@ def main(config_path):
     if "module_version" not in variables:
         variables["module_version"] = date()
     include_templated_vars(config)
-    include_stable_module_dep_versions(config)
+    # include_stable_module_dep_versions(config)
 
     pre_check(config)
     prep(config_path)

nci_environment/dea/modulefile.template

@@ -16,7 +16,35 @@ set base          ${module_path}
 
 module-whatis   "${module_description} ${module_version}"
 
-module load ${fixed_dea_env}
+# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html
+setenv PYTHONFAULTHANDLER 1
+
+# Set PYTHONUSERBASE based on the version of dea module.
+# This allows users to install python packages with "pip install --user <package>",
+setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local
+setenv GDAL_DATA ${module_path}/share/gdal
+setenv PROJ_LIB ${module_path}/share/proj
+setenv PROJ_NETWORK ON
+setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml
+setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas
+
+if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} {
+        puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))"
+        puts stderr "Unload all python modules, if you experience any issues."
+}
+
+prepend-path PYTHONPATH ${module_path}/share/qgis/python
+
+# Remove duplicate entries for module path dir and prepend again
+remove-path PATH ${module_path}/bin
+prepend-path PATH ${module_path}/bin
+
+# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH
+# variables to point to a directory based on the Environment Module version which is loaded so that extra
+# packages must be re-installed when a new dea module is released
+# Remove duplicate entries for HOME dir and prepend again
+remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin
+prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin
 
 setenv DATACUBE_CONFIG_PATH ${module_path}/datacube.conf
 setenv LC_ALL en_AU.utf8
@@ -26,19 +54,9 @@ if {[module-info mode load] && [is-loaded $$name/$$version]} {
         puts stderr "Warning: $$name/$$version exists in the python env ($$env(PYTHONPATH))"
 }
 
-# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH
-# variables to point to a directory based on the Environment Module version which is loaded so that extra
-# packages must be re-installed when a new dea-env module is released
-prepend-path PATH ${module_path}/bin
-prepend-path PYTHONPATH ${python_path}
-
-# Remove duplicate entries for HOME dir and prepend at the top
-remove-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin
-prepend-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin
-
 # Remove duplicate entries for HOME dir and prepend at the top
-remove-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages
-prepend-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages
+remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages
+prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages
 
 
 #############################################################

nci_environment/dea/modulespec.yaml

@@ -1,23 +1,25 @@
 variables:
   module_name: dea
   module_description: DEA tools for the NCI
-  modules_base: "/g/data/v10/public/modules"
+  modules_dir: "/g/data/v10/public/modules"
+  conda_path: "/g/data/v10/private/mambaforge/bin/mamba"
   dbhost: dea-db.nci.org.au
   dbport: 6432
-  python_version: 3.8
+  python_version: 3.9
 
 templated_variables:
-  module_path: "{modules_base}/{module_name}/{module_version}"
-  python_path: "{modules_base}/{module_name}/{module_version}/lib/python{python_version}/site-packages/"
+  module_path: "{modules_dir}/{module_name}/{module_version}"
+  python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/"
   dea_module: "{module_name}/{module_version}"
 
-stable_module_deps:
-- dea-env
+install_conda_packages: ../../docker/env.yaml
 
 install_pip_packages:
-  pip_cmd: "module load {fixed_dea_env}; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt"
+  pip_cmd: "module load python3/3.9.2; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt"
 
 copy_files:
+- src: ../../docker/env.yaml
+  dest: "{modules_dir}/{module_name}/{module_version}/environment.yaml"
 - src: requirements.txt
   dest: "{module_path}/requirements.txt"
 - src: datacube.conf
@@ -39,4 +41,5 @@ wget_files:
 
 template_files:
 - src: modulefile.template
-  dest: "{modules_base}/modulefiles/{module_name}/{module_version}"
+  dest: "{modules_dir}/modulefiles/{module_name}/{module_version}"
+  chmod: 0o444