From 0c31644f849a83c808b641a91c748207e1ae9b47 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Thu, 16 Nov 2023 05:57:16 +0000 Subject: [PATCH] initial work --- nci_environment/build_environment_module.py | 34 ++++++++-------- nci_environment/dea/modulefile.template | 44 +++++++++++++++------ nci_environment/dea/modulespec.yaml | 19 +++++---- 3 files changed, 59 insertions(+), 38 deletions(-) diff --git a/nci_environment/build_environment_module.py b/nci_environment/build_environment_module.py index 98750fe0..e2c0c404 100755 --- a/nci_environment/build_environment_module.py +++ b/nci_environment/build_environment_module.py @@ -9,11 +9,11 @@ - (opt) Conda environment to create - (opt) Pip style requirements.txt to install to a directory -It requires python 3.8+ and pyyaml. +It requires python 3.9+ and pyyaml. Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI: New DEA-Env Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.9.2 # if pyyaml is not installed in gadi $ pip install PyYAML --user @@ -22,7 +22,7 @@ New DEA Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.9.2 $ # Building a new DEA Module $ python3 build_environment_module.py dea/modulespec.yaml @@ -262,8 +262,8 @@ def copy_and_fill_templates(template_tasks, variables): src = Path(task["src"]) dest = Path(task["dest"]) - LOG.info("Copy and fill dea-env modulefile %s in %s", src, dest) - # Write the module file template to modulefiles/dea-env directory + LOG.info("Copy and fill dea modulefile %s in %s", src, dest) + # Write the module file template to modulefiles/dea directory write_template(src, variables, dest) if "chmod" in task: @@ -361,18 +361,18 @@ def run_final_commands_on_module(commands, module_path): run_command(cmd) -def include_stable_module_dep_versions(config): - """ - Include stable module dependency versions +# def include_stable_module_dep_versions(config): +# """ +# Include stable module dependency versions - :param config: Dictionary of configuration variables - :return: None - """ - stable_module_deps = config.get("stable_module_deps", []) - for dep in stable_module_deps: - default_version = find_default_version(dep) - dep = dep.replace("-", "_") - config["variables"][f"fixed_{dep}"] = default_version +# :param config: Dictionary of configuration variables +# :return: None +# """ +# stable_module_deps = config.get("stable_module_deps", []) +# for dep in stable_module_deps: +# default_version = find_default_version(dep) +# dep = dep.replace("-", "_") +# config["variables"][f"fixed_{dep}"] = default_version def main(config_path): @@ -394,7 +394,7 @@ def main(config_path): if "module_version" not in variables: variables["module_version"] = date() include_templated_vars(config) - include_stable_module_dep_versions(config) + # include_stable_module_dep_versions(config) pre_check(config) prep(config_path) diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template index d451542e..abf61993 100644 --- a/nci_environment/dea/modulefile.template +++ b/nci_environment/dea/modulefile.template @@ -16,7 +16,35 @@ set base ${module_path} module-whatis "${module_description} ${module_version}" -module load ${fixed_dea_env} +# Enable segfault tracebacks in py3. https://docs.python.org/3/library/faulthandler.html +setenv PYTHONFAULTHANDLER 1 + +# Set PYTHONUSERBASE based on the version of dea module. +# This allows users to install python packages with "pip install --user ", +setenv PYTHONUSERBASE ~/.dea-sandbox/${module_name}/${module_version}/local +setenv GDAL_DATA ${module_path}/share/gdal +setenv PROJ_LIB ${module_path}/share/proj +setenv PROJ_NETWORK ON +setenv UDUNITS2_XML_PATH ${module_path}/share/udunits/udunits2.xml +setenv GSETTINGS_SCHEMA_DIR ${module_path}/share/glib-2.0/schemas + +if {[module-info mode load] && [info exists env(PYTHONPATH)] && $$env(PYTHONPATH) != ""} { + puts stderr "Warning: ${module_name}/${module_version} exists in the python env ($$env(PYTHONPATH))" + puts stderr "Unload all python modules, if you experience any issues." +} + +prepend-path PYTHONPATH ${module_path}/share/qgis/python + +# Remove duplicate entries for module path dir and prepend again +remove-path PATH ${module_path}/bin +prepend-path PATH ${module_path}/bin + +# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH +# variables to point to a directory based on the Environment Module version which is loaded so that extra +# packages must be re-installed when a new dea module is released +# Remove duplicate entries for HOME dir and prepend again +remove-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin +prepend-path PATH ~/.dea-sandbox/${module_name}/${module_version}/local/bin setenv DATACUBE_CONFIG_PATH ${module_path}/datacube.conf setenv LC_ALL en_AU.utf8 @@ -26,19 +54,9 @@ if {[module-info mode load] && [is-loaded $$name/$$version]} { puts stderr "Warning: $$name/$$version exists in the python env ($$env(PYTHONPATH))" } -# To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH -# variables to point to a directory based on the Environment Module version which is loaded so that extra -# packages must be re-installed when a new dea-env module is released -prepend-path PATH ${module_path}/bin -prepend-path PYTHONPATH ${python_path} - -# Remove duplicate entries for HOME dir and prepend at the top -remove-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin -prepend-path PATH ~/.dea-sandbox/${fixed_dea_env}/local/bin - # Remove duplicate entries for HOME dir and prepend at the top -remove-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages -prepend-path PYTHONPATH ~/.dea-sandbox/${fixed_dea_env}/local/lib/python3.6/site-packages +remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages +prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages ############################################################# diff --git a/nci_environment/dea/modulespec.yaml b/nci_environment/dea/modulespec.yaml index 42919bab..09ac26f8 100644 --- a/nci_environment/dea/modulespec.yaml +++ b/nci_environment/dea/modulespec.yaml @@ -1,23 +1,25 @@ variables: module_name: dea module_description: DEA tools for the NCI - modules_base: "/g/data/v10/public/modules" + modules_dir: "/g/data/v10/public/modules" + conda_path: "/g/data/v10/private/mambaforge/bin/mamba" dbhost: dea-db.nci.org.au dbport: 6432 - python_version: 3.8 + python_version: 3.9 templated_variables: - module_path: "{modules_base}/{module_name}/{module_version}" - python_path: "{modules_base}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" + module_path: "{modules_dir}/{module_name}/{module_version}" + python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" dea_module: "{module_name}/{module_version}" -stable_module_deps: -- dea-env +install_conda_packages: ../../docker/env.yaml install_pip_packages: - pip_cmd: "module load {fixed_dea_env}; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" + pip_cmd: "module load python3/3.9.2; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" copy_files: +- src: ../../docker/env.yaml + dest: "{modules_dir}/{module_name}/{module_version}/environment.yaml" - src: requirements.txt dest: "{module_path}/requirements.txt" - src: datacube.conf @@ -39,4 +41,5 @@ wget_files: template_files: - src: modulefile.template - dest: "{modules_base}/modulefiles/{module_name}/{module_version}" + dest: "{modules_dir}/modulefiles/{module_name}/{module_version}" + chmod: 0o444 \ No newline at end of file