diff --git a/docker/env.yaml b/docker/env.yaml index 88a2b19..35e7582 100644 --- a/docker/env.yaml +++ b/docker/env.yaml @@ -55,6 +55,7 @@ dependencies: - distributed - docutils - ephem + - eodatasets3 - fiona - Flask - Flask-Babel @@ -92,6 +93,9 @@ dependencies: - nodejs - numexpr - numpy + - odc-algo + - odc-dscache + - odc-io - ordered-set - packaging - pandas diff --git a/docker/requirements.txt b/docker/requirements.txt index 698318d..1a2c2f6 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -9,7 +9,6 @@ jupyter-nbextensions-configurator # ODC/DEA: these are installed in builder stage otps -eodatasets3 # Dale's s2cloudmask # https://github.com/daleroberts/s2cloudmask @@ -19,10 +18,7 @@ opencv-python-headless opencv-contrib-python-headless datacube[performance,s3] -odc-algo odc-cloud[ASYNC] -odc-dscache -odc-io odc-stac odc-stats[ows] odc-ui diff --git a/nci_environment/README.md b/nci_environment/README.md index 743e859..cf334ab 100644 --- a/nci_environment/README.md +++ b/nci_environment/README.md @@ -2,25 +2,22 @@ These scripts are used to update and deploy DEA modules on NCI. -There are two modules, with date-based version numbers: +The *dea* module, with a date-based version number, contains third party dependencies +of all of the DEA code, installed via a `conda` environment. This `conda` environment is the same +as the one for dea-sandbox. - 1. The Python Environment module *dea-env* - - * This contains third party dependencies of all of the DEA code, installed via - a `conda` environment. - - 2. A *dea* module, which depends on the _environment module_: +Additionally, the environment includes: * [Open Data Cube Core](https://github.com/opendatacube/datacube-core/) - * [EO Datasets](https://github.com/GeoscienceAustralia/eo-datasets/) - * [Digital Earth AU](https://github.com/GeoscienceAustralia/digitalearthau/) * [Data Cube Stats](https://github.com/GeoscienceAustralia/datacube-stats/) * [Fractional Cover](https://github.com/GeoscienceAustralia/fc/) + * [Water Observation From Space](https://github.com/GeoscienceAustralia/wofs) + * Creates users accounts in the Production Database the first time it is loaded by a user. @@ -34,14 +31,11 @@ There are two modules, with date-based version numbers: This will load the latest version of `dea/` module. -It will also load `dea-env/` which contains all of the software -dependencies for using DEA. - ## Notes -Loading these module might conflict with other python modules you have loaded. +Loading this module might conflict with other python modules you have loaded. -The `dea-env` module will prevent conflicts with locally installed python packages by +The `dea` module will prevent conflicts with locally installed python packages by changing `PYTHONUSERBASE` for each release; pip install --user @@ -54,33 +48,23 @@ It includes a config file, which it specifies by setting the # Maintainer Instructions -Only run these scripts from Raijin. We've seen filesystem sync issues when +Only run these scripts from Gadi. We've seen filesystem sync issues when run from VDI. - module load python3/3.8.5 + module load python3/3.10.4 pip3 install --user pyyaml jinja2 -## Building a new _Environment Module_ - -It requires python 3.8+ and pyyaml. Run the following on raijin at the NCI: - - $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 - $ ./build_environment_module.py dea-env/modulespec.yaml - -This will build a new environment module for today. - -The module version number is the current date in format YYYYMMDD, as it is a snapshot -of all of our pip/conda dependencies on that date. - ## Building a new _DEA Module_ -A DEA module will specify one exact environment module. +It requires python 3.10+ and pyyaml. Run the following on gadi at the NCI: $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.8.5 + $ module load python3/3.10.4 $ ./build_environment_module.py dea/modulespec.yaml +The module version number is the current date in format YYYYMMDD, as it is a snapshot +of all of our pip/conda dependencies on that date. + ## Updating the Default Version Once a module has been tested and approved, it can be made the default. @@ -108,14 +92,13 @@ Eg. For `dea` this is: `/g/data/v10/public/modules/modulefiles/dea/.version` ## Setup Copy the 3 lines below and modify the VERSION value - to the dea and dea-env module version you would + to the dea module version you would like the tests to be run on. Paste them in a brand new shell session/terminal VERSION="20230710" module use /g/data/v10/public/modules/modulefiles module load dea/$VERSION - module load dea-env/$VERSION ## Execution On gadi, just run the tests with in this fashion: diff --git a/nci_environment/build_environment_module.py b/nci_environment/build_environment_module.py index e2c0c40..c70cafc 100755 --- a/nci_environment/build_environment_module.py +++ b/nci_environment/build_environment_module.py @@ -9,21 +9,13 @@ - (opt) Conda environment to create - (opt) Pip style requirements.txt to install to a directory -It requires python 3.9+ and pyyaml. +It requires python 3.10+ and pyyaml. Use a qsub interactive copyq job on raijin with sufficient memory to run the following commands at the NCI: -New DEA-Env Module $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.9.2 + $ module load python3/3.10.4 # if pyyaml is not installed in gadi $ pip install PyYAML --user - $ # Building a new Environment Module: - $ python3 build_environment_module.py dea-env/modulespec.yaml - -New DEA Module - $ module use /g/data/v10/public/modules/modulefiles/ - $ module load python3/3.9.2 - $ # Building a new DEA Module $ python3 build_environment_module.py dea/modulespec.yaml @@ -361,20 +353,6 @@ def run_final_commands_on_module(commands, module_path): run_command(cmd) -# def include_stable_module_dep_versions(config): -# """ -# Include stable module dependency versions - -# :param config: Dictionary of configuration variables -# :return: None -# """ -# stable_module_deps = config.get("stable_module_deps", []) -# for dep in stable_module_deps: -# default_version = find_default_version(dep) -# dep = dep.replace("-", "_") -# config["variables"][f"fixed_{dep}"] = default_version - - def main(config_path): """ Build new environment module @@ -394,7 +372,6 @@ def main(config_path): if "module_version" not in variables: variables["module_version"] = date() include_templated_vars(config) - # include_stable_module_dep_versions(config) pre_check(config) prep(config_path) diff --git a/nci_environment/dea/modulefile.template b/nci_environment/dea/modulefile.template index abf6199..e58d87e 100644 --- a/nci_environment/dea/modulefile.template +++ b/nci_environment/dea/modulefile.template @@ -39,6 +39,10 @@ prepend-path PYTHONPATH ${module_path}/share/qgis/python remove-path PATH ${module_path}/bin prepend-path PATH ${module_path}/bin +# Remove duplicate entries for python path and prepend again +remove-path PYTHONPATH ${python_path} +prepend-path PYTHONPATH ${python_path} + # To avoid user packages conflicting with Environment Module packages, point the PYTHONUSERBASE and PATH # variables to point to a directory based on the Environment Module version which is loaded so that extra # packages must be re-installed when a new dea module is released @@ -55,8 +59,8 @@ if {[module-info mode load] && [is-loaded $$name/$$version]} { } # Remove duplicate entries for HOME dir and prepend at the top -remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages -prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.9/site-packages +remove-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages +prepend-path PYTHONPATH ~/.dea-sandbox/${module_name}/${module_version}/local/lib/python3.10/site-packages ############################################################# diff --git a/nci_environment/dea/modulespec.yaml b/nci_environment/dea/modulespec.yaml index 09ac26f..4173d09 100644 --- a/nci_environment/dea/modulespec.yaml +++ b/nci_environment/dea/modulespec.yaml @@ -5,17 +5,19 @@ variables: conda_path: "/g/data/v10/private/mambaforge/bin/mamba" dbhost: dea-db.nci.org.au dbport: 6432 - python_version: 3.9 + python_version: '3.10' templated_variables: module_path: "{modules_dir}/{module_name}/{module_version}" python_path: "{modules_dir}/{module_name}/{module_version}/lib/python{python_version}/site-packages/" dea_module: "{module_name}/{module_version}" + pip_path: "{modules_dir}/{module_name}/{module_version}/bin/pip3" install_conda_packages: ../../docker/env.yaml install_pip_packages: - pip_cmd: "module load python3/3.9.2; pip install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; pip install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" + # need to specify which python/pip to use otherwise it defaults to the incorrect one + pip_cmd: "{pip_path} install --no-warn-script-location --prefix {module_path} --requirement requirements.txt; {pip_path} install --no-warn-script-location --prefix {module_path} --requirement requirements-private.txt" copy_files: - src: ../../docker/env.yaml diff --git a/nci_environment/dea/requirements.txt b/nci_environment/dea/requirements.txt index bad0c98..ff29c81 100644 --- a/nci_environment/dea/requirements.txt +++ b/nci_environment/dea/requirements.txt @@ -1,8 +1,4 @@ datacube -eodatasets3 -odc-algo odc-apps-dc-tools -odc-dscache odc-geom -odc-io odc-ui