cate 4.0 - JupyterLab Integration

.github/workflows/cate-workflow.yml

@@ -8,8 +8,8 @@ on:
 jobs:
   unittest:
     env:
-      XCUBE_VERSION: 0.13.0.dev3
-      XCUBE_CCI_VERSION: 0.9.7
+      XCUBE_VERSION: 1.0.4
+      XCUBE_CCI_VERSION: 0.9.8
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -40,7 +40,7 @@ jobs:
           tar xvzf v"${XCUBE_CCI_VERSION}".tar.gz
           cd xcube-cci-"${XCUBE_CCI_VERSION}"
           python setup.py install
-          mamba install -y -c conda-forge aiohttp nest-asyncio lxml pydap cartopy jsonschema
+          mamba install -y -c conda-forge aiohttp nest-asyncio lxml pydap==3.3 cartopy jsonschema
       - name: setup-cate
         shell: bash -l {0}
         run: |

CHANGES.md

@@ -1,4 +1,26 @@
-## Version 3.1.7 (in development)
+## Version 4.0.0 (in development)
+
+* Cate is now designed to work inside of Jupyter Lab and standalone.
+  For this reason, the Cate App UI is now bundled with the Cate server
+  and served via the endpoint "/app". 
+
+  Current Cate App version is 4.0.0-dev.6. 
+
+* Now using workspace identifiers instead of base directories in resource 
+  URLs of the WebAPI. This way we no longer need to URL-encode workspace 
+  directories in WebAPI URLs, which did not work with 
+  [jupyter-server-proxy](https://jupyter-server-proxy.readthedocs.io/).
+
+* The "local" data store is now configured to use a given 
+  server root path.
+
+* Cached data is now written into a sub-folder `cate-local/{data_store_id}` 
+  within Cate server's root path unless another folder is specified. 
+
+* Removed the `cate upd` CLI command.
+
+* Added support of environment variable `CATE_DATA_PATH` to set path where to
+  find basic cate configuration files.
 
 ## Version 3.1.6
 

appveyor.yml

@@ -56,7 +56,7 @@ build_script:
   # Install xcube-cci
   - cd ..
   - cd xcube-cci
-  - mamba install -c conda-forge lxml pydap aiohttp nest-asyncio
+  - mamba install -c conda-forge lxml pydap==3.3 aiohttp nest-asyncio
   - pip install -e .
   # Install cate-specific packages
   - cd ..

cate/cli/main.py

@@ -227,8 +227,11 @@ def _parse_write_arg(write_arg) -> Tuple[NullableStr, NullableStr, NullableStr]:
     name, path = name_and_path if len(name_and_path) == 2 else (None, write_arg)
     path_and_format = path.rsplit(',', maxsplit=1)
     path, format_name = path_and_format if len(path_and_format) == 2 else (path, None)
-    return name if name else None, path if path else None, format_name.upper() \
-        if format_name else None
+    return (
+        name if name else None,
+        path if path else None,
+        format_name.upper() if format_name else None
+    )
 
 
 def _parse_op_args(raw_args: List[str],
@@ -1374,129 +1377,6 @@ def _execute_copy(cls, command_args):
                   "Please check constraint.")
 
 
-class UpdateCommand(Command):
-    """
-    The ``update`` command is used to update an existing cate environment to a specific or the
-    latest cate version.
-    """
-
-    @classmethod
-    def name(cls):
-        return 'upd'
-
-    @classmethod
-    def parser_kwargs(cls):
-        return dict(help='Update an existing cate environment to a specific or to the latest cate '
-                         'version',
-                    description='Update an existing cate environment to a specific or to the '
-                                'latest cate version.')
-
-    @classmethod
-    def configure_parser(cls, parser):
-        parser.add_argument('-y', '--yes', dest='yes', action='store_true', default=False,
-                            help='Do not ask for confirmation.')
-        parser.add_argument('-i', '--info', dest='show_info', action='store_true', default=False,
-                            help='Show version information only; do not update yet.')
-        parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False,
-                            help='Only display what would have been done.')
-        parser.add_argument('version', metavar='VERSION', nargs='?', default=None,
-                            help='A cate version identifier, e.g. "1.0.3"; '
-                                 'the version identifier must have the form "major.minor.micro" '
-                                 'and may comprise a development release suffix, '
-                                 'e.g. "1.2.0.dev4"')
-
-    def execute(self, command_args):
-        current_version = __version__
-        desired_version = command_args.version
-        show_info = command_args.show_info
-        dry_run = command_args.dry_run
-
-        from cate.util.process import run_subprocess
-        if sys.platform == 'win32':
-            conda_path = os.path.join(sys.prefix, 'Scripts', 'conda.exe')
-        else:
-            conda_path = os.path.join(sys.prefix, 'bin', 'conda')
-
-        import subprocess
-
-        package = 'cate-cli'
-        channel = 'ccitools'
-        command = [conda_path, 'search', '--channel', channel, package]
-        completed_process = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout = completed_process.stdout.decode("utf-8") if completed_process.stdout else None
-        stderr = completed_process.stderr.decode("utf-8") if completed_process.stderr else None
-        if stderr:
-            raise CommandError(stderr)
-
-        available_versions = []
-        latest_version = None
-        if stdout:
-            package_info = [row.split() for row in stdout.split('\n')]
-            package_info.reverse()
-            for entry in package_info:
-                available_version = None
-                if len(entry) == 4 and entry[0] == package and entry[-1] == channel:
-                    available_version = entry[1]
-                elif len(entry) == 3 and entry[-1] == channel:
-                    available_version = entry[0]
-                if available_version:
-                    available_versions.append(available_version)
-                    if not latest_version:
-                        latest_version = available_version
-
-        if not latest_version:
-            raise CommandError('failed to retrieve latest cate version')
-
-        if show_info:
-            print('Latest version is %s' % latest_version)
-            print('Current version is %s' % current_version)
-            if desired_version:
-                available = desired_version in available_versions
-                print(f'Desired version is %s (%s)'
-                      % (desired_version, 'available' if available else 'not available'))
-            print('Available versions:')
-            for available_version in available_versions:
-                print(' ', available_version)
-            return
-
-        if not desired_version:
-            desired_version = latest_version
-
-        if desired_version == current_version:
-            if latest_version == current_version:
-                print('Current cate version is %s and up-to-date' % current_version)
-            else:
-                print('Current cate version is already %s' % current_version)
-            return
-
-        if desired_version not in available_versions:
-            raise CommandError(f'desired cate version {desired_version} is not available; '
-                               'type "cate upd --info" to show available versions')
-
-        if command_args.yes or dry_run:
-            answer = 'y'
-        else:
-            prompt = f'Do you really want to change from {current_version} to {desired_version} ' \
-                     f'(y/[n])? '
-            answer = input(prompt)
-        if not answer or answer.lower() != 'y':
-            return
-
-        command = [conda_path, 'install', '--yes', '--channel',
-                   channel, '--channel', 'conda-forge']
-        if dry_run:
-            command.append('--dry-run')
-        command.append('%s=%s' % (package, desired_version))
-
-        def stdout_handler(text):
-            sys.stdout.write(text)
-
-        def stderr_handler(text):
-            sys.stdout.write(text)
-
-        run_subprocess(command, stdout_handler=stdout_handler, stderr_handler=stderr_handler)
-
-
 class IOCommand(SubCommandCommand):
     """
     The ``io`` command implements various operations w.r.t. supported data and file formats.
@@ -1588,7 +1468,6 @@ def _execute_list(cls, command_args):
     ResourceCommand,
     RunCommand,
     IOCommand,
-    UpdateCommand,
     # PluginCommand,
 ]
 

cate/conf/conf.py

@@ -24,7 +24,7 @@
 import logging
 import os.path
 import sys
-from typing import Any, Dict, Optional, Sequence, Union
+from typing import Any, Dict, Optional, Sequence, Union, Hashable
 
 from .defaults import GLOBAL_CONF_FILE, LOCAL_CONF_FILE, LOCATION_FILE, VERSION_CONF_FILE, \
     VARIABLE_DISPLAY_SETTINGS, DEFAULT_DATA_PATH, DEFAULT_VERSION_DATA_PATH, DEFAULT_COLOR_MAP, DEFAULT_RES_PATTERN, \
@@ -116,12 +116,13 @@ def get_http_proxy() -> Optional[str]:
     return http_proxy
 
 
-def is_default_variable(var_name: str) -> bool:
+def is_default_variable(var_name: Hashable) -> bool:
     default_variables = get_config().get('default_variables', DEFAULT_VARIABLES)
     return var_name in default_variables
 
 
-def get_variable_display_settings(var_name: str) -> Optional[Dict[str, Any]]:
+def get_variable_display_settings(var_name: Hashable) \
+        -> Optional[Dict[str, Any]]:
     """
     Get the global variable display settings which is a combination of defaults.
     :return:

cate/conf/defaults.py

@@ -25,8 +25,11 @@
 
 from ..version import __version__
 
+CATE_LOCAL_DIR_NAME = 'cate-local'
+
 DEFAULT_DATA_DIR_NAME = '.cate'
-DEFAULT_DATA_PATH = os.path.join(os.path.expanduser('~'), DEFAULT_DATA_DIR_NAME)
+DEFAULT_DATA_PATH = os.environ.get('CATE_DATA_PATH',
+    os.path.join(os.path.expanduser('~'), DEFAULT_DATA_DIR_NAME))
 DEFAULT_VERSION_DATA_PATH = os.path.join(DEFAULT_DATA_PATH, __version__)
 
 GLOBAL_CONF_FILE = os.path.join(DEFAULT_DATA_PATH, 'conf.py')

cate/core/ds.py

@@ -1,15 +1,15 @@
 # The MIT License (MIT)
-# Copyright (c) 2016, 2017 by the ESA CCI Toolbox development team and contributors
+# Copyright (c) 2016-2023 by the ESA CCI Toolbox team and contributors
 #
-# Permission is hereby granted, free of charge, to any person obtaining a copy of
-# this software and associated documentation files (the "Software"), to deal in
-# the Software without restriction, including without limitation the rights to
-# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-# of the Software, and to permit persons to whom the Software is furnished to do
-# so, subject to the following conditions:
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
 #
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
@@ -85,7 +85,6 @@
 import datetime
 import glob
 import logging
-import re
 from typing import Sequence, Optional, Union, Any, Dict, Set, Tuple
 
 import geopandas as gpd
@@ -100,6 +99,7 @@
 from xcube.util.progress import add_progress_observers
 from .cdm import get_lon_dim_name, get_lat_dim_name
 from .types import PolygonLike, TimeRangeLike, VarNamesLike, ValidationError
+from ..conf.defaults import CATE_LOCAL_DIR_NAME
 from ..util.monitor import ChildMonitor
 from ..util.monitor import Monitor
 
@@ -110,14 +110,6 @@
              "Norman Fomferra (Brockmann Consult GmbH), " \
              "Marco Zühlke (Brockmann Consult GmbH)"
 
-URL_REGEX = re.compile(
-    r'^(?:http|ftp)s?://'  # http:// or https://
-    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain
-    r'localhost|'  # localhost...
-    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
-    r'(?::\d+)?'  # optional port
-    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
-
 _LOG = logging.getLogger('cate')
 
 DATA_STORE_POOL = xcube_store.DataStorePool()
@@ -456,15 +448,17 @@ def open_dataset(dataset_id: str,
             with add_progress_observers(XcubeProgressObserver(ChildMonitor(monitor, cache_work))):
                 dataset, dataset_id = make_local(data=dataset,
                                                  local_name=local_ds_id,
-                                                 orig_dataset_name=dataset_id)
+                                                 orig_dataset_name=dataset_id,
+                                                 data_store_id=data_store_id)
 
     return dataset, dataset_id
 
 
 def make_local(data: Any,
                *,
                local_name: Optional[str] = None,
-               orig_dataset_name: Optional[str] = None) -> Tuple[Any, str]:
+               orig_dataset_name: Optional[str] = None,
+               data_store_id: Optional[str] = None) -> Tuple[Any, str]:
     local_data_store_id = 'local'
     local_store = DATA_STORE_POOL.get_store(local_data_store_id)
     if local_store is None:
@@ -485,6 +479,10 @@ def make_local(data: Any,
         while local_store.has_data(local_name):
             i += 1
             local_name = f'local.{orig_dataset_name}.{i}{extension}'
+    if '/' not in local_name:
+        local_name_prefix = f'{CATE_LOCAL_DIR_NAME}/{data_store_id}/' \
+            if data_store_id is not None else f'{CATE_LOCAL_DIR_NAME}/'
+        local_name = local_name_prefix + local_name
     local_data_id = local_store.write_data(data=data,
                                            data_id=local_name,
                                            replace=True)