Plugin system

docs/source/conf.py

@@ -63,11 +63,12 @@
 # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
 
 intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
     "rez": ("https://rez.readthedocs.io/en/stable/", None),
 }
 
 # Force usage of :external:
-intersphinx_disabled_reftypes = ["*"]
+# intersphinx_disabled_reftypes = ["*"]
 
 
 # -- Custom ------------------------------------------------------------------

docs/source/index.rst

@@ -69,5 +69,6 @@ automatically created by the `install.py <https://github.com/AcademySoftwareFoun
    command
    transition
    metadata
+   plugins
    faq
    changelog

docs/source/plugins.rst

@@ -0,0 +1,56 @@
+=======
+Plugins
+=======
+
+
+.. py:decorator:: rez_pip.plugins.hookimpl
+
+   Decorator used to register a plugin hook.
+
+Hooks
+=====
+
+.. py:function:: prePipResolve(packages: list[str], requirements: list[str]) -> None
+
+   The pre-pip resolve hook allows a plugin to run some checks *before* resolving the
+   requested packages using pip. The hook **must** not modify the content of the
+   arguments passed to it.
+
+   Some use cases are allowing or disallowing the installation of some packages.
+
+   :param packages: List of packages requested by the user.
+   :param requirements: List of `requirements files <https://pip.pypa.io/en/stable/reference/requirements-file-format/#requirements-file-format>`_ if any.
+
+.. py:function:: postPipResolve(packages: list[rez_pip.pip.PackageInfo]) -> None
+
+   The post-pip resolve hook allows a plugin to run some checks *after* resolving the
+   requested packages using pip. The hook **must** not modify the content of the
+   arguments passed to it.
+
+   Some use cases are allowing or disallowing the installation of some packages.
+
+   :param packages: List of resolved packages.
+
+.. py:function:: groupPackages(packages: list[rez_pip.pip.PackageInfo]) -> list[rez_pip.pip.PackageGroup]:
+
+   Merge packages into groups of packages. The name and version of the first package
+   in the group will be used as the name and version for the rez package.
+
+   The hook **must** pop grouped packages out of the "packages" variable.
+
+   :param packages: List of resolved packages.
+   :returns: A list of package groups.
+
+.. py:function:: metadata(package: rez.package_maker.PackageMaker) -> None
+
+   Modify/inject metadata in the rez package. The plugin is expected to modify
+   "package" in place.
+
+   :param package: An instanciate PackageMaker.
+
+.. py:function:: cleanup(dist: importlib.metadata.Distribution, path: str) -> None
+
+   Cleanup a package post-installation.
+
+   :param dist: Python distribution.
+   :param path: Root path of the rez variant.

pyproject.toml

@@ -21,6 +21,9 @@ dependencies = [
     "dataclasses-json",
     "rich",
     "importlib_metadata>=4.6 ; python_version < '3.10'",
+    # 1.3 introduces type hints.
+    "pluggy>=1.2",
+    "typing-extensions; python_version < '3.8'"
 ]
 
 classifiers = [

pytest.ini

@@ -7,12 +7,12 @@ addopts =
     --cov-report=term-missing
     --cov-report=xml
     --cov-report=html
-    --durations=0
+    #--durations=0
 
 norecursedirs = rez_repo
 
 markers =
     integration: mark the tests as integration tests
     py37: mark the tests has using a Python 3.7 rez package
-    py39: mark the tests has using a Python 3.7 rez package
+    py39: mark the tests has using a Python 3.9 rez package
     py311: mark the tests has using a Python 3.11 rez package

src/rez_pip/cli.py

@@ -6,28 +6,26 @@
 import logging
 import argparse
 import textwrap
-import pathlib
 import tempfile
+import itertools
 import subprocess
 
-if sys.version_info >= (3, 10):
-    import importlib.metadata as importlib_metadata
-else:
-    import importlib_metadata
-
 import rich
 import rich.text
 import rich.panel
+import rich.table
 import rez.version
 import rich.markup
 import rich.logging
 
 import rez_pip.pip
 import rez_pip.rez
 import rez_pip.data
+import rez_pip.plugins
 import rez_pip.install
 import rez_pip.download
 import rez_pip.exceptions
+from rez_pip.compat import importlib_metadata
 
 _LOG = logging.getLogger("rez_pip.cli")
 
@@ -120,6 +118,10 @@ def _createParser() -> argparse.ArgumentParser:
         help="Print debug information that you can use when reporting an issue on GitHub.",
     )
 
+    debugGroup.add_argument(
+        "--list-plugins", action="store_true", help="List all registered plugins"
+    )
+
     parser.usage = f"""
 
   %(prog)s [options] <package(s)>
@@ -202,37 +204,50 @@ def _run(args: argparse.Namespace, pipArgs: typing.List[str], pipWorkArea: str)
             )
 
         _LOG.info(f"Resolved {len(packages)} dependencies for python {pythonVersion}")
+        packageGroups: typing.List[rez_pip.pip.PackageGroup] = list(
+            itertools.chain(*rez_pip.plugins.getHook().groupPackages(packages=packages))  # type: ignore[arg-type]
+        )
+
+        # Remove empty groups
+        packageGroups = [group for group in packageGroups if group]
+
+        # Add packages that were not grouped.
+        packageGroups += [rez_pip.pip.PackageGroup([package]) for package in packages]
 
         # TODO: Should we postpone downloading to the last minute if we can?
         _LOG.info("[bold]Downloading...")
-        wheels = rez_pip.download.downloadPackages(packages, wheelsDir)
-        _LOG.info(f"[bold]Downloaded {len(wheels)} wheels")
 
-        dists: typing.Dict[importlib_metadata.Distribution, bool] = {}
+        downloadedWheels = rez_pip.download.downloadPackages(packageGroups, wheelsDir)
+        foundLocally = [
+            p
+            for group in packageGroups
+            for p in group.packages
+            if not p.isDownloadRequired()
+        ]
+
+        _LOG.info(
+            f"[bold]Downloaded {len(downloadedWheels)} wheels, skipped {len(foundLocally)} because they resolved to local files"
+        )
 
         with rich.get_console().status(
             f"[bold]Installing wheels into {installedWheelsDir!r}"
         ):
-            for package, wheel in zip(packages, wheels):
-                _LOG.info(f"[bold]Installing {package.name}-{package.version} wheel")
-                dist, isPure = rez_pip.install.installWheel(
-                    package, pathlib.Path(wheel), installedWheelsDir
-                )
-
-                dists[dist] = isPure
-
-        distNames = [dist.name for dist in dists.keys()]
+            for group in packageGroups:
+                for package in group.packages:
+                    _LOG.info(f"[bold]Installing {package.name} {package.localPath}")
+                    dist = rez_pip.install.installWheel(
+                        package,
+                        package.localPath,
+                        os.path.join(installedWheelsDir, package.name),
+                    )
+                    group.dists.append(dist)
 
         with rich.get_console().status("[bold]Creating rez packages..."):
-            for dist, package in zip(dists, packages):
-                isPure = dists[dist]
+            for group in packageGroups:
                 rez_pip.rez.createPackage(
-                    dist,
-                    isPure,
+                    group,
                     rez.version.Version(pythonVersion),
-                    distNames,
                     installedWheelsDir,
-                    wheelURL=package.download_info.url,
                     prefix=args.prefix,
                     release=args.release,
                 )
@@ -313,10 +328,24 @@ def _debug(
     )
 
 
+def _printPlugins() -> None:
+    table = rich.table.Table("Name", "Hooks", box=None)
+    for plugin, hooks in rez_pip.plugins._getHookImplementations().items():
+        table.add_row(plugin, ", ".join(hooks))
+    rich.get_console().print(table)
+
+
 def run() -> int:
     pipWorkArea = tempfile.mkdtemp(prefix="rez-pip-target")
     args, pipArgs = _parseArgs(sys.argv[1:])
 
+    # Initialize the plugin system
+    rez_pip.plugins.getManager()
+
+    if args.list_plugins:
+        _printPlugins()
+        return 0
+
     try:
         _validateArgs(args)
 

src/rez_pip/compat.py

@@ -0,0 +1,13 @@
+import sys
+
+if sys.version_info <= (3, 8):
+    from typing import Sequence, MutableSequence, Mapping
+else:
+    from collections.abc import Sequence, MutableSequence, Mapping
+
+if sys.version_info >= (3, 10):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+__all__ = ["Sequence", "MutableSequence", "Mapping", "importlib_metadata"]

src/rez_pip/download.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import typing
 import asyncio
 import hashlib
@@ -9,25 +8,21 @@
 import aiohttp
 import rich.progress
 
-if sys.version_info >= (3, 10):
-    import importlib.metadata as importlib_metadata
-else:
-    import importlib_metadata
-
 import rez_pip.pip
+from rez_pip.compat import importlib_metadata
 
 _LOG = logging.getLogger(__name__)
 _lock = asyncio.Lock()
 
 
 def downloadPackages(
-    packages: typing.List[rez_pip.pip.PackageInfo], dest: str
+    packageGroups: typing.List[rez_pip.pip.PackageGroup], dest: str
 ) -> typing.List[str]:
-    return asyncio.run(_downloadPackages(packages, dest))
+    return asyncio.run(_downloadPackages(packageGroups, dest))
 
 
 async def _downloadPackages(
-    packages: typing.List[rez_pip.pip.PackageInfo], dest: str
+    packageGroups: typing.List[rez_pip.pip.PackageGroup], dest: str
 ) -> typing.List[str]:
     items: typing.List[
         typing.Coroutine[typing.Any, typing.Any, typing.Optional[str]]
@@ -47,18 +42,33 @@ async def _downloadPackages(
             tasks: typing.Dict[str, rich.progress.TaskID] = {}
 
             # Create all the downlod tasks first
-            for package in packages:
-                tasks[package.name] = progress.add_task(package.name)
+            numPackages = 0
+            for group in packageGroups:
+                for package in group.packages:
+                    if not package.isDownloadRequired():
+                        continue
 
-            # Then create the "total" progress bar. This ensures that total is at the bottom.
-            mainTask = progress.add_task(f"[bold]Total (0/{len(packages)})", total=0)
+                    numPackages += 1
+                    tasks[package.name] = progress.add_task(package.name)
 
-            for package in packages:
-                items.append(
-                    _download(
-                        package, dest, session, progress, tasks[package.name], mainTask
+            # Then create the "total" progress bar. This ensures that total is at the bottom.
+            mainTask = progress.add_task(f"[bold]Total (0/{numPackages})", total=0)
+
+            for group in packageGroups:
+                for package in group.packages:
+                    if not package.isDownloadRequired():
+                        continue
+
+                    items.append(
+                        _download(
+                            package,
+                            dest,
+                            session,
+                            progress,
+                            tasks[package.name],
+                            mainTask,
+                        )
                     )
-                )
 
             wheels = await asyncio.gather(*items)
 
@@ -152,4 +162,5 @@ async def _download(
             mainTaskID, description=f"[bold]Total ({len(completedItems)}/{total})"
         )
 
+    package.localPath = wheelPath
     return wheelPath