Initial commit adding a plugin system and a PySide6 plugin

Signed-off-by: Jean-Christophe Morin <[email protected]>

docs/source/conf.py

@@ -63,11 +63,12 @@
 # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
 
 intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
     "rez": ("https://rez.readthedocs.io/en/stable/", None),
 }
 
 # Force usage of :external:
-intersphinx_disabled_reftypes = ["*"]
+# intersphinx_disabled_reftypes = ["*"]
 
 
 # -- Custom ------------------------------------------------------------------

docs/source/index.rst

@@ -69,5 +69,6 @@ automatically created by the `install.py <https://github.com/AcademySoftwareFoun
    command
    transition
    metadata
+   plugins
    faq
    changelog

docs/source/plugins.rst

@@ -0,0 +1,49 @@
+=======
+Plugins
+=======
+
+
+.. py:decorator:: rez_pip.plugins.hookimpl
+
+   Decorator used to register a plugin hook.
+
+Hooks
+=====
+
+.. py:function:: prePipResolve(packages: list[str], requirements: list[str]) -> None
+
+   The pre-pip resolve hook allows a plugin to run some checks *before* resolving the
+   requested packages using pip. The hook **must** not modify the content of the
+   arguments passed to it.
+
+   Some use cases are allowing or disallowing the installation of some packages.
+
+   :param packages: List of packages requested by the user.
+   :param requirements: List of `requirements files <https://pip.pypa.io/en/stable/reference/requirements-file-format/#requirements-file-format>`_ if any.
+
+.. py:function:: postPipResolve(packages: list[rez_pip.pip.PackageInfo]) -> None
+
+   The post-pip resolve hook allows a plugin to run some checks *after* resolving the
+   requested packages using pip. The hook **must** not modify the content of the
+   arguments passed to it.
+
+   Some use cases are allowing or disallowing the installation of some packages.
+
+   :param packages: List of resolved packages.
+
+.. py:function:: groupPackages(packages: list[rez_pip.pip.PackageInfo]) -> list[rez_pip.pip.PackageGroup]:
+
+   Merge packages into groups of packages. The name and version of the first package
+   in the group will be used as the name and version for the rez package.
+
+   The hook **must** pop grouped packages out of the "packages" variable.
+
+   :param packages: List of resolved packages.
+   :returns: A list of package groups.
+
+.. py:function:: metadata(package: rez.package_maker.PackageMaker) -> None
+
+   Modify/inject metadata in the rez package. The plugin is expected to modify
+   "package" in place.
+
+   :param package: An instanciate PackageMaker.

src/rez_pip/cli.py

@@ -8,6 +8,7 @@
 import textwrap
 import pathlib
 import tempfile
+import itertools
 import subprocess
 
 if sys.version_info >= (3, 10):
@@ -18,13 +19,15 @@
 import rich
 import rich.text
 import rich.panel
+import rich.table
 import rez.version
 import rich.markup
 import rich.logging
 
 import rez_pip.pip
 import rez_pip.rez
 import rez_pip.data
+import rez_pip.plugins
 import rez_pip.install
 import rez_pip.download
 import rez_pip.exceptions
@@ -113,6 +116,10 @@ def _createParser() -> argparse.ArgumentParser:
         help="Print debug information that you can use when reporting an issue on GitHub.",
     )
 
+    debugGroup.add_argument(
+        "--list-plugins", action="store_true", help="List all registered plugins"
+    )
+
     parser.usage = f"""
 
   %(prog)s [options] <package(s)>
@@ -195,37 +202,55 @@ def _run(args: argparse.Namespace, pipArgs: typing.List[str], pipWorkArea: str)
             )
 
         _LOG.info(f"Resolved {len(packages)} dependencies for python {pythonVersion}")
+        packageGroups: typing.List[rez_pip.pip.PackageGroup] = list(
+            itertools.chain(*rez_pip.plugins.getHook().groupPackages(packages=packages))  # type: ignore[arg-type]
+        )
+        packageGroups += [rez_pip.pip.PackageGroup([package]) for package in packages]
 
         # TODO: Should we postpone downloading to the last minute if we can?
         _LOG.info("[bold]Downloading...")
-        wheels = rez_pip.download.downloadPackages(packages, wheelsDir)
-        _LOG.info(f"[bold]Downloaded {len(wheels)} wheels")
 
-        dists: typing.Dict[importlib_metadata.Distribution, bool] = {}
+        wheelsToDownload = []
+        localWheels = []
+        for group in packageGroups:
+            for url in group.downloadUrls:
+                print(url)
+                if url.startswith("file://"):
+                    localWheels.append(url[7:])
+                else:
+                    wheelsToDownload.extend(group.packages)
+
+        downloadedWheels = rez_pip.download.downloadPackages(
+            wheelsToDownload, wheelsDir
+        )
+        _LOG.info(f"[bold]Downloaded {len(downloadedWheels)} wheels")
+
+        localWheels += downloadedWheels
 
+        # Here, we could have a mapping of <merged package>: <dists> and pass that to installWheel
         with rich.get_console().status(
             f"[bold]Installing wheels into {installedWheelsDir!r}"
         ):
-            for package, wheel in zip(packages, wheels):
-                _LOG.info(f"[bold]Installing {package.name}-{package.version} wheel")
-                dist, isPure = rez_pip.install.installWheel(
-                    package, pathlib.Path(wheel), installedWheelsDir
-                )
-
-                dists[dist] = isPure
-
-        distNames = [dist.name for dist in dists.keys()]
+            for group in packageGroups:
+                for package, wheel in zip(group.packages, group.downloadUrls):
+                    _LOG.info(f"[bold]Installing {wheel}")
+                    dist = rez_pip.install.installWheel(
+                        package,
+                        pathlib.Path(
+                            wheel[7:] if wheel.startswith("file://") else wheel
+                        ),
+                        os.path.join(installedWheelsDir, package.name),
+                    )
+                    group.dists.append(dist)
 
         with rich.get_console().status("[bold]Creating rez packages..."):
-            for dist, package in zip(dists, packages):
-                isPure = dists[dist]
+            for group in packageGroups:
+                print(list(package.name for package in group.packages))
                 rez_pip.rez.createPackage(
-                    dist,
-                    isPure,
+                    group.dists,
                     rez.version.Version(pythonVersion),
-                    distNames,
                     installedWheelsDir,
-                    wheelURL=package.download_info.url,
+                    group.downloadUrls,
                     prefix=args.prefix,
                     release=args.release,
                 )
@@ -306,10 +331,24 @@ def _debug(
     )
 
 
+def _printPlugins() -> None:
+    table = rich.table.Table("Name", "Hooks", box=None)
+    for plugin, hooks in rez_pip.plugins._getHookImplementations().items():
+        table.add_row(plugin, ", ".join(hooks))
+    rich.get_console().print(table)
+
+
 def run() -> int:
     pipWorkArea = tempfile.mkdtemp(prefix="rez-pip-target")
     args, pipArgs = _parseArgs(sys.argv[1:])
 
+    # Initialize the plugin system
+    rez_pip.plugins.getManager()
+
+    if args.list_plugins:
+        _printPlugins()
+        return 0
+
     try:
         _validateArgs(args)
 

src/rez_pip/install.py

@@ -37,9 +37,12 @@
     ScriptSection = Literal["console", "gui"]
 
 
-def isWheelPure(source: installer.sources.WheelSource) -> bool:
-    stream = source.read_dist_info("WHEEL")
-    metadata = installer.utils.parse_metadata_file(stream)
+def isWheelPure(dist: importlib_metadata.Distribution) -> bool:
+    path = next(
+        f for f in dist.files if os.fspath(f.locate()).endswith(".dist-info/WHEEL")
+    )
+    with open(path.locate()) as fd:
+        metadata = installer.utils.parse_metadata_file(fd.read())
     return typing.cast(str, metadata["Root-Is-Purelib"]) == "true"
 
 
@@ -70,7 +73,7 @@ def installWheel(
     package: rez_pip.pip.PackageInfo,
     wheelPath: pathlib.Path,
     targetPath: str,
-) -> typing.Tuple[importlib_metadata.Distribution, bool]:
+) -> importlib_metadata.Distribution:
     # TODO: Technically, target should be optional. We will always want to install in "pip install --target"
     #       mode. So right now it's a CLI option for debugging purposes.
 
@@ -81,11 +84,8 @@ def installWheel(
         script_kind=installer.utils.get_launcher_kind(),
     )
 
-    isPure = True
     _LOG.debug(f"Installing {wheelPath} into {targetPath!r}")
     with installer.sources.WheelFile.open(wheelPath) as source:
-        isPure = isWheelPure(source)
-
         installer.install(
             source=source,
             destination=destination,
@@ -118,7 +118,7 @@ def installWheel(
         if not dist.files:
             raise RuntimeError(f"{path!r} does not exist!")
 
-    return dist, isPure
+    return dist
 
 
 # TODO: Document where this code comes from.

src/rez_pip/pip.py

@@ -8,9 +8,15 @@
 import subprocess
 import dataclasses
 
+if sys.version_info >= (3, 10):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
 import dataclasses_json
 
 import rez_pip.data
+import rez_pip.plugins
 import rez_pip.exceptions
 
 _LOG = logging.getLogger(__name__)
@@ -58,6 +64,21 @@ def version(self) -> str:
         return self.metadata.version
 
 
+class PackageGroup:
+    """A group of package"""
+
+    packages: typing.List[PackageInfo]
+    dists: typing.List[importlib_metadata.Distribution]
+
+    def __init__(self, packages: typing.List[PackageInfo]) -> None:
+        self.packages = packages
+        self.dists = []
+
+    @property
+    def downloadUrls(self) -> typing.List[str]:
+        return [p.download_info.url for p in self.packages]
+
+
 def getBundledPip() -> str:
     return os.path.join(os.path.dirname(rez_pip.data.__file__), "pip.pyz")
 
@@ -71,7 +92,9 @@ def getPackages(
     constraints: typing.List[str],
     extraArgs: typing.List[str],
 ) -> typing.List[PackageInfo]:
-    # python pip.pyz install -q requests --dry-run --ignore-installed --python-version 2.7 --only-binary=:all: --target /tmp/asd --report -
+    rez_pip.plugins.getHook().prePipResolve(
+        packages=packageNames, requirements=requirements
+    )
 
     _fd, tmpFile = tempfile.mkstemp(prefix="pip-install-output", text=True)
     os.close(_fd)
@@ -138,6 +161,8 @@ def getPackages(
         packageInfo = PackageInfo.from_dict(rawPackage)
         packages.append(packageInfo)
 
+    rez_pip.plugins.getHook().postPipResolve(packages=packages)
+
     return packages