diff --git a/docs/source/conf.py b/docs/source/conf.py index 5d33d50..4d763ad 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -63,11 +63,12 @@ # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), "rez": ("https://rez.readthedocs.io/en/stable/", None), } # Force usage of :external: -intersphinx_disabled_reftypes = ["*"] +# intersphinx_disabled_reftypes = ["*"] # -- Custom ------------------------------------------------------------------ diff --git a/docs/source/index.rst b/docs/source/index.rst index 3cd4830..3aff58c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -69,5 +69,6 @@ automatically created by the `install.py None + + The pre-pip resolve hook allows a plugin to run some checks *before* resolving the + requested packages using pip. The hook **must** not modify the content of the + arguments passed to it. + + Some use cases are allowing or disallowing the installation of some packages. + + :param packages: List of packages requested by the user. + :param requirements: List of `requirements files `_ if any. + +.. py:function:: postPipResolve(packages: list[rez_pip.pip.PackageInfo]) -> None + + The post-pip resolve hook allows a plugin to run some checks *after* resolving the + requested packages using pip. The hook **must** not modify the content of the + arguments passed to it. + + Some use cases are allowing or disallowing the installation of some packages. + + :param packages: List of resolved packages. + +.. py:function:: groupPackages(packages: list[rez_pip.pip.PackageInfo]) -> list[rez_pip.pip.PackageGroup]: + + Merge packages into groups of packages. The name and version of the first package + in the group will be used as the name and version for the rez package. + + The hook **must** pop grouped packages out of the "packages" variable. + + :param packages: List of resolved packages. + :returns: A list of package groups. + +.. py:function:: metadata(package: rez.package_maker.PackageMaker) -> None + + Modify/inject metadata in the rez package. The plugin is expected to modify + "package" in place. + + :param package: An instanciate PackageMaker. diff --git a/src/rez_pip/cli.py b/src/rez_pip/cli.py index b9442f4..db71d5c 100644 --- a/src/rez_pip/cli.py +++ b/src/rez_pip/cli.py @@ -8,6 +8,7 @@ import textwrap import pathlib import tempfile +import itertools import subprocess if sys.version_info >= (3, 10): @@ -18,6 +19,7 @@ import rich import rich.text import rich.panel +import rich.table import rez.version import rich.markup import rich.logging @@ -25,6 +27,7 @@ import rez_pip.pip import rez_pip.rez import rez_pip.data +import rez_pip.plugins import rez_pip.install import rez_pip.download import rez_pip.exceptions @@ -113,6 +116,10 @@ def _createParser() -> argparse.ArgumentParser: help="Print debug information that you can use when reporting an issue on GitHub.", ) + debugGroup.add_argument( + "--list-plugins", action="store_true", help="List all registered plugins" + ) + parser.usage = f""" %(prog)s [options] @@ -195,37 +202,55 @@ def _run(args: argparse.Namespace, pipArgs: typing.List[str], pipWorkArea: str) ) _LOG.info(f"Resolved {len(packages)} dependencies for python {pythonVersion}") + packageGroups: typing.List[rez_pip.pip.PackageGroup] = list( + itertools.chain(*rez_pip.plugins.getHook().groupPackages(packages=packages)) # type: ignore[arg-type] + ) + packageGroups += [rez_pip.pip.PackageGroup([package]) for package in packages] # TODO: Should we postpone downloading to the last minute if we can? _LOG.info("[bold]Downloading...") - wheels = rez_pip.download.downloadPackages(packages, wheelsDir) - _LOG.info(f"[bold]Downloaded {len(wheels)} wheels") - dists: typing.Dict[importlib_metadata.Distribution, bool] = {} + wheelsToDownload = [] + localWheels = [] + for group in packageGroups: + for url in group.downloadUrls: + print(url) + if url.startswith("file://"): + localWheels.append(url[7:]) + else: + wheelsToDownload.extend(group.packages) + + downloadedWheels = rez_pip.download.downloadPackages( + wheelsToDownload, wheelsDir + ) + _LOG.info(f"[bold]Downloaded {len(downloadedWheels)} wheels") + + localWheels += downloadedWheels + # Here, we could have a mapping of : and pass that to installWheel with rich.get_console().status( f"[bold]Installing wheels into {installedWheelsDir!r}" ): - for package, wheel in zip(packages, wheels): - _LOG.info(f"[bold]Installing {package.name}-{package.version} wheel") - dist, isPure = rez_pip.install.installWheel( - package, pathlib.Path(wheel), installedWheelsDir - ) - - dists[dist] = isPure - - distNames = [dist.name for dist in dists.keys()] + for group in packageGroups: + for package, wheel in zip(group.packages, group.downloadUrls): + _LOG.info(f"[bold]Installing {wheel}") + dist = rez_pip.install.installWheel( + package, + pathlib.Path( + wheel[7:] if wheel.startswith("file://") else wheel + ), + os.path.join(installedWheelsDir, package.name), + ) + group.dists.append(dist) with rich.get_console().status("[bold]Creating rez packages..."): - for dist, package in zip(dists, packages): - isPure = dists[dist] + for group in packageGroups: + print(list(package.name for package in group.packages)) rez_pip.rez.createPackage( - dist, - isPure, + group.dists, rez.version.Version(pythonVersion), - distNames, installedWheelsDir, - wheelURL=package.download_info.url, + group.downloadUrls, prefix=args.prefix, release=args.release, ) @@ -306,10 +331,24 @@ def _debug( ) +def _printPlugins() -> None: + table = rich.table.Table("Name", "Hooks", box=None) + for plugin, hooks in rez_pip.plugins._getHookImplementations().items(): + table.add_row(plugin, ", ".join(hooks)) + rich.get_console().print(table) + + def run() -> int: pipWorkArea = tempfile.mkdtemp(prefix="rez-pip-target") args, pipArgs = _parseArgs(sys.argv[1:]) + # Initialize the plugin system + rez_pip.plugins.getManager() + + if args.list_plugins: + _printPlugins() + return 0 + try: _validateArgs(args) diff --git a/src/rez_pip/install.py b/src/rez_pip/install.py index 3d180b5..5d602a6 100644 --- a/src/rez_pip/install.py +++ b/src/rez_pip/install.py @@ -37,9 +37,12 @@ ScriptSection = Literal["console", "gui"] -def isWheelPure(source: installer.sources.WheelSource) -> bool: - stream = source.read_dist_info("WHEEL") - metadata = installer.utils.parse_metadata_file(stream) +def isWheelPure(dist: importlib_metadata.Distribution) -> bool: + path = next( + f for f in dist.files if os.fspath(f.locate()).endswith(".dist-info/WHEEL") + ) + with open(path.locate()) as fd: + metadata = installer.utils.parse_metadata_file(fd.read()) return typing.cast(str, metadata["Root-Is-Purelib"]) == "true" @@ -70,7 +73,7 @@ def installWheel( package: rez_pip.pip.PackageInfo, wheelPath: pathlib.Path, targetPath: str, -) -> typing.Tuple[importlib_metadata.Distribution, bool]: +) -> importlib_metadata.Distribution: # TODO: Technically, target should be optional. We will always want to install in "pip install --target" # mode. So right now it's a CLI option for debugging purposes. @@ -81,11 +84,8 @@ def installWheel( script_kind=installer.utils.get_launcher_kind(), ) - isPure = True _LOG.debug(f"Installing {wheelPath} into {targetPath!r}") with installer.sources.WheelFile.open(wheelPath) as source: - isPure = isWheelPure(source) - installer.install( source=source, destination=destination, @@ -118,7 +118,7 @@ def installWheel( if not dist.files: raise RuntimeError(f"{path!r} does not exist!") - return dist, isPure + return dist # TODO: Document where this code comes from. diff --git a/src/rez_pip/pip.py b/src/rez_pip/pip.py index b02cfaf..adcb6ac 100644 --- a/src/rez_pip/pip.py +++ b/src/rez_pip/pip.py @@ -8,9 +8,15 @@ import subprocess import dataclasses +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + import dataclasses_json import rez_pip.data +import rez_pip.plugins import rez_pip.exceptions _LOG = logging.getLogger(__name__) @@ -58,6 +64,21 @@ def version(self) -> str: return self.metadata.version +class PackageGroup: + """A group of package""" + + packages: typing.List[PackageInfo] + dists: typing.List[importlib_metadata.Distribution] + + def __init__(self, packages: typing.List[PackageInfo]) -> None: + self.packages = packages + self.dists = [] + + @property + def downloadUrls(self) -> typing.List[str]: + return [p.download_info.url for p in self.packages] + + def getBundledPip() -> str: return os.path.join(os.path.dirname(rez_pip.data.__file__), "pip.pyz") @@ -71,7 +92,9 @@ def getPackages( constraints: typing.List[str], extraArgs: typing.List[str], ) -> typing.List[PackageInfo]: - # python pip.pyz install -q requests --dry-run --ignore-installed --python-version 2.7 --only-binary=:all: --target /tmp/asd --report - + rez_pip.plugins.getHook().prePipResolve( + packages=packageNames, requirements=requirements + ) _fd, tmpFile = tempfile.mkstemp(prefix="pip-install-output", text=True) os.close(_fd) @@ -138,6 +161,8 @@ def getPackages( packageInfo = PackageInfo.from_dict(rawPackage) packages.append(packageInfo) + rez_pip.plugins.getHook().postPipResolve(packages=packages) + return packages diff --git a/src/rez_pip/plugins/PySide6.py b/src/rez_pip/plugins/PySide6.py new file mode 100644 index 0000000..cc02953 --- /dev/null +++ b/src/rez_pip/plugins/PySide6.py @@ -0,0 +1,120 @@ +"""PySide6 plugin. + +For PySide6, we need a merge hook. If User says "install PySide6", we need to install PySide6, PySide6-Addons and PySide6-Essentials and shiboken6. + +But PySide6, PySide6-Addons and PySide6-Essentials have to be merged. Additionally, shiboken6 needs to be broken down to remove PySide6 (core). +Because shiboken6 vendors PySide6-core... See https://inspector.pypi.io/project/shiboken6/6.6.1/packages/bb/72/e54f758e49e8da0dcd9490d006c41a814b0e56898ce4ca054d60cdba97bd/shiboken6-6.6.1-cp38-abi3-manylinux_2_28_x86_64.whl/. + +On Windows, the PySide6/openssl folder has to be added to PATH, see https://inspector.pypi.io/project/pyside6/6.6.1/packages/ec/3d/1da1b88d74cb5318466156bac91f17ad4272c6c83a973e107ad9a9085009/PySide6-6.6.1-cp38-abi3-win_amd64.whl/PySide6/__init__.py#line.81. + +So it's at least a 3 steps process: +1. Merge PySide6, PySide6-Essentials and PySide6-Addons into the same install. Unvendor shiboken. +2. Install shiboken + cleanup. The Cleanup could be its own hook here specific to shiboken. +""" +import os +import sys +import shutil +import typing +import logging + +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + +import packaging.utils +import packaging.version +import packaging.specifiers +import packaging.requirements + +import rez_pip.pip +import rez_pip.plugins +import rez_pip.exceptions + +# PySide6 was initiall a single package that had shiboken as a dependency. +# Starting from 6.3.0, the package was spit in 3, PySide6, PySide6-Essentials and +# PySide6-Addons. + + +_LOG = logging.getLogger(__name__) + + +@rez_pip.plugins.hookimpl +def prePipResolve( + packages: typing.List[str], +) -> None: + _LOG.debug(f"prePipResolve start") + pyside6Seen = False + variantsSeens = [] + + for package in packages: + req = packaging.requirements.Requirement(package) + name = packaging.utils.canonicalize_name(req.name) + + if name == "pyside6": + pyside6Seen = True + elif name in ["pyside6-essentials", "pyside6-addons"]: + variantsSeens.append(req.name) + + if variantsSeens and not pyside6Seen: + variants = " and ".join(variantsSeens) + verb = "was" if len(variantsSeens) == 1 else "were" + raise rez_pip.exceptions.RezPipError( + f"{variants} {verb} requested but PySide6 was not. You must explicitly request PySide6 in addition to {variants}." + ) + + +@rez_pip.plugins.hookimpl +def postPipResolve(packages: typing.List[rez_pip.pip.PackageInfo]) -> None: + """ + This hook is implemented out of extra caution. We really don't want PySide6-Addons + or PySide6-Essentials to be installed without PySide6. + + In this case, we cover cases where a user requests a package X and that package + depends on PySide6-Addons or PySide6-Essentials. + """ + pyside6Seen = False + variantsSeens = [] + + for package in packages: + name = packaging.utils.canonicalize_name(package.name) + if name == "pyside6": + pyside6Seen = True + elif name in ["pyside6-essentials", "pyside6-addons"]: + variantsSeens.append(package.name) + + if variantsSeens and not pyside6Seen: + variants = " and ".join(variantsSeens) + verb = "is" if len(variantsSeens) == 1 else "are" + raise rez_pip.exceptions.RezPipError( + f"{variants} {verb} part of the resolved packages but PySide6 was not. Dependencies and or you must explicitly request PySide6 in addition to {variants}." + ) + + +@rez_pip.plugins.hookimpl +def groupPackages( + packages: typing.List[rez_pip.pip.PackageInfo], +) -> typing.List[rez_pip.pip.PackageGroup]: + data = [] + for index, package in enumerate(packages[:]): + if packaging.utils.canonicalize_name(package.name) in [ + "pyside6", + "pyside6-addons", + "pyside6-essentials", + ]: + data.append(package) + packages.remove(package) + return [rez_pip.pip.PackageGroup(data)] + + +@rez_pip.plugins.hookimpl +def cleanup(dist: importlib_metadata.Distribution, path: str) -> None: + if packaging.utils.canonicalize_name(dist.name) not in [ + "pyside6", + "pyside6-addons", + "pyside6-essentials", + ]: + return + + shutil.rmtree(os.path.join(path, "python", "shiboken6")) + shutil.rmtree(os.path.join(path, "python", "shiboken6_generator")) diff --git a/src/rez_pip/plugins/__init__.py b/src/rez_pip/plugins/__init__.py new file mode 100644 index 0000000..1f98442 --- /dev/null +++ b/src/rez_pip/plugins/__init__.py @@ -0,0 +1,118 @@ +"""Plugin system.""" +import sys +import typing +import logging +import functools + +import pluggy + +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + +import rez.package_maker + +if typing.TYPE_CHECKING: + import rez_pip.pip + +__all__ = [ + "hookimpl", +] + + +def __dir__() -> typing.List[str]: + return __all__ + + +__LOG = logging.getLogger(__name__) + +F = typing.TypeVar("F", bound=typing.Callable[..., typing.Any]) +hookspec = typing.cast(typing.Callable[[F], F], pluggy.HookspecMarker("rez-pip")) +hookimpl = typing.cast(typing.Callable[[F], F], pluggy.HookimplMarker("rez-pip")) + + +class PluginSpec: + @hookspec + def prePipResolve( + self, packages: typing.List[str], requirements: typing.List[str] + ) -> None: + """ + Take an action before resolving the packages using pip. + The packages argument should not be modified in any way. + """ + + @hookspec + def postPipResolve(self, packages: typing.List["rez_pip.pip.PackageInfo"]) -> None: + """ + Take an action after resolving the packages using pip. + The packages argument should not be modified in any way. + """ + + @hookspec + def groupPackages( # type: ignore[empty-body] + self, packages: typing.List["rez_pip.pip.PackageInfo"] + ) -> typing.List["rez_pip.pip.PackageGroup"]: + """ + Merge packages into groups of packages. The name and version of the first package + in the group will be used as the name and version for the rez package. + + The hook must pop grouped packages out of the "packages" variable. + """ + + @hookspec + def cleanup(self, dist: importlib_metadata.Distribution, path: str) -> None: + """Cleanup installed distribution""" + + @hookspec + def metadata(self, package: rez.package_maker.PackageMaker) -> None: + """ + Modify/inject metadata in the rez package. The plugin is expected to modify + "package" in place. + """ + + +@functools.lru_cache() +def getManager() -> pluggy.PluginManager: + """ + Returns the plugin manager. The return value will be cached on first call + and the cached value will be return in subsequent calls. + """ + import rez_pip.plugins.PySide6 + import rez_pip.plugins.shiboken6 + + manager = pluggy.PluginManager("rez-pip") + # manager.trace.root.setwriter(print) + # manager.enable_tracing() + + manager.add_hookspecs(PluginSpec) + + manager.register(rez_pip.plugins.PySide6) + manager.register(rez_pip.plugins.shiboken6) + + manager.load_setuptools_entrypoints("rez-pip") + + # print(list(itertools.chain(*manager.hook.prePipResolve(packages=["asd"])))) + return manager + + +def getHook() -> PluginSpec: + """ + Returns the hook attribute from the manager. This is allows + to have type hints at the caller sites. + + Inspired by https://stackoverflow.com/a/54695761. + """ + manager = getManager() + return typing.cast(PluginSpec, manager.hook) + + +def _getHookImplementations() -> typing.Dict[str, typing.List[str]]: + manager = getManager() + + implementations = {} + for name, plugin in manager.list_name_plugin(): + implementations[name] = [ + caller.name for caller in manager.get_hookcallers(plugin) + ] + return implementations diff --git a/src/rez_pip/plugins/shiboken6.py b/src/rez_pip/plugins/shiboken6.py new file mode 100644 index 0000000..27424b7 --- /dev/null +++ b/src/rez_pip/plugins/shiboken6.py @@ -0,0 +1,20 @@ +import os +import sys +import shutil + +import packaging.utils + +import rez_pip.plugins + +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + +@rez_pip.plugins.hookimpl +def cleanup(dist: importlib_metadata.Distribution, path: str) -> None: + if packaging.utils.canonicalize_name(dist.name) == "shiboken6": + path = os.path.join(path, "python", "PySide6") + print(f"Removing {path!r}") + shutil.rmtree(path) diff --git a/src/rez_pip/rez.py b/src/rez_pip/rez.py index 11f5def..c5796ff 100644 --- a/src/rez_pip/rez.py +++ b/src/rez_pip/rez.py @@ -20,29 +20,46 @@ import rez_pip.pip import rez_pip.utils +import rez_pip.plugins _LOG = logging.getLogger(__name__) def createPackage( - dist: importlib_metadata.Distribution, - isPure: bool, + dists: typing.List[importlib_metadata.Distribution], pythonVersion: rez.version.Version, - nameCasings: typing.List[str], installedWheelsDir: str, - wheelURL: str, + urls: typing.List[str], prefix: typing.Optional[str] = None, release: bool = False, ) -> None: - _LOG.info(f"Creating rez package for {dist.name}") - name = rez_pip.utils.pythontDistributionNameToRez(dist.name) - version = rez_pip.utils.pythonDistributionVersionToRez(dist.version) - - requirements = rez_pip.utils.getRezRequirements(dist, pythonVersion, isPure, []) - - requires = requirements.requires - variant_requires = requirements.variant_requires - metadata = requirements.metadata + _LOG.info( + "Creating rez package for {0}".format(" + ".join(dist.name for dist in dists)) + ) + name = rez_pip.utils.pythontDistributionNameToRez(dists[0].name) + version = rez_pip.utils.pythonDistributionVersionToRez(dists[0].version) + + requires = [] + variant_requires = [] + metadata: typing.Dict[str, typing.Any] = {} + isPure = True + for dist in dists: + requirements = rez_pip.utils.getRezRequirements(dist, pythonVersion, []) + if not metadata: + # For now we only use the metadata from the first package. Far from ideal... + metadata = requirements.metadata + + # TODO: Remove grouped packages (PySide-Addons, etc) + requires += [ + require for require in requirements.requires if require not in requires + ] + variant_requires += [ + require + for require in requirements.variant_requires + if require not in variant_requires + ] + if isPure: + isPure = metadata["is_pure_python"] if prefix: packagesPath = prefix @@ -63,21 +80,30 @@ def make_root(variant: rez.packages.Variant, path: str) -> None: _LOG.info( rf"Installing {variant.qualified_package_name} \[{formattedRequirements}]" ) - if not dist.files: - raise RuntimeError( - f"{dist.name} package has no files registered! Something is wrong maybe?" - ) - - wheelsDirAbsolute = pathlib.Path(installedWheelsDir).resolve() - for src in dist.files: - srcAbsolute = src.locate().resolve() - dest = os.path.join(path, srcAbsolute.relative_to(wheelsDirAbsolute)) - if not os.path.exists(os.path.dirname(dest)): - os.makedirs(os.path.dirname(dest)) - - _LOG.debug(f"Copying {str(srcAbsolute)!r} to {str(dest)!r}") - shutil.copyfile(srcAbsolute, dest) - shutil.copystat(srcAbsolute, dest) + for dist in dists: + if not dist.files: + raise RuntimeError( + f"{dist.name} package has no files registered! Something is wrong maybe?" + ) + + wheelsDirAbsolute = pathlib.Path(installedWheelsDir).resolve() + for src in dist.files: + srcAbsolute: pathlib.Path = src.locate().resolve() + dest = os.path.join( + path, + os.path.sep.join( + srcAbsolute.relative_to(wheelsDirAbsolute).parts[1:] + ), + ) + # print(dest) + if not os.path.exists(os.path.dirname(dest)): + os.makedirs(os.path.dirname(dest)) + + _LOG.debug(f"Copying {str(srcAbsolute)!r} to {str(dest)!r}") + shutil.copyfile(srcAbsolute, dest) + shutil.copystat(srcAbsolute, dest) + + rez_pip.plugins.getHook().cleanup(dist=dist, path=path) with rez.package_maker.make_package( name, packagesPath, make_root=make_root, skip_existing=True, warn_on_skip=False @@ -113,8 +139,8 @@ def make_root(variant: rez.packages.Variant, path: str) -> None: pkg.pip = { "name": dist.name, "version": dist.version, - "is_pure_python": metadata["is_pure_python"], - "wheel_url": wheelURL, + "is_pure_python": isPure, + "wheel_urls": urls, "rez_pip_version": importlib_metadata.version("rez-pip"), } @@ -126,6 +152,8 @@ def make_root(variant: rez.packages.Variant, path: str) -> None: pkg.pip["metadata"] = remainingMetadata + rez_pip.plugins.getHook().metadata(package=pkg) + _LOG.info( f"[bold]Created {len(pkg.installed_variants)} variants and skipped {len(pkg.skipped_variants)}" ) diff --git a/src/rez_pip/utils.py b/src/rez_pip/utils.py index 641dd01..e4acc00 100644 --- a/src/rez_pip/utils.py +++ b/src/rez_pip/utils.py @@ -14,6 +14,8 @@ import packaging.specifiers import packaging.requirements +import rez_pip.install + _LOG = logging.getLogger(__name__) @@ -466,7 +468,6 @@ def convertMarker(marker: str) -> typing.List[str]: def getRezRequirements( installedDist: importlib_metadata.Distribution, pythonVersion: rez.version.Version, - isPure: bool, nameCasings: typing.Optional[typing.List[str]] = None, ) -> RequirementsDict: """Get requirements of the given dist, in rez-compatible format. @@ -517,6 +518,7 @@ def getRezRequirements( # python build frontends during install has_entry_points_scripts = bool(installedDist.entry_points) + isPure = rez_pip.install.isWheelPure(installedDist) # assume package is platform- and arch- specific if it isn't pure python if not isPure or has_entry_points_scripts: sys_requires.update(["platform", "arch"])