From 3bcb5a4d5205db6da0ec8a956db04f88c675afbd Mon Sep 17 00:00:00 2001 From: Jean-Christophe Morin Date: Sun, 21 Jan 2024 13:36:47 -0500 Subject: [PATCH] Use py-rattler instea of conda to create the python package Signed-off-by: Jean-Christophe Morin --- .gitignore | 2 +- pytest.ini | 4 +- tests/conftest.py | 99 +++++++++++++++++------------------------- tests/requirements.txt | 4 +- 4 files changed, 45 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index 7a75dcf..d43eddd 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,6 @@ dmypy.json .vscode /docs/build/ tests/data/rez_repo/ -tests/data/_tmp_download/ +tests/data/_conda/ docs/bin/ .idea/ diff --git a/pytest.ini b/pytest.ini index d9f1853..45a1076 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,7 +9,9 @@ addopts = --cov-report=html --durations=0 -norecursedirs = rez_repo +norecursedirs = + rez_repo + _conda markers = integration: mark the tests as integration tests diff --git a/tests/conftest.py b/tests/conftest.py index b6c9a80..8650741 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import os import time import typing +import asyncio import pathlib import zipfile import platform @@ -10,6 +11,7 @@ import urllib.request import pytest +import rattler import rez.config import rez.packages import rez.package_bind @@ -18,7 +20,7 @@ from . import utils DATA_ROOT_DIR = os.path.join(os.path.dirname(__file__), "data") -DOWNLOAD_DIR = os.path.join(DATA_ROOT_DIR, "_tmp_download") +CONDA_DIR = os.path.join(DATA_ROOT_DIR, "_conda") phaseReportKey = pytest.StashKey[typing.Dict[str, pytest.CollectReport]]() @@ -152,53 +154,18 @@ def rezRepo() -> typing.Generator[str, None, None]: rez.config.config.release_packages_path = originalConfig.release_packages_path -def downloadPythonVersion( - version: str, printer_session: typing.Callable[[str], None] -) -> str: - """ - Download a Python interpreter for a given version - - :param version: Python version to download. - :param printer_session: Pytest session printer. - :returns: Path to downloaded artifact. - """ - try: - os.makedirs(DOWNLOAD_DIR) - except FileExistsError: - pass - - url = f"https://globalcdn.nuget.org/packages/python.{version}.nupkg" - - path = os.path.join( - DOWNLOAD_DIR, f"python-{version}-{platform.system().lower()}.nupkg" - ) - - if os.path.exists(path): - printer_session(f"Skipping {url} because {path!r} already exists") - return path - - printer_session(f"Downloading {url} to {path!r}") - with urllib.request.urlopen(url) as archive: - with open(path, "wb") as targetFile: - targetFile.write(archive.read()) - - return path - - @pytest.fixture( scope="session", params=[ pytest.param( - # Nuget doesn't have 3.7.16 - "3.7.9" if platform.system() == "Windows" else "3.7.16", + "3.7.16", marks=pytest.mark.py37, ), pytest.param( - # Nuget doesn't have 3.9.16 - "3.9.13" if platform.system() == "Windows" else "3.9.16", + "3.9.16", marks=pytest.mark.py39, ), - pytest.param("3.11.3", marks=pytest.mark.py311), + pytest.param("3.11.5", marks=pytest.mark.py311), ], ) def pythonRezPackage( @@ -220,24 +187,11 @@ def make_root(variant: rez.packages.Variant, path: str) -> None: dest = os.path.join(path, "python") - if platform.system() == "Windows": - archivePath = downloadPythonVersion(version, printer_session) - with zipfile.ZipFile(archivePath) as archive: - printer_session(f"Extracting {archivePath!r} to {dest!r}") - archive.extractall(path=dest) - else: - import conda.cli.python_api - - printer_session( - f"Installing Python {version} by creating a conda environment at {dest!r}" - ) - conda.cli.python_api.run_command( - conda.cli.python_api.Commands.CREATE, - "--prefix", - dest, - f"conda-forge:python={version}", - "pip", - ) + printer_session( + f"Installing Python {version} by creating a conda environment at {dest!r}" + ) + + asyncio.run(createCondaEnvironment(version, dest)) with rez.package_maker.make_package( "python", @@ -253,8 +207,8 @@ def make_root(variant: rez.packages.Variant, path: str) -> None: ] if platform.system() == "Windows": commands = [ - "env.PATH.prepend('{root}/python/tools')", - "env.PATH.prepend('{root}/python/tools/DLLs')", + "env.PATH.prepend('{root}/python/Library/bin')", + "env.PATH.prepend('{root}/python/Library/lib')", ] pkg.commands = "\n".join(commands) @@ -265,3 +219,30 @@ def make_root(variant: rez.packages.Variant, path: str) -> None: ) return version + + +async def createCondaEnvironment(pythonVersion: str, prefixPath: str): + # print("Started fetching repo_data") + repodata = await rattler.fetch_repo_data( + channels=[rattler.Channel("main")], + platforms=[rattler.Platform.current(), rattler.Platform("noarch")], + cache_path=os.path.join(CONDA_DIR, "cache"), + callback=None, + ) + # print("Finished fetching repo_data") + + # print("Solving dependencies") + solved_dependencies = rattler.solve( + specs=[rattler.MatchSpec(f"python={pythonVersion}")], + available_packages=repodata, + virtual_packages=[p.into_generic() for p in rattler.VirtualPackage.current()], + ) + # print("Solved required dependencies") + + # print("Creating environment") + await rattler.link( + dependencies=solved_dependencies, + target_prefix=prefixPath, + cache_dir=os.path.join(CONDA_DIR, "pkgs"), + ) + # print(f"Created environment") diff --git a/tests/requirements.txt b/tests/requirements.txt index 9b93af5..8101103 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,6 +7,4 @@ mock; python_version < "3.8" types-mock; python_version < "3.8" build hatchling -git+https://github.com/conda/conda-package-handling@2.1.0; platform_system != "Windows" -# 23.1.0 is the last version to support Python 3.7 -git+https://github.com/conda/conda@23.1.0; platform_system != "Windows" +py-rattler