Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use py-rattler instead of conda to create the python package #90

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,6 @@ dmypy.json
.vscode
/docs/build/
tests/data/rez_repo/
tests/data/_tmp_download/
tests/data/_conda/
docs/bin/
.idea/
4 changes: 3 additions & 1 deletion pytest.ini
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ addopts =
--cov-report=html
--durations=0

norecursedirs = rez_repo
norecursedirs =
rez_repo
_conda

markers =
integration: mark the tests as integration tests
Expand Down
99 changes: 40 additions & 59 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import time
import typing
import asyncio
import pathlib
import zipfile
import platform
Expand All @@ -10,6 +11,7 @@
import urllib.request

import pytest
import rattler
import rez.config
import rez.packages
import rez.package_bind
Expand All @@ -18,7 +20,7 @@
from . import utils

DATA_ROOT_DIR = os.path.join(os.path.dirname(__file__), "data")
DOWNLOAD_DIR = os.path.join(DATA_ROOT_DIR, "_tmp_download")
CONDA_DIR = os.path.join(DATA_ROOT_DIR, "_conda")

phaseReportKey = pytest.StashKey[typing.Dict[str, pytest.CollectReport]]()

Expand Down Expand Up @@ -152,53 +154,18 @@ def rezRepo() -> typing.Generator[str, None, None]:
rez.config.config.release_packages_path = originalConfig.release_packages_path


def downloadPythonVersion(
version: str, printer_session: typing.Callable[[str], None]
) -> str:
"""
Download a Python interpreter for a given version

:param version: Python version to download.
:param printer_session: Pytest session printer.
:returns: Path to downloaded artifact.
"""
try:
os.makedirs(DOWNLOAD_DIR)
except FileExistsError:
pass

url = f"https://globalcdn.nuget.org/packages/python.{version}.nupkg"

path = os.path.join(
DOWNLOAD_DIR, f"python-{version}-{platform.system().lower()}.nupkg"
)

if os.path.exists(path):
printer_session(f"Skipping {url} because {path!r} already exists")
return path

printer_session(f"Downloading {url} to {path!r}")
with urllib.request.urlopen(url) as archive:
with open(path, "wb") as targetFile:
targetFile.write(archive.read())

return path


@pytest.fixture(
scope="session",
params=[
pytest.param(
# Nuget doesn't have 3.7.16
"3.7.9" if platform.system() == "Windows" else "3.7.16",
"3.7.16",
marks=pytest.mark.py37,
),
pytest.param(
# Nuget doesn't have 3.9.16
"3.9.13" if platform.system() == "Windows" else "3.9.16",
"3.9.16",
marks=pytest.mark.py39,
),
pytest.param("3.11.3", marks=pytest.mark.py311),
pytest.param("3.11.5", marks=pytest.mark.py311),
],
)
def pythonRezPackage(
Expand All @@ -220,24 +187,11 @@ def make_root(variant: rez.packages.Variant, path: str) -> None:

dest = os.path.join(path, "python")

if platform.system() == "Windows":
archivePath = downloadPythonVersion(version, printer_session)
with zipfile.ZipFile(archivePath) as archive:
printer_session(f"Extracting {archivePath!r} to {dest!r}")
archive.extractall(path=dest)
else:
import conda.cli.python_api

printer_session(
f"Installing Python {version} by creating a conda environment at {dest!r}"
)
conda.cli.python_api.run_command(
conda.cli.python_api.Commands.CREATE,
"--prefix",
dest,
f"conda-forge:python={version}",
"pip",
)
printer_session(
f"Installing Python {version} by creating a conda environment at {dest!r}"
)

asyncio.run(createCondaEnvironment(version, dest))

with rez.package_maker.make_package(
"python",
Expand All @@ -253,8 +207,8 @@ def make_root(variant: rez.packages.Variant, path: str) -> None:
]
if platform.system() == "Windows":
commands = [
"env.PATH.prepend('{root}/python/tools')",
"env.PATH.prepend('{root}/python/tools/DLLs')",
"env.PATH.prepend('{root}/python/Library/bin')",
"env.PATH.prepend('{root}/python/Library/lib')",
]

pkg.commands = "\n".join(commands)
Expand All @@ -265,3 +219,30 @@ def make_root(variant: rez.packages.Variant, path: str) -> None:
)

return version


async def createCondaEnvironment(pythonVersion: str, prefixPath: str):
# print("Started fetching repo_data")
repodata = await rattler.fetch_repo_data(
channels=[rattler.Channel("main")],
platforms=[rattler.Platform.current(), rattler.Platform("noarch")],
cache_path=os.path.join(CONDA_DIR, "cache"),
callback=None,
)
# print("Finished fetching repo_data")

# print("Solving dependencies")
solved_dependencies = rattler.solve(
specs=[rattler.MatchSpec(f"python={pythonVersion}")],
available_packages=repodata,
virtual_packages=[p.into_generic() for p in rattler.VirtualPackage.current()],
)
# print("Solved required dependencies")

# print("Creating environment")
await rattler.link(
dependencies=solved_dependencies,
target_prefix=prefixPath,
cache_dir=os.path.join(CONDA_DIR, "pkgs"),
)
# print(f"Created environment")
4 changes: 1 addition & 3 deletions tests/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,4 @@ mock; python_version < "3.8"
types-mock; python_version < "3.8"
build
hatchling
git+https://github.com/conda/[email protected]; platform_system != "Windows"
# 23.1.0 is the last version to support Python 3.7
git+https://github.com/conda/[email protected]; platform_system != "Windows"
py-rattler
Loading