From e57ce9242a1597dd12c1260799d46fc9fbd5a249 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Morin Date: Sun, 28 Apr 2024 17:41:23 -0400 Subject: [PATCH] Fix download tests and add a test for skipped downloads Signed-off-by: Jean-Christophe Morin --- tests/test_download.py | 318 ++++++++++++++++++++++++++--------------- 1 file changed, 202 insertions(+), 116 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index cf8530e..91e830a 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -27,32 +27,54 @@ def rezPipVersion(): yield +class Package: + def __init__(self, name: str, content: str, local: bool): + self.name = name + self.content = content + self.local = local + + +class Group: + def __init__(self, packages: typing.List[Package]): + self.packages = packages + + def getPackage(self, name: str) -> Package: + for package in self.packages: + if package.name == name: + return package + raise KeyError(name) + + @pytest.mark.parametrize( - "packages", + "groups", [ - {"package-a": "package-a data"}, - {"package-a": "package-a data", "package-b": "package-b data"}, + [Group([Package("package-a", "package-a data", False)])], + [ + Group([Package("package-a", "package-a data", False)]), + Group([Package("package-b", "package-b data", False)]), + ], ], - ids=["single-package", "multiple-packages"], + ids=["one-group-with-one-package", "multiple-groups-with-one-package"], ) -def test_download(packages: typing.Dict[str, str], tmp_path: pathlib.Path): +def test_download(groups: typing.List[Group], tmp_path: pathlib.Path): sideEffects = tuple() - for content in packages.values(): - mockedContent = mock.MagicMock() - mockedContent.return_value.__aiter__.return_value = [ - [ - content.encode("utf-8"), - None, + for group in groups: + for package in group.packages: + mockedContent = mock.MagicMock() + mockedContent.return_value.__aiter__.return_value = [ + [ + package.content.encode("utf-8"), + None, + ] ] - ] - sideEffects += ( - mock.Mock( - headers={"content-length": 100}, - status=200, - content=mock.Mock(iter_chunks=mockedContent), - ), - ) + sideEffects += ( + mock.Mock( + headers={"content-length": 100}, + status=200, + content=mock.Mock(iter_chunks=mockedContent), + ), + ) mockedGet = mock.AsyncMock() mockedGet.__aenter__.side_effect = sideEffects @@ -60,43 +82,83 @@ def test_download(packages: typing.Dict[str, str], tmp_path: pathlib.Path): with mock.patch.object(aiohttp.ClientSession, "get") as mocked: mocked.return_value = mockedGet - wheels = rez_pip.download.downloadPackages( - [ - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), - download_info=rez_pip.pip.DownloadInfo( - url=f"https://example.com/{package}.whl", - archive_info=rez_pip.pip.ArchiveInfo("hash", {}), - ), - is_direct=True, - requested=True, + _groups = [] + for group in groups: + infos = [] + for package in group.packages: + infos.append( + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata( + name=package.name, version="1.0.0" + ), + download_info=rez_pip.pip.DownloadInfo( + url=f"https://example.com/{package.name}.whl", + archive_info=rez_pip.pip.ArchiveInfo("hash", {}), + ), + is_direct=True, + requested=True, + ) ) - for package in packages - ], - os.fspath(tmp_path), - ) + _groups.append(rez_pip.pip.PackageGroup(infos)) + + wheels = rez_pip.download.downloadPackages(_groups, os.fspath(tmp_path)) assert sorted(wheels) == sorted( - [os.fspath(tmp_path / f"{package}.whl") for package in packages] + [ + os.fspath(tmp_path / f"{package.name}.whl") + for group in groups + for package in group.packages + ] ) - for wheel in wheels: - with open(wheel, "r") as fd: - content = fd.read() - assert packages[os.path.basename(wheel).split(".")[0]] == content + wheelsMapping = {os.path.basename(wheel).split(".")[0]: wheel for wheel in wheels} + + for group in groups: + for package in group.packages: + with open(wheelsMapping[package.name], "r") as fd: + content = fd.read() + assert content == package.content assert mocked.call_args_list == [ mock.call( - f"https://example.com/{package}.whl", + f"https://example.com/{package.name}.whl", headers={ "Content-Type": "application/octet-stream", "User-Agent": "rez-pip/1.2.3.4.5", }, ) - for package in packages + for group in groups + for package in group.packages ] +def test_download_skip_local(tmp_path: pathlib.Path): + groups = [ + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata(name="package-a", version="1.0.0"), + download_info=rez_pip.pip.DownloadInfo( + url="file:///example.com/package-a", + archive_info=rez_pip.pip.ArchiveInfo("hash-a", {}), + ), + is_direct=True, + requested=True, + ) + ] + ) + ] + + mockedGet = mock.AsyncMock() + + with mock.patch.object(aiohttp.ClientSession, "get") as mocked: + mocked.return_value = mockedGet + wheels = rez_pip.download.downloadPackages(groups, os.fspath(tmp_path)) + + assert not mocked.called + assert wheels == [] + + def test_download_multiple_packages_with_failure(tmp_path: pathlib.Path): mockedContent = mock.MagicMock() mockedContent.return_value.__aiter__.return_value = [ @@ -127,27 +189,35 @@ def test_download_multiple_packages_with_failure(tmp_path: pathlib.Path): with pytest.raises(RuntimeError): rez_pip.download.downloadPackages( [ - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata( - name="package-a", version="1.0.0" - ), - download_info=rez_pip.pip.DownloadInfo( - url="https://example.com/package-a", - archive_info=rez_pip.pip.ArchiveInfo("hash-a", {}), - ), - is_direct=True, - requested=True, + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata( + name="package-a", version="1.0.0" + ), + download_info=rez_pip.pip.DownloadInfo( + url="https://example.com/package-a", + archive_info=rez_pip.pip.ArchiveInfo("hash-a", {}), + ), + is_direct=True, + requested=True, + ) + ] ), - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata( - name="package-b", version="1.0.0" - ), - download_info=rez_pip.pip.DownloadInfo( - url="https://example.com/package-b", - archive_info=rez_pip.pip.ArchiveInfo("hash-b", {}), - ), - is_direct=True, - requested=True, + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata( + name="package-b", version="1.0.0" + ), + download_info=rez_pip.pip.DownloadInfo( + url="https://example.com/package-b", + archive_info=rez_pip.pip.ArchiveInfo("hash-b", {}), + ), + is_direct=True, + requested=True, + ) + ] ), ], os.fspath(tmp_path), @@ -178,7 +248,7 @@ def test_download_multiple_packages_with_failure(tmp_path: pathlib.Path): def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): """Test that wheels are re-used if the sha256 matches""" sideEffects = tuple() - packages = [] + groups = [] for package in ["package-a", "package-b"]: content = f"{package} data".encode("utf-8") @@ -186,17 +256,21 @@ def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): hash = hashlib.new("sha256") hash.update(content) - packages.append( - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), - download_info=rez_pip.pip.DownloadInfo( - url=f"https://example.com/{package}.whl", - archive_info=rez_pip.pip.ArchiveInfo( - "hash-a", {"sha256": hash.hexdigest()} - ), - ), - is_direct=True, - requested=True, + groups.append( + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), + download_info=rez_pip.pip.DownloadInfo( + url=f"https://example.com/{package}.whl", + archive_info=rez_pip.pip.ArchiveInfo( + "hash-a", {"sha256": hash.hexdigest()} + ), + ), + is_direct=True, + requested=True, + ) + ] ) ) @@ -222,7 +296,7 @@ def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): with mock.patch.object(aiohttp.ClientSession, "get") as mocked: mocked.return_value = mockedGet1 - rez_pip.download.downloadPackages(packages, str(tmp_path)) + rez_pip.download.downloadPackages(groups, str(tmp_path)) assert mocked.call_args_list == [ mock.call( @@ -241,7 +315,7 @@ def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): ), ] - packages = [] + groups = [] # package-b will be re-used for package in ["package-c", "package-b"]: content = f"{package} data".encode("utf-8") @@ -249,17 +323,21 @@ def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): hash = hashlib.new("sha256") hash.update(content) - packages.append( - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), - download_info=rez_pip.pip.DownloadInfo( - url=f"https://example.com/{package}.whl", - archive_info=rez_pip.pip.ArchiveInfo( - "hash-a", {"sha256": hash.hexdigest()} - ), - ), - is_direct=True, - requested=True, + groups.append( + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), + download_info=rez_pip.pip.DownloadInfo( + url=f"https://example.com/{package}.whl", + archive_info=rez_pip.pip.ArchiveInfo( + "hash-a", {"sha256": hash.hexdigest()} + ), + ), + is_direct=True, + requested=True, + ) + ] ) ) @@ -285,7 +363,7 @@ def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): with mock.patch.object(aiohttp.ClientSession, "get") as mocked: mocked.return_value = mockedGet2 - wheels = rez_pip.download.downloadPackages(packages, str(tmp_path)) + wheels = rez_pip.download.downloadPackages(groups, str(tmp_path)) assert mocked.call_args_list == [ mock.call( @@ -305,7 +383,7 @@ def test_download_reuse_if_same_hash(tmp_path: pathlib.Path): def test_download_redownload_if_hash_changes(tmp_path: pathlib.Path): """Test that wheels are re-used if the sha256 matches""" sideEffects = tuple() - packages = [] + groups = [] for package in ["package-a", "package-b"]: content = f"{package} data".encode("utf-8") @@ -313,17 +391,21 @@ def test_download_redownload_if_hash_changes(tmp_path: pathlib.Path): hash = hashlib.new("sha256") hash.update(content) - packages.append( - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), - download_info=rez_pip.pip.DownloadInfo( - url=f"https://example.com/{package}.whl", - archive_info=rez_pip.pip.ArchiveInfo( - "hash-a", {"sha256": hash.hexdigest()} - ), - ), - is_direct=True, - requested=True, + groups.append( + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), + download_info=rez_pip.pip.DownloadInfo( + url=f"https://example.com/{package}.whl", + archive_info=rez_pip.pip.ArchiveInfo( + "hash-a", {"sha256": hash.hexdigest()} + ), + ), + is_direct=True, + requested=True, + ) + ] ) ) @@ -349,7 +431,7 @@ def test_download_redownload_if_hash_changes(tmp_path: pathlib.Path): with mock.patch.object(aiohttp.ClientSession, "get") as mocked: mocked.return_value = mockedGet1 - rez_pip.download.downloadPackages(packages, str(tmp_path)) + rez_pip.download.downloadPackages(groups, str(tmp_path)) assert mocked.call_args_list == [ mock.call( @@ -368,26 +450,30 @@ def test_download_redownload_if_hash_changes(tmp_path: pathlib.Path): ), ] - packages = [] + groups = [] # package-b will be re-used for package in ["package-a", "package-b"]: content = f"{package} data".encode("utf-8") - packages.append( - rez_pip.pip.PackageInfo( - metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), - download_info=rez_pip.pip.DownloadInfo( - url=f"https://example.com/{package}.whl", - archive_info=rez_pip.pip.ArchiveInfo( - # - # Bad sha256. This will trigger a new download - # - "hash-a", - {"sha256": "asd"}, - ), - ), - is_direct=True, - requested=True, + groups.append( + rez_pip.pip.PackageGroup( + [ + rez_pip.pip.PackageInfo( + metadata=rez_pip.pip.Metadata(name=package, version="1.0.0"), + download_info=rez_pip.pip.DownloadInfo( + url=f"https://example.com/{package}.whl", + archive_info=rez_pip.pip.ArchiveInfo( + # + # Bad sha256. This will trigger a new download + # + "hash-a", + {"sha256": "asd"}, + ), + ), + is_direct=True, + requested=True, + ) + ] ) ) @@ -413,7 +499,7 @@ def test_download_redownload_if_hash_changes(tmp_path: pathlib.Path): with mock.patch.object(aiohttp.ClientSession, "get") as mocked: mocked.return_value = mockedGet2 - wheels = rez_pip.download.downloadPackages(packages, str(tmp_path)) + wheels = rez_pip.download.downloadPackages(groups, str(tmp_path)) assert mocked.call_args_list == [ mock.call(