Skip to content

Commit

Permalink
Relaxing Path-yness and clean up public API
Browse files Browse the repository at this point in the history
  • Loading branch information
gershnik committed Jun 8, 2024
1 parent 412c562 commit 89806ef
Show file tree
Hide file tree
Showing 10 changed files with 91 additions and 55 deletions.
43 changes: 19 additions & 24 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,11 @@ Currently repositories are required to be signed and you need to provide signing

```python
from repopulator import AptRepo, PgpSigner
from pathlib import Path

repo = AptRepo()

package1 = repo.add_package(Path('/path/to/awesome_3.14_amd64.deb'))
package2 = repo.add_package(Path('/path/to/awesome_3.14_arm64.deb'))
package1 = repo.add_package('/path/to/awesome_3.14_amd64.deb')
package2 = repo.add_package('/path/to/awesome_3.14_arm64.deb')

dist = repo.add_distribution('jammy',
origin='my packages',
Expand All @@ -74,75 +73,71 @@ repo.assign_package(package2, dist, component='main')

signer = PgpSigner('name_of_key_to_use', 'password_of_that_key')

repo.export(Path('/path/of/new/repo'), signer)
repo.export('/path/of/new/repo', signer)

```

#### YUM/DNF

```python
from repopulator import RpmRepo, PgpSigner
from pathlib import Path

repo = RpmRepo()
repo.add_package(Path('/path/to/awesome-3.14-1.el9.x86_64.rpm'))
repo.add_package(Path('/path/to/awesome-3.14-1.el9.aarch64.rpm'))
repo.add_package('/path/to/awesome-3.14-1.el9.x86_64.rpm')
repo.add_package('/path/to/awesome-3.14-1.el9.aarch64.rpm')

signer = PgpSigner('name_of_key_to_use', 'password_of_that_key')

repo.export(Path('/path/of/new/repo'), signer)
repo.export('/path/of/new/repo', signer)

```

#### Pacman

```python
from repopulator import PacmanRepo, PgpSigner
from pathlib import Path

repo = PacmanRepo('myrepo')
# if .sig file is present next to the .zst file it will be used for signature
# otherwise new signature will be generated at export time
repo.add_package(Path('/path/to/awesome-3.14-1-x86_64.pkg.tar.zst'))
repo.add_package('/path/to/awesome-3.14-1-x86_64.pkg.tar.zst')
repo.add_package('/path/to/another-1.2-1-x86_64.pkg.tar.zst')

signer = PgpSigner('name_of_key_to_use', 'password_of_that_key')

repo.export(Path('/path/of/new/repo'), signer)
repo.export('/path/of/new/repo', signer)

```

#### Alpine apk

```python
from repopulator import PacmanRepo, PkiSigner
from pathlib import Path

repo = PacmanRepo('my repo description')
repo.add_package(Path('/path/to/awesome-3.14-r0.apk'))
repo.add_package(Path('/path/to/another-1.23-r0.apk'))
repo.add_package('/path/to/awesome-3.14-r0.apk')
repo.add_package('/path/to/another-1.23-r0.apk')

signer = PkiSigner(Path('/path/to/private/key'), 'password_or_None')
signer = PkiSigner('/path/to/private/key', 'password_or_None')

# The last argument is the 'name' of the signer to use
# Unlike `pkg` tool we do not parse it out of private key filename
# and do not require you to name key files in certain way
repo.export(Path('/path/of/new/repo'), signer, '[email protected]')
# Unlike `pkg` tool we do not parse signer name out of private key filename
# so you can name your key files whatever you wish
repo.export('/path/of/new/repo', signer, signer_name = '[email protected]')

```

#### FreeBSD pkg

```python
from repopulator import FreeBSDRepo, PkiSigner
from pathlib import Path

repo = FreeBSDRepo()
repo.add_package(Path('/path/to/awesome-3.14.pkg'))
repo.add_package(Path('/path/to/another-1.2.pkg'))
repo.add_package('/path/to/awesome-3.14.pkg')
repo.add_package('/path/to/another-1.2.pkg')

signer = PkiSigner(Path('/path/to/private/key'), 'password_or_None')
signer = PkiSigner('/path/to/private/key', 'password_or_None')

repo.export(Path('/path/of/new/repo'), signer)
repo.export('/path/of/new/repo', signer)

```

2 changes: 2 additions & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ plugins:
default_handler: python
handlers:
python:
import:
- https://docs.python.org/3/objects.inv
options:
paths: [src]
docstring_style: google
Expand Down
19 changes: 11 additions & 8 deletions src/repopulator/alpine.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@
from pathlib import Path
from datetime import datetime, timezone
from io import BytesIO
from os import PathLike

from repopulator.pki_signer import PkiSigner

from .util import NoPublicConstructor, PackageParsingException, VersionKey, lower_bound
from .util import NoPublicConstructor, PackageParsingException, VersionKey, ensure_one_line_str, lower_bound, path_from_pathlike

from typing import IO, Any, KeysView, Mapping, Optional, Sequence

Expand Down Expand Up @@ -195,10 +196,10 @@ def __init__(self, desc: str):
when performing `apk update`
"""

self.__desc = desc
self.__desc = ensure_one_line_str(desc, 'desc')
self.__packages: dict[str, list[AlpinePackage]] = {}

def add_package(self, path: Path, force_arch: Optional[str] = None) -> AlpinePackage:
def add_package(self, path: str | PathLike[str], force_arch: Optional[str] = None) -> AlpinePackage:
"""Adds a package to the repository
Args:
Expand All @@ -210,6 +211,7 @@ def add_package(self, path: Path, force_arch: Optional[str] = None) -> AlpinePac
an AlpinePackage object for the added package
"""

path = path_from_pathlike(path)
package = AlpinePackage._load(path, force_arch)
if package.arch == 'noarch':
raise ValueError('package has "noarch" architecture, you must use force_arch parameter to specify which repo architecture to assign it to')
Expand Down Expand Up @@ -258,7 +260,7 @@ def packages(self, arch: str) -> Sequence[AlpinePackage]:
"""Packages for a given architecture"""
return self.__packages[arch]

def export(self, root: Path, signer: PkiSigner, key_name: str,
def export(self, root: str | PathLike[str], signer: PkiSigner, signer_name: str,
now: Optional[datetime] = None, keep_expanded: bool = False):
"""Export the repository into a given folder
Expand All @@ -274,7 +276,7 @@ def export(self, root: Path, signer: PkiSigner, key_name: str,
signer: A PkiSigner instance to use for signing the repository. Note that this is used to only sign the
repository itself, not the packages in it. The packages need to be signed ahead of time which usually
happens automatically if you use `abuild` tool
key_name: The "name" of the signer to use. It is usually something like "[email protected]"
signer_name: The "name" of the signer to use. It is usually something like "[email protected]"
(see https://wiki.alpinelinux.org/wiki/Abuild_and_Helpers#Setting_up_the_build_environment for details).
Unlike what `pkg` tool does it is not parsed out of private key filename - you have to pass it here manually.
now: optional timestamp to use when generating files (including various timestamp fields *inside* files).
Expand All @@ -285,6 +287,7 @@ def export(self, root: Path, signer: PkiSigner, key_name: str,
if now is None:
now = datetime.now(timezone.utc)

root = path_from_pathlike(root)
expanded = root / 'expanded'
if expanded.exists():
shutil.rmtree(expanded)
Expand Down Expand Up @@ -321,7 +324,7 @@ def norm(info: tarfile.TarInfo):
archive.add(apkindex, arcname=apkindex.name, filter=norm)

sig_tgz = expanded_arch_dir / 'sig.tgz'
self.__create_index_signature(index_tgz, sig_tgz, signer, key_name, now)
self.__create_index_signature(index_tgz, sig_tgz, signer, signer_name, now)

arch_dir = root / arch
arch_dir.mkdir(parents=True, exist_ok=True)
Expand All @@ -340,13 +343,13 @@ def norm(info: tarfile.TarInfo):
shutil.rmtree(expanded)

@staticmethod
def __create_index_signature(path: Path, sig_path: Path, signer: PkiSigner, key_name: str, now: datetime):
def __create_index_signature(path: Path, sig_path: Path, signer: PkiSigner, signer_name: str, now: datetime):
signature = signer.get_alpine_signature(path)
with open(sig_path, 'wb') as f_out:
with gzip.GzipFile(filename='', mode='wb', fileobj=f_out, mtime=int(now.timestamp())) as f_zip:
python_typing_is_dumb: Any = f_zip
with tarfile.open(mode="w:", fileobj=python_typing_is_dumb) as archive:
info = tarfile.TarInfo(f'.SIGN.RSA.{key_name}.rsa.pub')
info = tarfile.TarInfo(f'.SIGN.RSA.{signer_name}.rsa.pub')
info.uid = 0
info.gid = 0
info.uname = ''
Expand Down
25 changes: 15 additions & 10 deletions src/repopulator/apt.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from typing import AbstractSet, Any, BinaryIO, Dict, KeysView, Mapping, Optional, Sequence

from .pgp_signer import PgpSigner
from .util import NoPublicConstructor, PackageParsingException, VersionKey, lower_bound, file_digest
from .util import NoPublicConstructor, PackageParsingException, VersionKey, ensure_one_line_str, lower_bound, file_digest, path_from_pathlike


class AptPackage(metaclass=NoPublicConstructor):
Expand Down Expand Up @@ -166,7 +166,7 @@ class AptDistribution(metaclass=NoPublicConstructor):

@classmethod
def _new(cls,
path: PurePosixPath | str,
path: PurePosixPath,
origin: str,
label: str,
suite: str,
Expand All @@ -175,7 +175,7 @@ def _new(cls,
return cls._create(path, origin, label, suite, version, description)

def __init__(self,
path: PurePosixPath | str,
path: PurePosixPath,
origin: str,
label: str,
suite: str,
Expand All @@ -185,11 +185,7 @@ def __init__(self,
Use AptRepo.add_distribution to create instances of this class
"""

path = path if isinstance(path, PurePosixPath) else PurePosixPath(path)
if path.is_absolute():
raise ValueError('path value must be a relative path')
self.__path = path

self.origin = origin
self.label = label
self.suite = suite
Expand Down Expand Up @@ -380,6 +376,14 @@ def add_distribution(self,
a new AptDistribution object
"""
path = path if isinstance(path, PurePosixPath) else PurePosixPath(path)
if path.is_absolute():
raise ValueError('path value must be a relative path')
origin = ensure_one_line_str(origin, 'origin')
label = ensure_one_line_str(label, 'label')
suite = ensure_one_line_str(suite, 'sutie')
version = ensure_one_line_str(version, 'version')
description = ensure_one_line_str(description, 'description')
dist = AptDistribution._new(path, origin=origin, label=label, suite=suite, version=version, description=description)
if dist in self.__distributions:
raise ValueError('Duplicate distribution')
Expand All @@ -399,7 +403,7 @@ def del_distribution(self, dist: AptDistribution):
except KeyError:
pass

def add_package(self, path: Path) -> AptPackage:
def add_package(self, path: str | os.PathLike[str]) -> AptPackage:
"""Adds a package to the repository
Adding a package to the repository simply adds it to the pool of available packages.
Expand All @@ -412,6 +416,7 @@ def add_package(self, path: Path) -> AptPackage:
an AptPackage object for the added package
"""

path = path_from_pathlike(path)
package = AptPackage._load(path, path.name)
idx = lower_bound(self.__packages, package, lambda x, y: x.repo_filename < y.repo_filename)
if idx < len(self.__packages) and self.__packages[idx].repo_filename == package.repo_filename:
Expand Down Expand Up @@ -478,7 +483,7 @@ def packages(self) -> Sequence[AptPackage]:
"""Packages in this repository"""
return self.__packages

def export(self, root: Path, signer: PgpSigner, now: Optional[datetime] = None):
def export(self, root: str | os.PathLike[str], signer: PgpSigner, now: Optional[datetime] = None):
"""Export the repository into a given folder.
This actually creates an on-disk repository suitable to serve to APT clients. If the directory to export to
Expand All @@ -499,7 +504,7 @@ def export(self, root: Path, signer: PgpSigner, now: Optional[datetime] = None):
if now is None:
now = datetime.now(timezone.utc)


root = path_from_pathlike(root)
dists = root / 'dists'
if dists.exists():
shutil.rmtree(dists)
Expand Down
9 changes: 6 additions & 3 deletions src/repopulator/freebsd.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@

from pathlib import Path
from datetime import datetime, timezone
from os import PathLike

from typing import Any, BinaryIO, Mapping, Optional, Sequence

from .pki_signer import PkiSigner
from .util import NoPublicConstructor, PackageParsingException, lower_bound, VersionKey, file_digest
from .util import NoPublicConstructor, PackageParsingException, lower_bound, VersionKey, file_digest, path_from_pathlike


class FreeBSDPackage(metaclass=NoPublicConstructor):
Expand Down Expand Up @@ -122,14 +123,15 @@ def __init__(self):
"""Constructor for FreeBSDRepo class"""
self.__packages: list[FreeBSDPackage] = []

def add_package(self, path: Path) -> FreeBSDPackage:
def add_package(self, path: str | PathLike[str]) -> FreeBSDPackage:
"""Adds a package to the repository
Args:
path: the path to `.pkg` file for the package.
Returns:
a FreeBSDPackage object for the added package
"""
path = path_from_pathlike(path)
package = FreeBSDPackage._load(path, path.name)
for existing in self.__packages:
if existing.repo_filename == package.repo_filename:
Expand Down Expand Up @@ -164,7 +166,7 @@ def packages(self) -> Sequence[FreeBSDPackage]:
return self.__packages


def export(self, root: Path, signer: PkiSigner, now: Optional[datetime] = None, keep_expanded: bool = False):
def export(self, root: str | PathLike[str], signer: PkiSigner, now: Optional[datetime] = None, keep_expanded: bool = False):
"""Export the repository into a given folder
This actually creates an on-disk repository suitable to serve to `pkg` clients. If the directory to export to
Expand All @@ -185,6 +187,7 @@ def export(self, root: Path, signer: PkiSigner, now: Optional[datetime] = None,
if now is None:
now = datetime.now(timezone.utc)

root = path_from_pathlike(root)
packagesite = root / 'packagesite'
if packagesite.exists():
shutil.rmtree(packagesite)
Expand Down
8 changes: 5 additions & 3 deletions src/repopulator/pacman.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@

from pathlib import Path
from datetime import datetime, timezone
from os import PathLike

from .pgp_signer import PgpSigner
from .util import NoPublicConstructor, PackageParsingException, VersionKey, file_digest, lower_bound
from .util import NoPublicConstructor, PackageParsingException, VersionKey, ensure_one_line_str, file_digest, lower_bound, path_from_pathlike

from typing import IO, Any, BinaryIO, KeysView, Mapping, Optional, Sequence

Expand Down Expand Up @@ -195,10 +196,10 @@ def __init__(self, name: str):
Args:
name: repository name.
"""
self.__name = name
self.__name = ensure_one_line_str(name, 'name')
self.__packages: dict[str, list[PacmanPackage]] = {}

def add_package(self, path: Path) -> PacmanPackage:
def add_package(self, path: str | PathLike[str]) -> PacmanPackage:
"""Adds a package to the repository
Args:
Expand All @@ -207,6 +208,7 @@ def add_package(self, path: Path) -> PacmanPackage:
Returns:
a PacmanPackage object for the added package
"""
path = path_from_pathlike(path)
package = PacmanPackage._load(path, path.name)
arch_packages = self.__packages.setdefault(package.arch, [])
for idx, existing in enumerate(arch_packages):
Expand Down
Loading

0 comments on commit 89806ef

Please sign in to comment.