diff --git a/README.md b/README.md index 8858bf4..3b57fb5 100644 --- a/README.md +++ b/README.md @@ -55,12 +55,11 @@ Currently repositories are required to be signed and you need to provide signing ```python from repopulator import AptRepo, PgpSigner -from pathlib import Path repo = AptRepo() -package1 = repo.add_package(Path('/path/to/awesome_3.14_amd64.deb')) -package2 = repo.add_package(Path('/path/to/awesome_3.14_arm64.deb')) +package1 = repo.add_package('/path/to/awesome_3.14_amd64.deb') +package2 = repo.add_package('/path/to/awesome_3.14_arm64.deb') dist = repo.add_distribution('jammy', origin='my packages', @@ -74,7 +73,7 @@ repo.assign_package(package2, dist, component='main') signer = PgpSigner('name_of_key_to_use', 'password_of_that_key') -repo.export(Path('/path/of/new/repo'), signer) +repo.export('/path/of/new/repo', signer) ``` @@ -82,15 +81,14 @@ repo.export(Path('/path/of/new/repo'), signer) ```python from repopulator import RpmRepo, PgpSigner -from pathlib import Path repo = RpmRepo() -repo.add_package(Path('/path/to/awesome-3.14-1.el9.x86_64.rpm')) -repo.add_package(Path('/path/to/awesome-3.14-1.el9.aarch64.rpm')) +repo.add_package('/path/to/awesome-3.14-1.el9.x86_64.rpm') +repo.add_package('/path/to/awesome-3.14-1.el9.aarch64.rpm') signer = PgpSigner('name_of_key_to_use', 'password_of_that_key') -repo.export(Path('/path/of/new/repo'), signer) +repo.export('/path/of/new/repo', signer) ``` @@ -98,16 +96,16 @@ repo.export(Path('/path/of/new/repo'), signer) ```python from repopulator import PacmanRepo, PgpSigner -from pathlib import Path repo = PacmanRepo('myrepo') # if .sig file is present next to the .zst file it will be used for signature # otherwise new signature will be generated at export time -repo.add_package(Path('/path/to/awesome-3.14-1-x86_64.pkg.tar.zst')) +repo.add_package('/path/to/awesome-3.14-1-x86_64.pkg.tar.zst') +repo.add_package('/path/to/another-1.2-1-x86_64.pkg.tar.zst') signer = PgpSigner('name_of_key_to_use', 'password_of_that_key') -repo.export(Path('/path/of/new/repo'), signer) +repo.export('/path/of/new/repo', signer) ``` @@ -115,18 +113,16 @@ repo.export(Path('/path/of/new/repo'), signer) ```python from repopulator import PacmanRepo, PkiSigner -from pathlib import Path repo = PacmanRepo('my repo description') -repo.add_package(Path('/path/to/awesome-3.14-r0.apk')) -repo.add_package(Path('/path/to/another-1.23-r0.apk')) +repo.add_package('/path/to/awesome-3.14-r0.apk') +repo.add_package('/path/to/another-1.23-r0.apk') -signer = PkiSigner(Path('/path/to/private/key'), 'password_or_None') +signer = PkiSigner('/path/to/private/key', 'password_or_None') -# The last argument is the 'name' of the signer to use -# Unlike `pkg` tool we do not parse it out of private key filename -# and do not require you to name key files in certain way -repo.export(Path('/path/of/new/repo'), signer, 'mymail@mydomain.com-1234abcd') +# Unlike `pkg` tool we do not parse signer name out of private key filename +# so you can name your key files whatever you wish +repo.export('/path/of/new/repo', signer, signer_name = 'mymail@mydomain.com-1234abcd') ``` @@ -134,15 +130,14 @@ repo.export(Path('/path/of/new/repo'), signer, 'mymail@mydomain.com-1234abcd') ```python from repopulator import FreeBSDRepo, PkiSigner -from pathlib import Path repo = FreeBSDRepo() -repo.add_package(Path('/path/to/awesome-3.14.pkg')) -repo.add_package(Path('/path/to/another-1.2.pkg')) +repo.add_package('/path/to/awesome-3.14.pkg') +repo.add_package('/path/to/another-1.2.pkg') -signer = PkiSigner(Path('/path/to/private/key'), 'password_or_None') +signer = PkiSigner('/path/to/private/key', 'password_or_None') -repo.export(Path('/path/of/new/repo'), signer) +repo.export('/path/of/new/repo', signer) ``` diff --git a/mkdocs.yml b/mkdocs.yml index 5964456..072aa86 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,6 +18,8 @@ plugins: default_handler: python handlers: python: + import: + - https://docs.python.org/3/objects.inv options: paths: [src] docstring_style: google diff --git a/src/repopulator/alpine.py b/src/repopulator/alpine.py index 4a69d14..0638d2e 100644 --- a/src/repopulator/alpine.py +++ b/src/repopulator/alpine.py @@ -19,10 +19,11 @@ from pathlib import Path from datetime import datetime, timezone from io import BytesIO +from os import PathLike from repopulator.pki_signer import PkiSigner -from .util import NoPublicConstructor, PackageParsingException, VersionKey, lower_bound +from .util import NoPublicConstructor, PackageParsingException, VersionKey, ensure_one_line_str, lower_bound, path_from_pathlike from typing import IO, Any, KeysView, Mapping, Optional, Sequence @@ -195,10 +196,10 @@ def __init__(self, desc: str): when performing `apk update` """ - self.__desc = desc + self.__desc = ensure_one_line_str(desc, 'desc') self.__packages: dict[str, list[AlpinePackage]] = {} - def add_package(self, path: Path, force_arch: Optional[str] = None) -> AlpinePackage: + def add_package(self, path: str | PathLike[str], force_arch: Optional[str] = None) -> AlpinePackage: """Adds a package to the repository Args: @@ -210,6 +211,7 @@ def add_package(self, path: Path, force_arch: Optional[str] = None) -> AlpinePac an AlpinePackage object for the added package """ + path = path_from_pathlike(path) package = AlpinePackage._load(path, force_arch) if package.arch == 'noarch': raise ValueError('package has "noarch" architecture, you must use force_arch parameter to specify which repo architecture to assign it to') @@ -258,7 +260,7 @@ def packages(self, arch: str) -> Sequence[AlpinePackage]: """Packages for a given architecture""" return self.__packages[arch] - def export(self, root: Path, signer: PkiSigner, key_name: str, + def export(self, root: str | PathLike[str], signer: PkiSigner, signer_name: str, now: Optional[datetime] = None, keep_expanded: bool = False): """Export the repository into a given folder @@ -274,7 +276,7 @@ def export(self, root: Path, signer: PkiSigner, key_name: str, signer: A PkiSigner instance to use for signing the repository. Note that this is used to only sign the repository itself, not the packages in it. The packages need to be signed ahead of time which usually happens automatically if you use `abuild` tool - key_name: The "name" of the signer to use. It is usually something like "mymail@mydomain.com-1234abcd" + signer_name: The "name" of the signer to use. It is usually something like "mymail@mydomain.com-1234abcd" (see https://wiki.alpinelinux.org/wiki/Abuild_and_Helpers#Setting_up_the_build_environment for details). Unlike what `pkg` tool does it is not parsed out of private key filename - you have to pass it here manually. now: optional timestamp to use when generating files (including various timestamp fields *inside* files). @@ -285,6 +287,7 @@ def export(self, root: Path, signer: PkiSigner, key_name: str, if now is None: now = datetime.now(timezone.utc) + root = path_from_pathlike(root) expanded = root / 'expanded' if expanded.exists(): shutil.rmtree(expanded) @@ -321,7 +324,7 @@ def norm(info: tarfile.TarInfo): archive.add(apkindex, arcname=apkindex.name, filter=norm) sig_tgz = expanded_arch_dir / 'sig.tgz' - self.__create_index_signature(index_tgz, sig_tgz, signer, key_name, now) + self.__create_index_signature(index_tgz, sig_tgz, signer, signer_name, now) arch_dir = root / arch arch_dir.mkdir(parents=True, exist_ok=True) @@ -340,13 +343,13 @@ def norm(info: tarfile.TarInfo): shutil.rmtree(expanded) @staticmethod - def __create_index_signature(path: Path, sig_path: Path, signer: PkiSigner, key_name: str, now: datetime): + def __create_index_signature(path: Path, sig_path: Path, signer: PkiSigner, signer_name: str, now: datetime): signature = signer.get_alpine_signature(path) with open(sig_path, 'wb') as f_out: with gzip.GzipFile(filename='', mode='wb', fileobj=f_out, mtime=int(now.timestamp())) as f_zip: python_typing_is_dumb: Any = f_zip with tarfile.open(mode="w:", fileobj=python_typing_is_dumb) as archive: - info = tarfile.TarInfo(f'.SIGN.RSA.{key_name}.rsa.pub') + info = tarfile.TarInfo(f'.SIGN.RSA.{signer_name}.rsa.pub') info.uid = 0 info.gid = 0 info.uname = '' diff --git a/src/repopulator/apt.py b/src/repopulator/apt.py index 5d61b14..9c6ffaa 100644 --- a/src/repopulator/apt.py +++ b/src/repopulator/apt.py @@ -21,7 +21,7 @@ from typing import AbstractSet, Any, BinaryIO, Dict, KeysView, Mapping, Optional, Sequence from .pgp_signer import PgpSigner -from .util import NoPublicConstructor, PackageParsingException, VersionKey, lower_bound, file_digest +from .util import NoPublicConstructor, PackageParsingException, VersionKey, ensure_one_line_str, lower_bound, file_digest, path_from_pathlike class AptPackage(metaclass=NoPublicConstructor): @@ -166,7 +166,7 @@ class AptDistribution(metaclass=NoPublicConstructor): @classmethod def _new(cls, - path: PurePosixPath | str, + path: PurePosixPath, origin: str, label: str, suite: str, @@ -175,7 +175,7 @@ def _new(cls, return cls._create(path, origin, label, suite, version, description) def __init__(self, - path: PurePosixPath | str, + path: PurePosixPath, origin: str, label: str, suite: str, @@ -185,11 +185,7 @@ def __init__(self, Use AptRepo.add_distribution to create instances of this class """ - path = path if isinstance(path, PurePosixPath) else PurePosixPath(path) - if path.is_absolute(): - raise ValueError('path value must be a relative path') self.__path = path - self.origin = origin self.label = label self.suite = suite @@ -380,6 +376,14 @@ def add_distribution(self, a new AptDistribution object """ + path = path if isinstance(path, PurePosixPath) else PurePosixPath(path) + if path.is_absolute(): + raise ValueError('path value must be a relative path') + origin = ensure_one_line_str(origin, 'origin') + label = ensure_one_line_str(label, 'label') + suite = ensure_one_line_str(suite, 'sutie') + version = ensure_one_line_str(version, 'version') + description = ensure_one_line_str(description, 'description') dist = AptDistribution._new(path, origin=origin, label=label, suite=suite, version=version, description=description) if dist in self.__distributions: raise ValueError('Duplicate distribution') @@ -399,7 +403,7 @@ def del_distribution(self, dist: AptDistribution): except KeyError: pass - def add_package(self, path: Path) -> AptPackage: + def add_package(self, path: str | os.PathLike[str]) -> AptPackage: """Adds a package to the repository Adding a package to the repository simply adds it to the pool of available packages. @@ -412,6 +416,7 @@ def add_package(self, path: Path) -> AptPackage: an AptPackage object for the added package """ + path = path_from_pathlike(path) package = AptPackage._load(path, path.name) idx = lower_bound(self.__packages, package, lambda x, y: x.repo_filename < y.repo_filename) if idx < len(self.__packages) and self.__packages[idx].repo_filename == package.repo_filename: @@ -478,7 +483,7 @@ def packages(self) -> Sequence[AptPackage]: """Packages in this repository""" return self.__packages - def export(self, root: Path, signer: PgpSigner, now: Optional[datetime] = None): + def export(self, root: str | os.PathLike[str], signer: PgpSigner, now: Optional[datetime] = None): """Export the repository into a given folder. This actually creates an on-disk repository suitable to serve to APT clients. If the directory to export to @@ -499,7 +504,7 @@ def export(self, root: Path, signer: PgpSigner, now: Optional[datetime] = None): if now is None: now = datetime.now(timezone.utc) - + root = path_from_pathlike(root) dists = root / 'dists' if dists.exists(): shutil.rmtree(dists) diff --git a/src/repopulator/freebsd.py b/src/repopulator/freebsd.py index 0162c0f..b49570f 100644 --- a/src/repopulator/freebsd.py +++ b/src/repopulator/freebsd.py @@ -17,11 +17,12 @@ from pathlib import Path from datetime import datetime, timezone +from os import PathLike from typing import Any, BinaryIO, Mapping, Optional, Sequence from .pki_signer import PkiSigner -from .util import NoPublicConstructor, PackageParsingException, lower_bound, VersionKey, file_digest +from .util import NoPublicConstructor, PackageParsingException, lower_bound, VersionKey, file_digest, path_from_pathlike class FreeBSDPackage(metaclass=NoPublicConstructor): @@ -122,7 +123,7 @@ def __init__(self): """Constructor for FreeBSDRepo class""" self.__packages: list[FreeBSDPackage] = [] - def add_package(self, path: Path) -> FreeBSDPackage: + def add_package(self, path: str | PathLike[str]) -> FreeBSDPackage: """Adds a package to the repository Args: @@ -130,6 +131,7 @@ def add_package(self, path: Path) -> FreeBSDPackage: Returns: a FreeBSDPackage object for the added package """ + path = path_from_pathlike(path) package = FreeBSDPackage._load(path, path.name) for existing in self.__packages: if existing.repo_filename == package.repo_filename: @@ -164,7 +166,7 @@ def packages(self) -> Sequence[FreeBSDPackage]: return self.__packages - def export(self, root: Path, signer: PkiSigner, now: Optional[datetime] = None, keep_expanded: bool = False): + def export(self, root: str | PathLike[str], signer: PkiSigner, now: Optional[datetime] = None, keep_expanded: bool = False): """Export the repository into a given folder This actually creates an on-disk repository suitable to serve to `pkg` clients. If the directory to export to @@ -185,6 +187,7 @@ def export(self, root: Path, signer: PkiSigner, now: Optional[datetime] = None, if now is None: now = datetime.now(timezone.utc) + root = path_from_pathlike(root) packagesite = root / 'packagesite' if packagesite.exists(): shutil.rmtree(packagesite) diff --git a/src/repopulator/pacman.py b/src/repopulator/pacman.py index 5ce19d0..fa7e6e1 100644 --- a/src/repopulator/pacman.py +++ b/src/repopulator/pacman.py @@ -17,9 +17,10 @@ from pathlib import Path from datetime import datetime, timezone +from os import PathLike from .pgp_signer import PgpSigner -from .util import NoPublicConstructor, PackageParsingException, VersionKey, file_digest, lower_bound +from .util import NoPublicConstructor, PackageParsingException, VersionKey, ensure_one_line_str, file_digest, lower_bound, path_from_pathlike from typing import IO, Any, BinaryIO, KeysView, Mapping, Optional, Sequence @@ -195,10 +196,10 @@ def __init__(self, name: str): Args: name: repository name. """ - self.__name = name + self.__name = ensure_one_line_str(name, 'name') self.__packages: dict[str, list[PacmanPackage]] = {} - def add_package(self, path: Path) -> PacmanPackage: + def add_package(self, path: str | PathLike[str]) -> PacmanPackage: """Adds a package to the repository Args: @@ -207,6 +208,7 @@ def add_package(self, path: Path) -> PacmanPackage: Returns: a PacmanPackage object for the added package """ + path = path_from_pathlike(path) package = PacmanPackage._load(path, path.name) arch_packages = self.__packages.setdefault(package.arch, []) for idx, existing in enumerate(arch_packages): diff --git a/src/repopulator/pgp_signer.py b/src/repopulator/pgp_signer.py index 8e1aba8..4001bf9 100644 --- a/src/repopulator/pgp_signer.py +++ b/src/repopulator/pgp_signer.py @@ -10,8 +10,11 @@ import subprocess from pathlib import Path +from os import PathLike from typing import Optional +from .util import path_from_pathlike + class PgpSigner: """Implementation of PGP signing @@ -23,16 +26,16 @@ class PgpSigner: You are required to supply key name and password for signing. Signing is done non-interactively without any user prompts. """ - def __init__(self, *, key_name: str, key_pwd: str, homedir: Optional[str | Path] = None): + def __init__(self, *, key_name: str, key_pwd: str, homedir: Optional[str | PathLike[str]] = None): """Constructor for PgpSigner class Args: key_name: name or identifier of the key to use key_pwd: password of the key homedir: GPG home directory. If not specified the gpg defaults are used (including - honoring GNUPGHOME environment variable) + honoring GNUPGHOME environment variable) """ - self.__homedir = homedir + self.__homedir = path_from_pathlike(homedir) if homedir is not None else None self.__key_name = key_name self.__key_pwd = key_pwd diff --git a/src/repopulator/pki_signer.py b/src/repopulator/pki_signer.py index c0a0ecd..873ff03 100644 --- a/src/repopulator/pki_signer.py +++ b/src/repopulator/pki_signer.py @@ -9,6 +9,7 @@ import hashlib from pathlib import Path +from os import PathLike from cryptography.hazmat.backends.openssl.backend import backend as openssl_backend from cryptography.hazmat.primitives import hashes as crypto_hashes @@ -29,7 +30,7 @@ class PkiSigner: that it will be used for. """ - def __init__(self, priv_key_path: Path, priv_key_passwd: Optional[str]): + def __init__(self, priv_key_path: str | PathLike[str], priv_key_passwd: Optional[str]): """Constructor for PkiSigner class The private key file is read once during the construction and not used again diff --git a/src/repopulator/rpm.py b/src/repopulator/rpm.py index 149999f..a055f88 100644 --- a/src/repopulator/rpm.py +++ b/src/repopulator/rpm.py @@ -25,7 +25,7 @@ from .rpmfile import open as rpmfile_open from .pgp_signer import PgpSigner -from .util import ImmutableDict, NoPublicConstructor, find_if, lower_bound, VersionKey, file_digest, indent_tree +from .util import ImmutableDict, NoPublicConstructor, find_if, lower_bound, VersionKey, file_digest, indent_tree, path_from_pathlike _RPMSENSE_ANY = 0 _RPMSENSE_LESS = 1 << 1 @@ -506,7 +506,7 @@ def __init__(self): """Constructor for RpmRepo class""" self.__packages: list[RpmPackage] = [] - def add_package(self, path: Path) -> RpmPackage: + def add_package(self, path: str | os.PathLike[str]) -> RpmPackage: """Adds a package to the repository Args: @@ -514,6 +514,7 @@ def add_package(self, path: Path) -> RpmPackage: Returns: an RpmPackage object for the added package """ + path = path_from_pathlike(path) package = RpmPackage._load(path, path.name) for existing in self.__packages: if existing.pkgid == package.pkgid: @@ -547,7 +548,7 @@ def packages(self) -> Sequence[RpmPackage]: """Packages in the repository""" return self.__packages - def export(self, root: Path, signer: PgpSigner, now: Optional[datetime] = None, keep_expanded: bool = False): + def export(self, root: str | os.PathLike[str], signer: PgpSigner, now: Optional[datetime] = None, keep_expanded: bool = False): """Export the repository into a given folder This actually creates an on-disk repository suitable to serve to `pacman` clients. If the directory to export to @@ -568,6 +569,7 @@ def export(self, root: Path, signer: PgpSigner, now: Optional[datetime] = None, if now is None: now = datetime.now(timezone.utc) + root = path_from_pathlike(root) repodata = root / 'repodata' if repodata.exists(): shutil.rmtree(repodata) diff --git a/src/repopulator/util.py b/src/repopulator/util.py index bdd3843..ffa5b47 100644 --- a/src/repopulator/util.py +++ b/src/repopulator/util.py @@ -12,6 +12,9 @@ import xml.etree.ElementTree as ET +from os import PathLike +from pathlib import Path + from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Type, TypeVar Key = TypeVar('Key') @@ -39,6 +42,23 @@ def lower_bound(seq: Sequence[Any], obj: Any, comp: Callable[[Any, Any], bool] = class PackageParsingException(Exception): """Raised when package parsing fails""" +def path_from_pathlike(arg: str | PathLike[str]): + """Coerces a pathlike argument to a Path""" + return Path(arg) + +def ensure_str(arg: Any, arg_name: str) -> str: + """ensures that the arg is str""" + if isinstance(arg, str): + return arg + raise TypeError(f'{arg_name} must be str') + +def ensure_one_line_str(arg: Any, arg_name: str) -> str: + """ensures that the arg is str and has no line breaks""" + arg = ensure_str(arg, arg_name) + if arg.find('\n') != -1: + raise ValueError(f'{arg_name} must not contain line breaks') + return arg + class VersionKey: """Representation of a package version