Release 0.1.0

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

.github/workflows/python-app.yml

@@ -0,0 +1,46 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python application
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest pytest-asyncio pytest-cov
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest and generate coverage report
+      run: |
+        pytest --cov=./ --cov-report=xml
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml
+        files: ./coverage1.xml,./coverage2.xml
+        directory: ./coverage/reports/
+        flags: unittests
+        name: codecov-umbrella
+        fail_ci_if_error: true
+        verbose: true

README.md

@@ -1,9 +1,14 @@
 # Rabbit Ark
 
+[![Code Style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black)
+
+> **Warning!** Only Python 3.9 or higher can guarantee normal operation.  
 > Scalable downloader that downloads asynchronously
 
 - [Rabbit Ark](#rabbit-ark)
   - [Description](#description)
+    - [Download](#download)
+    - [How to use](#how-to-use)
   - [Supported Sites](#supported-sites)
   - [Script](#script)
   - [Special Thanks](#special-thanks)
@@ -12,14 +17,36 @@
 
 This program is inspired by [YouTube-dl](https://github.com/ytdl-org/youtube-dl/) and [Hitomi Downloader](https://github.com/KurtBestor/Hitomi-Downloader)
 
-The download is processed using the [aiomultiprocess](https://github.com/omnilib/aiomultiprocess) module.
+### Download
+
+You can Download in [here](https://github.com/Saebasol/rabbit-ark/releases)
+
+### How to use
+
+First you should specify which extractor to use
+
+```sh
+rabbitark pixiv
+```
+
+Then provide downloadable information, such as a link, via the --downloadable argument.
+
+```sh
+
+rabbitark pixiv --downloadable pixiv-artwork-url
+```
+
+After a while, you can see that the folder is created and the images are down.
+
+For more information, check the -h argument.
 
 ## Supported Sites
 
-|     Site      | URL                 | Extractor release life cycle |
-| :-----------: | ------------------- | ---------------------------- |
-| **Hitomi.la** | <https://hitomi.la> | Alpha                        |
-|   **Pixiv**   | <https://pixiv.net> | Alpha                        |
+|    Site     | URL                   | Extractor release life cycle |
+| :---------: | --------------------- | ---------------------------- |
+| **Hitomi**  | <https://hitomi.la>   | Alpha                        |
+|  **Pixiv**  | <https://pixiv.net>   | Alpha                        |
+| **Youtube** | <https://youtube.com> | Alpha                        |
 
 ## Script
 

rabbitark/__init__.py

@@ -0,0 +1,10 @@
+import sys
+from collections import namedtuple
+
+__version__ = "0.1.0"
+
+__author__ = "Ryu JuHeon"
+
+VersionInfo = namedtuple("VersionInfo", "major minor micro releaselevel serial")
+
+version_info = VersionInfo(major=0, minor=1, micro=0, releaselevel="alpha", serial=0)

rabbitark/__main__.py

@@ -1,14 +1,32 @@
 import argparse
-import asyncio
+import logging
+import platform
+import sys
+from asyncio.events import get_event_loop
 
-from rabbitark.config import config
+from rabbitark.config import Config
+from rabbitark.extractor import load
 from rabbitark.rabbitark import RabbitArk
-from rabbitark.utils.utils import load_cookie_txt
+from rabbitark.utils import load_cookie_txt
 
-parser = argparse.ArgumentParser("rabbitark")
+logger = logging.getLogger("rabbitark")
+logger.setLevel(logging.DEBUG)
+
+# formatter
+formatter = logging.Formatter("%(asctime)s - (%(name)s) - [%(levelname)s]: %(message)s")
+
+# console handler
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+
+config = Config()
 
+parser = argparse.ArgumentParser("rabbitark")
 
-parser.add_argument("extractor", type=str, help="Specifies the extractor to use")
+parser.add_argument("extractor", type=str, help="Specifies the extractor name")
 
 parser.add_argument(
     "--downloadable",
@@ -18,10 +36,20 @@
 
 parser.add_argument("--base", type=str, help="Specifies the pre-created folder")
 
-parser.add_argument("--folder", type=str, help="")
+parser.add_argument("--folder", type=str, help="Specifies the folder name")
 
 parser.add_argument("--cookies", type=str, help="load cookies.txt")
 
+parser.add_argument("--page", type=int, help="Youtube page limit")
+
+parser.add_argument("--custom_extractor", type=str, help="use custom extractor")
+
+parser.add_argument(
+    "--verbose", action="store_true", help="print debugging information"
+)
+
+parser.add_argument("--report", action="store_true", help="save debugging informaion")
+
 args = parser.parse_args()
 
 if args.base:
@@ -33,6 +61,31 @@
 if args.cookies:
     config.COOKIES = load_cookie_txt(args.cookies)
 
-if __name__ == "__main__":
-    ark = RabbitArk(args.extractor)
-    asyncio.run(ark.start(args.downloadable))
+if args.page:
+    config.YOUTUBE_PAGE_LIMIT = args.page
+
+if args.custom_extractor:
+    config.CUSTOM_EXTRACTOR = args.custom_extractor
+
+if args.verbose:
+    ch.setLevel(logging.DEBUG)
+
+if args.report:
+    fh = logging.FileHandler("rabbitark.log")
+    fh.setLevel(logging.DEBUG)
+    fh.setFormatter(formatter)
+    logger.addHandler(fh)
+
+logger.debug("system ver: %s %s", platform.python_implementation(), sys.version)
+logger.debug("platform: %s", platform.platform())
+logger.debug("args: %s", sys.argv[1:])
+
+logger.info("start import extractor")
+load()
+logger.info("sucessfully import extractor")
+
+logger.debug("start loop")
+get_event_loop().run_until_complete(
+    RabbitArk(args.extractor, config).start(args.downloadable)
+)
+logger.debug("complete loop")

rabbitark/abc.py

@@ -0,0 +1,21 @@
+from abc import ABC, ABCMeta, abstractmethod
+from typing import Any, Optional
+
+from aiohttp.client import ClientSession
+
+from rabbitark.config import Config
+from rabbitark.dataclass import DownloadInfo
+
+
+class BaseRequest(ABC):
+    @abstractmethod
+    def __init__(self) -> None:
+        self.session: Optional[ClientSession] = None
+
+
+class BaseExtractor(BaseRequest, metaclass=ABCMeta):
+    @abstractmethod
+    async def get_download_info(
+        self, download_source: Any, config: Config
+    ) -> DownloadInfo:
+        pass

rabbitark/config.py

@@ -1,10 +1,24 @@
-class _Config:
-    __slots__ = ["BASE_DIRECTORY", "FOLDER", "COOKIES"]
+from typing import Optional
 
-    def __init__(self):
+
+class Config:
+    __slots__: list[str] = [
+        "BASE_DIRECTORY",
+        "FOLDER",
+        "COOKIES",
+        "YOUTUBE_PAGE_LIMIT",
+        "CUSTOM_EXTRACTOR",
+        "REQUEST_PER_SESSION",
+    ]
+
+    def __init__(self) -> None:
         self.BASE_DIRECTORY: str = "."
-        self.FOLDER: str = None
-        self.COOKIES: str = {}
+        self.FOLDER: Optional[str] = None
+        self.COOKIES: dict[str, Optional[str]] = {}
+        self.CUSTOM_EXTRACTOR: Optional[str] = None
 
+        # extractor config
+        self.YOUTUBE_PAGE_LIMIT: int = 6
 
-config = _Config()
+        # download config
+        self.REQUEST_PER_SESSION: int = 10

rabbitark/dataclass.py

@@ -0,0 +1,28 @@
+from typing import Any
+
+from rabbitark.utils import Optional, split
+
+
+class Image:
+    def __init__(self, url: str, filename: Optional[str] = None) -> None:
+        self.url: str = url
+        self.filename: str = filename or split(url)
+
+
+class DownloadInfo:
+    def __init__(
+        self, image: list[Image], title: Optional[str] = None, **kwargs: Any
+    ) -> None:
+        self.title = title
+        self.image = image
+        self.kwargs = kwargs
+
+    def to_download(self, path: str):
+        return {image.url: path + image.filename for image in self.image}
+
+
+class Response:
+    def __init__(self, status: int, message: Optional[Any], body: Any) -> None:
+        self.status = status
+        self.message = message
+        self.body = body

rabbitark/downloader.py

@@ -0,0 +1,56 @@
+from os.path import exists
+from typing import Any, Literal, Optional
+
+from aiofiles import open
+from aiofiles.os import mkdir
+from aiohttp.client import ClientSession
+
+from rabbitark.config import Config
+from rabbitark.dataclass import DownloadInfo
+from rabbitark.request import SessionPoolRequest
+
+
+class Downloader(SessionPoolRequest):
+    def __init__(self, config: Config) -> None:
+        self.config = config
+        super().__init__()
+
+    async def download(
+        self,
+        session: ClientSession,
+        url: str,
+        method: Literal["GET"],
+        _: Any,
+        **kwargs: Any,
+    ):
+        filename = kwargs.pop("filename")
+        response = await session.request(method, url, **kwargs)
+        async with open(filename[url], "wb") as f:
+            async for data, _ in response.content.iter_chunks():
+                await f.write(data)
+
+    async def create_folder(self, title: Optional[str] = None) -> str:
+        default_dir = f"{self.config.BASE_DIRECTORY}/{self.config.FOLDER}/"
+        if not exists(default_dir):
+            await mkdir(default_dir)
+
+        if title:
+            if not exists(f"{default_dir}/{title}"):
+                await mkdir(f"{default_dir}/{title}")
+
+            return f"{default_dir}/{title}/"
+
+        return default_dir
+
+    async def start_download(self, download_info: DownloadInfo):
+        directory = await self.create_folder(download_info.title)
+        filename_mapping = download_info.to_download(directory)
+        url_list = list(filename_mapping.keys())
+        await self.request_using_session_pool(
+            self.download,
+            url_list,
+            "GET",
+            request_per_session=self.config.REQUEST_PER_SESSION,
+            filename=filename_mapping,
+            **download_info.kwargs,
+        )