Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ API ] fix API errors #163

Closed
wants to merge 13 commits into from
32 changes: 19 additions & 13 deletions src/hentai/hentai.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
from importlib.resources import path as resource_path
from pathlib import Path
from typing import Iterable, List, Optional, Set, Tuple, Union
from urllib.parse import urljoin, urlparse
from urllib.parse import urljoin, urlparse, urlencode
from urllib.request import getproxies
from zipfile import ZIP_DEFLATED, ZipFile

Expand Down Expand Up @@ -197,6 +197,12 @@ class Tag:
url: str
count: int

def __eq__(self, other: Union[Tag, str]) -> bool:
if isinstance(other, Tag):
return self.id == other.id
elif isinstance(other, str):
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved
return self.name == other

@classmethod
def get(cls, tags: List[Tag], property_: str) -> str:
"""
Expand Down Expand Up @@ -419,22 +425,24 @@ class RequestHandler(object):
print(response.ok)
```
"""
__slots__ = ['timeout', 'total', 'status_forcelist', 'backoff_factor', 'user_agent', 'proxies']
__slots__ = ['timeout', 'total', 'status_forcelist', 'backoff_factor', 'user_agent', 'proxies', "updated_client"]
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved

_timeout = (5, 5)
_total = 5
_status_forcelist = [413, 429, 500, 502, 503, 504]
_backoff_factor = 1
_user_agent = None
_proxies = None
_updated_client = "https://translate.google.com/translate?sl=vi&tl=en&hl=vi&u={0}&client=webapp"
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved

def __init__(self,
timeout: Tuple[float, float]=_timeout,
total: int=_total,
status_forcelist: List[int]=_status_forcelist,
backoff_factor: int=_backoff_factor,
user_agent: str=_user_agent,
proxies: dict=_proxies):
proxies: dict=_proxies,
updated_client: str=_updated_client) -> None:
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved
"""
Instantiates a new request handler object.
"""
Expand All @@ -444,6 +452,7 @@ def __init__(self,
self.backoff_factor = backoff_factor
self.user_agent = user_agent
self.proxies = proxies
self.updated_client = updated_client

@property
def retry_strategy(self) -> Retry:
Expand Down Expand Up @@ -476,7 +485,7 @@ def get(self, url: str, **kwargs) -> Response:
Returns the GET request encoded in `utf-8`. Adds proxies to this session
on the fly if urllib is able to pick up the system's proxy settings.
"""
response = self.session.get(url, timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs)
response = self.session.get(self.updated_client.format(url), timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs)
response.encoding = 'utf-8'
return response

Expand Down Expand Up @@ -929,15 +938,15 @@ def download(doujins: List[Hentai], delay: float=0, progressbar: bool=False, zip
@staticmethod
def browse_homepage(start_page: int, end_page: int, handler: RequestHandler=RequestHandler(), progressbar: bool=False) -> Set[Hentai]:
"""
Return a list of `Hentai` objects that are currently featured on the homepage
Return a set of `Hentai` objects that are currently featured on the homepage
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved
in range of `[start_page, end_page]`. Each page contains as much as 25 results.
Enable `progressbar` for status feedback in terminal applications.
"""
if start_page > end_page:
raise ValueError(f"{COLORS['red']}{os.strerror(errno.EINVAL)}: start_page={start_page} <= {end_page}=end_page is False{COLORS['reset']}")
data = set()
for page in tqdm(**_progressbar_options(range(start_page, end_page+1), 'Browse', 'page', disable=progressbar)):
with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'), params={'page': page}) as response:
with handler.get(urljoin(Hentai.HOME, 'api/galleries/all' + "?" + urlencode({'page': page})) ) as response:
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved
for raw_json in response.json()['result']:
data.add(Hentai(json=raw_json))
return data
Expand Down Expand Up @@ -977,18 +986,16 @@ def search_by_query(query: str, page: int=1, sort: Sort=Sort.Popular, handler: R
Return a list of `Hentai` objects on page `page` that match this search
`query` sorted by `sort`.
"""
payload = {'query': query, 'page': page, 'sort': sort.value}
with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'), params=payload) as response:
with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'+ "?" + urlencode({'query': query, 'page': page, 'sort': sort.value}))) as response:
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved
return {Hentai(json=raw_json) for raw_json in response.json()['result']}

@staticmethod
def search_by_tag(id_: int, page: int=1, sort: Sort=Sort.Popular, handler: RequestHandler=RequestHandler()) -> Set[Hentai]:
"""
Return a list of `Hentai` objects on page `page` that match this tag
`id_` sorted by `sort`.
"""
payload = {'tag_id': id_, 'page': page, 'sort': sort.value}
with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"), params=payload) as response:
with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged" + "?" + urlencode({'tag_id': id_, 'page': page, 'sort': sort.value}))) as response:
krishsharma0413 marked this conversation as resolved.
Show resolved Hide resolved
return {Hentai(json=raw_json) for raw_json in response.json()['result']}

@staticmethod
Expand All @@ -1006,8 +1013,7 @@ def search_all_by_query(query: str, sort: Sort=Sort.Popular, handler: RequestHan
```
"""
data = set()
payload = {'query': query, 'page': 1, 'sort': sort.value}
with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'), params=payload) as response:
with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+ "?" + urlencode({'query': query, 'page': 1, 'sort': sort.value}))) as response:
for page in tqdm(**_progressbar_options(range(1, int(response.json()['num_pages'])+1), 'Search', 'page', disable=progressbar)):
for doujin in Utils.search_by_query(query, page, sort, handler):
data.add(doujin)
Expand Down
Loading