From 820220746d4392be4c4f82d20f906f492f2980f1 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Wed, 14 Sep 2022 23:48:04 +0530 Subject: [PATCH 01/11] still need work --- src/hentai/hentai.py | 31 +++++++++++++++++++++---------- tests/test_utils.py | 8 ++++---- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 624f550..1020f6a 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -419,7 +419,7 @@ class RequestHandler(object): print(response.ok) ``` """ - __slots__ = ['timeout', 'total', 'status_forcelist', 'backoff_factor', 'user_agent', 'proxies'] + __slots__ = ['timeout', 'total', 'status_forcelist', 'backoff_factor', 'user_agent', 'proxies', "updated_client"] _timeout = (5, 5) _total = 5 @@ -427,6 +427,7 @@ class RequestHandler(object): _backoff_factor = 1 _user_agent = None _proxies = None + _updated_client = "https://translate.google.com/translate?sl=vi&tl=en&hl=vi&u={0}&client=webapp" def __init__(self, timeout: Tuple[float, float]=_timeout, @@ -434,7 +435,8 @@ def __init__(self, status_forcelist: List[int]=_status_forcelist, backoff_factor: int=_backoff_factor, user_agent: str=_user_agent, - proxies: dict=_proxies): + proxies: dict=_proxies, + updated_client: str=_updated_client) -> None: """ Instantiates a new request handler object. """ @@ -444,6 +446,7 @@ def __init__(self, self.backoff_factor = backoff_factor self.user_agent = user_agent self.proxies = proxies + self.updated_client = updated_client @property def retry_strategy(self) -> Retry: @@ -476,7 +479,8 @@ def get(self, url: str, **kwargs) -> Response: Returns the GET request encoded in `utf-8`. Adds proxies to this session on the fly if urllib is able to pick up the system's proxy settings. """ - response = self.session.get(url, timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs) + print(self.updated_client.format(url)) + response = self.session.get(self.updated_client.format(url), timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs) response.encoding = 'utf-8' return response @@ -937,7 +941,8 @@ def browse_homepage(start_page: int, end_page: int, handler: RequestHandler=Requ raise ValueError(f"{COLORS['red']}{os.strerror(errno.EINVAL)}: start_page={start_page} <= {end_page}=end_page is False{COLORS['reset']}") data = set() for page in tqdm(**_progressbar_options(range(start_page, end_page+1), 'Browse', 'page', disable=progressbar)): - with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'), params={'page': page}) as response: + # with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'), params={'page': page}) as response: + with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'+f'?page={page}') ) as response: for raw_json in response.json()['result']: data.add(Hentai(json=raw_json)) return data @@ -977,8 +982,10 @@ def search_by_query(query: str, page: int=1, sort: Sort=Sort.Popular, handler: R Return a list of `Hentai` objects on page `page` that match this search `query` sorted by `sort`. """ - payload = {'query': query, 'page': page, 'sort': sort.value} - with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'), params=payload) as response: + # payload = {'query': query, 'page': page, 'sort': sort.value} + payload = f"?query={query}&page={page}&sort={sort.value}" + # with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'), params=payload) as response: + with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'+payload)) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @staticmethod @@ -987,8 +994,10 @@ def search_by_tag(id_: int, page: int=1, sort: Sort=Sort.Popular, handler: Reque Return a list of `Hentai` objects on page `page` that match this tag `id_` sorted by `sort`. """ - payload = {'tag_id': id_, 'page': page, 'sort': sort.value} - with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"), params=payload) as response: + # payload = {'tag_id': id_, 'page': page, 'sort': sort.value} + payload = f"?tag_id={id_}&page={page}&sort={sort.value}" + # with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"), params=payload) as response: + with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"+payload)) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @staticmethod @@ -1006,8 +1015,10 @@ def search_all_by_query(query: str, sort: Sort=Sort.Popular, handler: RequestHan ``` """ data = set() - payload = {'query': query, 'page': 1, 'sort': sort.value} - with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'), params=payload) as response: + # payload = {'query': query, 'page': 1, 'sort': sort.value} + payload = f"?query={query}&page=1&sort={sort.value}" + # with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+payload), params=payload) as response: + with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+payload)) as response: for page in tqdm(**_progressbar_options(range(1, int(response.json()['num_pages'])+1), 'Search', 'page', disable=progressbar)): for doujin in Utils.search_by_query(query, page, sort, handler): data.add(doujin) diff --git a/tests/test_utils.py b/tests/test_utils.py index a00f7b9..f624435 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -31,10 +31,10 @@ def test_random_hentai(self): response = requests.get(random_hentai.url) self.assertEqual(response.status_code, 200, msg=f"Failing URL: {response.url} (status code: {response.status_code})") - def test_download_queue(self): - Utils.download([self.tiny_evil], progressbar=True, zip_dir=True) - self.assertFalse(self.tiny_evil_dir.is_dir()) - self.assertTrue(self.tiny_evil_zip.is_file()) + # def test_download_queue(self): + # Utils.download([self.tiny_evil], progressbar=True, zip_dir=True) + # self.assertFalse(self.tiny_evil_dir.is_dir()) + # self.assertTrue(self.tiny_evil_zip.is_file()) def test_get_homepage(self): homepage = Utils.get_homepage().popular_now From d70db56180aaa1346ee56f6dfe83ec20439021c2 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Thu, 15 Sep 2022 00:19:22 +0530 Subject: [PATCH 02/11] test_random_hentai updated --- tests/test_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index f624435..f7c030f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -28,8 +28,7 @@ def tearDownClass(cls): def test_random_hentai(self): random_hentai = Utils.get_random_hentai() - response = requests.get(random_hentai.url) - self.assertEqual(response.status_code, 200, msg=f"Failing URL: {response.url} (status code: {response.status_code})") + self.assertEqual(type(random_hentai.json), dict, msg=f"Failing URL: {random_hentai.url} (possibility: doesn't exists(?) or blocked by Cloudflare)") # def test_download_queue(self): # Utils.download([self.tiny_evil], progressbar=True, zip_dir=True) @@ -68,7 +67,7 @@ def test_search_by_tag(self): self.assertTrue(doujin.num_pages, msg="ValueError: NumberOfPages") def test_search_all_by_query(self): - popular_3d = Utils.search_all_by_query(query="tag:3d", sort=Sort.PopularWeek) + popular_3d = Utils.search_all_by_query(query="tag:3dlive", sort=Sort.PopularWeek) for doujin in popular_3d: self.assertIsNotNone(doujin.json, msg="Result should not be 'None'.") self.assertTrue(doujin.id, msg="ValueError: ID") From a7d079da15de89bd218e0ffde5abb6e9b427dd93 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Thu, 15 Sep 2022 00:21:33 +0530 Subject: [PATCH 03/11] chore: removed print statement --- src/hentai/hentai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 1020f6a..3a2120a 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -479,7 +479,6 @@ def get(self, url: str, **kwargs) -> Response: Returns the GET request encoded in `utf-8`. Adds proxies to this session on the fly if urllib is able to pick up the system's proxy settings. """ - print(self.updated_client.format(url)) response = self.session.get(self.updated_client.format(url), timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs) response.encoding = 'utf-8' return response From 665f3e732e4db366e166c989524ce9189085c792 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Thu, 15 Sep 2022 00:44:34 +0530 Subject: [PATCH 04/11] chore: cleaning --- src/hentai/hentai.py | 9 +-------- tests/test_utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 3a2120a..2449005 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -940,7 +940,6 @@ def browse_homepage(start_page: int, end_page: int, handler: RequestHandler=Requ raise ValueError(f"{COLORS['red']}{os.strerror(errno.EINVAL)}: start_page={start_page} <= {end_page}=end_page is False{COLORS['reset']}") data = set() for page in tqdm(**_progressbar_options(range(start_page, end_page+1), 'Browse', 'page', disable=progressbar)): - # with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'), params={'page': page}) as response: with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'+f'?page={page}') ) as response: for raw_json in response.json()['result']: data.add(Hentai(json=raw_json)) @@ -981,21 +980,17 @@ def search_by_query(query: str, page: int=1, sort: Sort=Sort.Popular, handler: R Return a list of `Hentai` objects on page `page` that match this search `query` sorted by `sort`. """ - # payload = {'query': query, 'page': page, 'sort': sort.value} payload = f"?query={query}&page={page}&sort={sort.value}" - # with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'), params=payload) as response: with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'+payload)) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} - + @staticmethod def search_by_tag(id_: int, page: int=1, sort: Sort=Sort.Popular, handler: RequestHandler=RequestHandler()) -> Set[Hentai]: """ Return a list of `Hentai` objects on page `page` that match this tag `id_` sorted by `sort`. """ - # payload = {'tag_id': id_, 'page': page, 'sort': sort.value} payload = f"?tag_id={id_}&page={page}&sort={sort.value}" - # with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"), params=payload) as response: with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"+payload)) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @@ -1014,9 +1009,7 @@ def search_all_by_query(query: str, sort: Sort=Sort.Popular, handler: RequestHan ``` """ data = set() - # payload = {'query': query, 'page': 1, 'sort': sort.value} payload = f"?query={query}&page=1&sort={sort.value}" - # with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+payload), params=payload) as response: with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+payload)) as response: for page in tqdm(**_progressbar_options(range(1, int(response.json()['num_pages'])+1), 'Search', 'page', disable=progressbar)): for doujin in Utils.search_by_query(query, page, sort, handler): diff --git a/tests/test_utils.py b/tests/test_utils.py index f7c030f..b8d8232 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -30,10 +30,10 @@ def test_random_hentai(self): random_hentai = Utils.get_random_hentai() self.assertEqual(type(random_hentai.json), dict, msg=f"Failing URL: {random_hentai.url} (possibility: doesn't exists(?) or blocked by Cloudflare)") - # def test_download_queue(self): - # Utils.download([self.tiny_evil], progressbar=True, zip_dir=True) - # self.assertFalse(self.tiny_evil_dir.is_dir()) - # self.assertTrue(self.tiny_evil_zip.is_file()) + def test_download_queue(self): + Utils.download([self.tiny_evil], progressbar=True, zip_dir=True) + self.assertFalse(self.tiny_evil_dir.is_dir()) + self.assertTrue(self.tiny_evil_zip.is_file()) def test_get_homepage(self): homepage = Utils.get_homepage().popular_now From 74a70af73fa24cfd68930939f780f9a619d443d7 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Thu, 15 Sep 2022 01:41:08 +0530 Subject: [PATCH 05/11] updated test case --- tests/177013.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/177013.json b/tests/177013.json index ffe8fea..3abbc22 100644 --- a/tests/177013.json +++ b/tests/177013.json @@ -1 +1 @@ -{"id": 177013, "media_id": "987560", "title": {"english": "[ShindoLA] METAMORPHOSIS (Complete) [English]", "japanese": "", "pretty": "METAMORPHOSIS"}, "images": {"pages": [{"t": "j", "w": 1275, "h": 1844}, {"t": "j", "w": 1268, "h": 1844}, {"t": "j", "w": 1274, "h": 1843}, {"t": "j", "w": 1275, "h": 1845}, {"t": "j", "w": 1280, "h": 1827}, {"t": "j", "w": 1280, "h": 1834}, {"t": "j", "w": 1280, "h": 1837}, {"t": "j", "w": 1280, "h": 1834}, {"t": "j", "w": 1280, "h": 1818}, {"t": "j", "w": 1280, "h": 1835}, {"t": "j", "w": 1280, "h": 1827}, {"t": "j", "w": 1280, "h": 1840}, {"t": "j", "w": 1280, "h": 1818}, {"t": "j", "w": 1280, "h": 911}, {"t": "j", "w": 1273, "h": 1843}, {"t": "j", "w": 1280, "h": 1835}, {"t": "j", "w": 1276, "h": 1847}, {"t": "j", "w": 1280, "h": 1822}, {"t": "j", "w": 1280, "h": 918}, {"t": "j", "w": 1271, "h": 1841}, {"t": "j", "w": 1280, "h": 1841}, {"t": "j", "w": 1275, "h": 1840}, {"t": "j", "w": 1280, "h": 1821}, {"t": "j", "w": 1280, "h": 1840}, {"t": "j", "w": 1279, "h": 1842}, {"t": "j", "w": 1280, "h": 1832}, {"t": "j", "w": 928, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 930, "h": 1331}, {"t": "j", "w": 930, "h": 1331}, {"t": "j", "w": 932, "h": 1331}, {"t": "j", "w": 928, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 930, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 928, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 935, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 934, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 950, "h": 680}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 892, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 886, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 888, "h": 1300}, {"t": "j", "w": 888, "h": 1300}, {"t": "j", "w": 887, "h": 1300}, {"t": "j", "w": 887, "h": 1300}, {"t": "j", "w": 887, "h": 1300}, {"t": "j", "w": 861, "h": 1250}, {"t": "j", "w": 860, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 860, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 859, "h": 1250}, {"t": "j", "w": 860, "h": 1250}, {"t": "j", "w": 858, "h": 1250}, {"t": "j", "w": 861, "h": 1250}, {"t": "j", "w": 857, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 857, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 869, "h": 1250}, {"t": "j", "w": 1280, "h": 916}, {"t": "j", "w": 876, "h": 1250}, {"t": "j", "w": 869, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 868, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 870, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 868, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 869, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 1280, "h": 913}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 886, "h": 1232}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 867, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 869, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 877, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 877, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 877, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 863, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 879, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 879, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 867, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 868, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 868, "h": 1278}, {"t": "j", "w": 866, "h": 1278}], "cover": {"t": "j", "w": 350, "h": 506}, "thumbnail": {"t": "j", "w": 250, "h": 362}}, "scanlator": "", "upload_date": 1476793729, "tags": [{"id": 19018, "type": "tag", "name": "dark skin", "url": "/tag/dark-skin/", "count": 21056}, {"id": 8010, "type": "tag", "name": "group", "url": "/tag/group/", "count": 72150}, {"id": 7256, "type": "tag", "name": "mmf threesome", "url": "/tag/mmf-threesome/", "count": 9641}, {"id": 8739, "type": "tag", "name": "story arc", "url": "/tag/story-arc/", "count": 9145}, {"id": 13989, "type": "tag", "name": "ahegao", "url": "/tag/ahegao/", "count": 29344}, {"id": 14283, "type": "tag", "name": "anal", "url": "/tag/anal/", "count": 64587}, {"id": 29182, "type": "tag", "name": "blackmail", "url": "/tag/blackmail/", "count": 4590}, {"id": 29859, "type": "tag", "name": "blowjob", "url": "/tag/blowjob/", "count": 37321}, {"id": 32996, "type": "tag", "name": "deepthroat", "url": "/tag/deepthroat/", "count": 3275}, {"id": 22945, "type": "tag", "name": "double penetration", "url": "/tag/double-penetration/", "count": 24472}, {"id": 22079, "type": "tag", "name": "drugs", "url": "/tag/drugs/", "count": 6723}, {"id": 21112, "type": "tag", "name": "full body tattoo", "url": "/tag/full-body-tattoo/", "count": 586}, {"id": 25050, "type": "tag", "name": "gyaru", "url": "/tag/gyaru/", "count": 3364}, {"id": 29224, "type": "tag", "name": "impregnation", "url": "/tag/impregnation/", "count": 15414}, {"id": 22942, "type": "tag", "name": "incest", "url": "/tag/incest/", "count": 28107}, {"id": 27384, "type": "tag", "name": "mind break", "url": "/tag/mind-break/", "count": 13136}, {"id": 13722, "type": "tag", "name": "moral degeneration", "url": "/tag/moral-degeneration/", "count": 573}, {"id": 13720, "type": "tag", "name": "nakadashi", "url": "/tag/nakadashi/", "count": 42357}, {"id": 5820, "type": "tag", "name": "piercing", "url": "/tag/piercing/", "count": 5567}, {"id": 6343, "type": "tag", "name": "pregnant", "url": "/tag/pregnant/", "count": 9150}, {"id": 12695, "type": "tag", "name": "prostitution", "url": "/tag/prostitution/", "count": 5613}, {"id": 10314, "type": "tag", "name": "schoolgirl uniform", "url": "/tag/schoolgirl-uniform/", "count": 56168}, {"id": 24201, "type": "tag", "name": "stockings", "url": "/tag/stockings/", "count": 64784}, {"id": 7288, "type": "tag", "name": "vomit", "url": "/tag/vomit/", "count": 878}, {"id": 20035, "type": "tag", "name": "x-ray", "url": "/tag/x-ray/", "count": 20947}, {"id": 12227, "type": "language", "name": "english", "url": "/language/english/", "count": 69378}, {"id": 17249, "type": "language", "name": "translated", "url": "/language/translated/", "count": 109734}, {"id": 3981, "type": "artist", "name": "shindol", "url": "/artist/shindol/", "count": 279}, {"id": 33173, "type": "category", "name": "manga", "url": "/category/manga/", "count": 77142}, {"id": 10542, "type": "tag", "name": "snuff", "url": "/tag/snuff/", "count": 2358}, {"id": 53449, "type": "tag", "name": "already uploaded", "url": "/tag/already-uploaded/", "count": 2121}], "num_pages": 225, "num_favorites": 44548} \ No newline at end of file +{"id": 177013, "media_id": "987560", "title": {"english": "[ShindoLA] METAMORPHOSIS (Complete) [English]", "japanese": "", "pretty": "METAMORPHOSIS"}, "images": {"pages": [{"t": "j", "w": 1275, "h": 1844}, {"t": "j", "w": 1268, "h": 1844}, {"t": "j", "w": 1274, "h": 1843}, {"t": "j", "w": 1275, "h": 1845}, {"t": "j", "w": 1280, "h": 1827}, {"t": "j", "w": 1280, "h": 1834}, {"t": "j", "w": 1280, "h": 1837}, {"t": "j", "w": 1280, "h": 1834}, {"t": "j", "w": 1280, "h": 1818}, {"t": "j", "w": 1280, "h": 1835}, {"t": "j", "w": 1280, "h": 1827}, {"t": "j", "w": 1280, "h": 1840}, {"t": "j", "w": 1280, "h": 1818}, {"t": "j", "w": 1280, "h": 911}, {"t": "j", "w": 1273, "h": 1843}, {"t": "j", "w": 1280, "h": 1835}, {"t": "j", "w": 1276, "h": 1847}, {"t": "j", "w": 1280, "h": 1822}, {"t": "j", "w": 1280, "h": 918}, {"t": "j", "w": 1271, "h": 1841}, {"t": "j", "w": 1280, "h": 1841}, {"t": "j", "w": 1275, "h": 1840}, {"t": "j", "w": 1280, "h": 1821}, {"t": "j", "w": 1280, "h": 1840}, {"t": "j", "w": 1279, "h": 1842}, {"t": "j", "w": 1280, "h": 1832}, {"t": "j", "w": 928, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 930, "h": 1331}, {"t": "j", "w": 930, "h": 1331}, {"t": "j", "w": 932, "h": 1331}, {"t": "j", "w": 928, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 930, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 928, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 933, "h": 1331}, {"t": "j", "w": 935, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 934, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 950, "h": 680}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 929, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 931, "h": 1331}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 892, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 886, "h": 1300}, {"t": "j", "w": 890, "h": 1300}, {"t": "j", "w": 889, "h": 1300}, {"t": "j", "w": 891, "h": 1300}, {"t": "j", "w": 888, "h": 1300}, {"t": "j", "w": 888, "h": 1300}, {"t": "j", "w": 887, "h": 1300}, {"t": "j", "w": 887, "h": 1300}, {"t": "j", "w": 887, "h": 1300}, {"t": "j", "w": 861, "h": 1250}, {"t": "j", "w": 860, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 860, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 859, "h": 1250}, {"t": "j", "w": 860, "h": 1250}, {"t": "j", "w": 858, "h": 1250}, {"t": "j", "w": 861, "h": 1250}, {"t": "j", "w": 857, "h": 1250}, {"t": "j", "w": 863, "h": 1250}, {"t": "j", "w": 857, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 862, "h": 1250}, {"t": "j", "w": 869, "h": 1250}, {"t": "j", "w": 1280, "h": 916}, {"t": "j", "w": 876, "h": 1250}, {"t": "j", "w": 869, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 873, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 868, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 870, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 868, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 869, "h": 1250}, {"t": "j", "w": 872, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 1280, "h": 913}, {"t": "j", "w": 871, "h": 1250}, {"t": "j", "w": 886, "h": 1232}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 886, "h": 1233}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 867, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 869, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 877, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 877, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 877, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 863, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 879, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 879, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 867, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 864, "h": 1278}, {"t": "j", "w": 866, "h": 1278}, {"t": "j", "w": 876, "h": 1278}, {"t": "j", "w": 878, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 868, "h": 1278}, {"t": "j", "w": 865, "h": 1278}, {"t": "j", "w": 868, "h": 1278}, {"t": "j", "w": 866, "h": 1278}], "cover": {"t": "j", "w": 350, "h": 506}, "thumbnail": {"t": "j", "w": 250, "h": 362}}, "scanlator": "", "upload_date": 1476793729, "tags": [{"id": 3981, "type": "artist", "name": "shindol", "url": "/artist/shindol/", "count": 291}, {"id": 5820, "type": "tag", "name": "piercing", "url": "/tag/piercing/", "count": 7683}, {"id": 6343, "type": "tag", "name": "pregnant", "url": "/tag/pregnant/", "count": 11401}, {"id": 7256, "type": "tag", "name": "mmf threesome", "url": "/tag/mmf-threesome/", "count": 12960}, {"id": 7288, "type": "tag", "name": "vomit", "url": "/tag/vomit/", "count": 1113}, {"id": 8010, "type": "tag", "name": "group", "url": "/tag/group/", "count": 87666}, {"id": 8739, "type": "tag", "name": "story arc", "url": "/tag/story-arc/", "count": 14356}, {"id": 10314, "type": "tag", "name": "schoolgirl uniform", "url": "/tag/schoolgirl-uniform/", "count": 66673}, {"id": 10542, "type": "tag", "name": "snuff", "url": "/tag/snuff/", "count": 3340}, {"id": 12227, "type": "language", "name": "english", "url": "/language/english/", "count": 92747}, {"id": 12695, "type": "tag", "name": "prostitution", "url": "/tag/prostitution/", "count": 7766}, {"id": 13720, "type": "tag", "name": "nakadashi", "url": "/tag/nakadashi/", "count": 59187}, {"id": 13722, "type": "tag", "name": "moral degeneration", "url": "/tag/moral-degeneration/", "count": 824}, {"id": 13989, "type": "tag", "name": "ahegao", "url": "/tag/ahegao/", "count": 37438}, {"id": 14283, "type": "tag", "name": "anal", "url": "/tag/anal/", "count": 81867}, {"id": 17249, "type": "language", "name": "translated", "url": "/language/translated/", "count": 156871}, {"id": 19018, "type": "tag", "name": "dark skin", "url": "/tag/dark-skin/", "count": 28613}, {"id": 20035, "type": "tag", "name": "x-ray", "url": "/tag/x-ray/", "count": 27244}, {"id": 21112, "type": "tag", "name": "full body tattoo", "url": "/tag/full-body-tattoo/", "count": 561}, {"id": 22079, "type": "tag", "name": "drugs", "url": "/tag/drugs/", "count": 8631}, {"id": 22942, "type": "tag", "name": "incest", "url": "/tag/incest/", "count": 35377}, {"id": 22945, "type": "tag", "name": "double penetration", "url": "/tag/double-penetration/", "count": 29438}, {"id": 24201, "type": "tag", "name": "stockings", "url": "/tag/stockings/", "count": 80225}, {"id": 25050, "type": "tag", "name": "gyaru", "url": "/tag/gyaru/", "count": 5172}, {"id": 27384, "type": "tag", "name": "mind break", "url": "/tag/mind-break/", "count": 15997}, {"id": 29182, "type": "tag", "name": "blackmail", "url": "/tag/blackmail/", "count": 6668}, {"id": 29224, "type": "tag", "name": "impregnation", "url": "/tag/impregnation/", "count": 20523}, {"id": 29859, "type": "tag", "name": "blowjob", "url": "/tag/blowjob/", "count": 53572}, {"id": 32996, "type": "tag", "name": "deepthroat", "url": "/tag/deepthroat/", "count": 4374}, {"id": 33173, "type": "category", "name": "manga", "url": "/category/manga/", "count": 98402}, {"id": 53449, "type": "tag", "name": "already uploaded", "url": "/tag/already-uploaded/", "count": 2369}], "num_pages": 225, "num_favorites": 97746} \ No newline at end of file From 8da0aa51a8adf8e196d9c14bf1e4a404c1a65037 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Thu, 15 Sep 2022 01:50:41 +0530 Subject: [PATCH 06/11] chore: urlencode for params --- src/hentai/hentai.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 2449005..e1866e3 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -41,7 +41,7 @@ from importlib.resources import path as resource_path from pathlib import Path from typing import Iterable, List, Optional, Set, Tuple, Union -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse, urlencode from urllib.request import getproxies from zipfile import ZIP_DEFLATED, ZipFile @@ -940,7 +940,7 @@ def browse_homepage(start_page: int, end_page: int, handler: RequestHandler=Requ raise ValueError(f"{COLORS['red']}{os.strerror(errno.EINVAL)}: start_page={start_page} <= {end_page}=end_page is False{COLORS['reset']}") data = set() for page in tqdm(**_progressbar_options(range(start_page, end_page+1), 'Browse', 'page', disable=progressbar)): - with handler.get(urljoin(Hentai.HOME, 'api/galleries/all'+f'?page={page}') ) as response: + with handler.get(urljoin(Hentai.HOME, 'api/galleries/all' + "?" + urlencode({'page': page})) ) as response: for raw_json in response.json()['result']: data.add(Hentai(json=raw_json)) return data @@ -980,8 +980,7 @@ def search_by_query(query: str, page: int=1, sort: Sort=Sort.Popular, handler: R Return a list of `Hentai` objects on page `page` that match this search `query` sorted by `sort`. """ - payload = f"?query={query}&page={page}&sort={sort.value}" - with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'+payload)) as response: + with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'+ "?" + urlencode({'query': query, 'page': page, 'sort': sort.value}))) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @staticmethod @@ -990,8 +989,7 @@ def search_by_tag(id_: int, page: int=1, sort: Sort=Sort.Popular, handler: Reque Return a list of `Hentai` objects on page `page` that match this tag `id_` sorted by `sort`. """ - payload = f"?tag_id={id_}&page={page}&sort={sort.value}" - with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged"+payload)) as response: + with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged" + "?" + urlencode({'tag_id': id_, 'page': page, 'sort': sort.value}))) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @staticmethod @@ -1009,8 +1007,7 @@ def search_all_by_query(query: str, sort: Sort=Sort.Popular, handler: RequestHan ``` """ data = set() - payload = f"?query={query}&page=1&sort={sort.value}" - with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+payload)) as response: + with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+ "?" + urlencode({'query': query, 'page': 1, 'sort': sort.value}))) as response: for page in tqdm(**_progressbar_options(range(1, int(response.json()['num_pages'])+1), 'Search', 'page', disable=progressbar)): for doujin in Utils.search_by_query(query, page, sort, handler): data.add(doujin) From df2d5289c9baa4d2979ad2f7f55e01d55a164e26 Mon Sep 17 00:00:00 2001 From: Reset <77439837+ResetXD@users.noreply.github.com> Date: Thu, 22 Sep 2022 23:41:11 +0530 Subject: [PATCH 07/11] docstring return type doesn't match the method's --- src/hentai/hentai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index e1866e3..1f88c3a 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -932,7 +932,7 @@ def download(doujins: List[Hentai], delay: float=0, progressbar: bool=False, zip @staticmethod def browse_homepage(start_page: int, end_page: int, handler: RequestHandler=RequestHandler(), progressbar: bool=False) -> Set[Hentai]: """ - Return a list of `Hentai` objects that are currently featured on the homepage + Return a set of `Hentai` objects that are currently featured on the homepage in range of `[start_page, end_page]`. Each page contains as much as 25 results. Enable `progressbar` for status feedback in terminal applications. """ From f90774381089a649c60b22e6be2f7999fed7493c Mon Sep 17 00:00:00 2001 From: ResetXD Date: Thu, 22 Sep 2022 23:55:57 +0530 Subject: [PATCH 08/11] feat: unneccessary/extra --- src/hentai/hentai.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 1f88c3a..34d9aac 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -197,6 +197,12 @@ class Tag: url: str count: int + def __eq__(self, other: Union[Tag, str]) -> bool: + if isinstance(other, Tag): + return self.id == other.id + elif isinstance(other, str): + return self.name == other + @classmethod def get(cls, tags: List[Tag], property_: str) -> str: """ From 1cf8655ec46e77d09e1d6750271601845aab7805 Mon Sep 17 00:00:00 2001 From: Reset Date: Mon, 26 Sep 2022 16:55:13 +0530 Subject: [PATCH 09/11] chore: changes --- src/hentai/hentai.py | 36 +++++++++++++++++++++++++++--------- tests/test_utils.py | 4 ---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 34d9aac..5983d11 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -198,6 +198,24 @@ class Tag: count: int def __eq__(self, other: Union[Tag, str]) -> bool: + """ + Compare for equality by `Tag` or a tag's name. + + Example + ------- + + ``` + from hentai import Hentai, Tag + + doujin = Hentai(177013) + + # english, translated + print(Tag.get(doujin.language, 'name')) + + # True + print(Tag.get(doujin.language, 'name') == 'english, translated') + ``` + """ if isinstance(other, Tag): return self.id == other.id elif isinstance(other, str): @@ -425,7 +443,7 @@ class RequestHandler(object): print(response.ok) ``` """ - __slots__ = ['timeout', 'total', 'status_forcelist', 'backoff_factor', 'user_agent', 'proxies', "updated_client"] + __slots__ = ['timeout', 'total', 'status_forcelist', 'backoff_factor', 'user_agent', 'proxies', "enable_alt_client", "alt_url"] _timeout = (5, 5) _total = 5 @@ -433,7 +451,7 @@ class RequestHandler(object): _backoff_factor = 1 _user_agent = None _proxies = None - _updated_client = "https://translate.google.com/translate?sl=vi&tl=en&hl=vi&u={0}&client=webapp" + _enable_alt_client = True def __init__(self, timeout: Tuple[float, float]=_timeout, @@ -442,7 +460,7 @@ def __init__(self, backoff_factor: int=_backoff_factor, user_agent: str=_user_agent, proxies: dict=_proxies, - updated_client: str=_updated_client) -> None: + enable_alt_client: bool=_enable_alt_client) -> None: """ Instantiates a new request handler object. """ @@ -452,7 +470,7 @@ def __init__(self, self.backoff_factor = backoff_factor self.user_agent = user_agent self.proxies = proxies - self.updated_client = updated_client + self.alt_url = "https://translate.google.com/translate?sl=vi&tl=en&hl=vi&u={0}&client=webapp" if enable_alt_client else "{0}" @property def retry_strategy(self) -> Retry: @@ -485,7 +503,7 @@ def get(self, url: str, **kwargs) -> Response: Returns the GET request encoded in `utf-8`. Adds proxies to this session on the fly if urllib is able to pick up the system's proxy settings. """ - response = self.session.get(self.updated_client.format(url), timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs) + response = self.session.get(self.alt_url.format(url), timeout=self.timeout, proxies=self.proxies or getproxies(), **kwargs) response.encoding = 'utf-8' return response @@ -946,7 +964,7 @@ def browse_homepage(start_page: int, end_page: int, handler: RequestHandler=Requ raise ValueError(f"{COLORS['red']}{os.strerror(errno.EINVAL)}: start_page={start_page} <= {end_page}=end_page is False{COLORS['reset']}") data = set() for page in tqdm(**_progressbar_options(range(start_page, end_page+1), 'Browse', 'page', disable=progressbar)): - with handler.get(urljoin(Hentai.HOME, 'api/galleries/all' + "?" + urlencode({'page': page})) ) as response: + with handler.get(urljoin(Hentai.HOME, f"api/galleries/all?{urlencode({'page': page})}") ) as response: for raw_json in response.json()['result']: data.add(Hentai(json=raw_json)) return data @@ -986,7 +1004,7 @@ def search_by_query(query: str, page: int=1, sort: Sort=Sort.Popular, handler: R Return a list of `Hentai` objects on page `page` that match this search `query` sorted by `sort`. """ - with handler.get(urljoin(Hentai.HOME, 'api/galleries/search'+ "?" + urlencode({'query': query, 'page': page, 'sort': sort.value}))) as response: + with handler.get(urljoin(Hentai.HOME, f"api/galleries/search?{urlencode({'query': query, 'page': page, 'sort': sort.value})}")) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @staticmethod @@ -995,7 +1013,7 @@ def search_by_tag(id_: int, page: int=1, sort: Sort=Sort.Popular, handler: Reque Return a list of `Hentai` objects on page `page` that match this tag `id_` sorted by `sort`. """ - with handler.get(urljoin(Hentai.HOME, "api/galleries/tagged" + "?" + urlencode({'tag_id': id_, 'page': page, 'sort': sort.value}))) as response: + with handler.get(urljoin(Hentai.HOME, f"api/galleries/tagged?{urlencode({'tag_id': id_, 'page': page, 'sort': sort.value})}")) as response: return {Hentai(json=raw_json) for raw_json in response.json()['result']} @staticmethod @@ -1013,7 +1031,7 @@ def search_all_by_query(query: str, sort: Sort=Sort.Popular, handler: RequestHan ``` """ data = set() - with handler.get(urljoin(Hentai.HOME, '/api/galleries/search'+ "?" + urlencode({'query': query, 'page': 1, 'sort': sort.value}))) as response: + with handler.get(urljoin(Hentai.HOME, f"/api/galleries/search?{urlencode({'query': query, 'page': 1, 'sort': sort.value})}")) as response: for page in tqdm(**_progressbar_options(range(1, int(response.json()['num_pages'])+1), 'Search', 'page', disable=progressbar)): for doujin in Utils.search_by_query(query, page, sort, handler): data.add(doujin) diff --git a/tests/test_utils.py b/tests/test_utils.py index b8d8232..eca8afd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,10 +5,6 @@ import unittest import platform from pathlib import Path -from urllib.parse import urljoin - -import requests - from src.hentai import Format, Hentai, Option, Sort, Utils remove_file = lambda file: file.unlink() if file.exists() else None From 1d91fe6776e72a62d4a9c46eac43eb3260d56f8d Mon Sep 17 00:00:00 2001 From: Reset Date: Mon, 26 Sep 2022 16:57:59 +0530 Subject: [PATCH 10/11] error message --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index eca8afd..627b053 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -24,7 +24,7 @@ def tearDownClass(cls): def test_random_hentai(self): random_hentai = Utils.get_random_hentai() - self.assertEqual(type(random_hentai.json), dict, msg=f"Failing URL: {random_hentai.url} (possibility: doesn't exists(?) or blocked by Cloudflare)") + self.assertEqual(type(random_hentai.json), dict, msg=f"Failing URL: {random_hentai.url} (resource might not exist or is blocked by Cloudflare protection)") def test_download_queue(self): Utils.download([self.tiny_evil], progressbar=True, zip_dir=True) From faed62bbb54377bcac6ff52bf2e768d928ef88d1 Mon Sep 17 00:00:00 2001 From: ResetXD Date: Sun, 2 Oct 2022 23:38:56 +0530 Subject: [PATCH 11/11] changes that i dont understand --- src/hentai/hentai.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hentai/hentai.py b/src/hentai/hentai.py index 5983d11..24dc646 100644 --- a/src/hentai/hentai.py +++ b/src/hentai/hentai.py @@ -451,7 +451,7 @@ class RequestHandler(object): _backoff_factor = 1 _user_agent = None _proxies = None - _enable_alt_client = True + _enable_alt_client = False def __init__(self, timeout: Tuple[float, float]=_timeout, @@ -536,6 +536,7 @@ class Hentai(RequestHandler): HOME = "https://nhentai.net/" _URL = urljoin(HOME, '/g/') _API = urljoin(HOME, '/api/gallery/') + _enable_alt_client = True def __init__(self, id_: int=0, @@ -545,14 +546,15 @@ def __init__(self, backoff_factor: int=RequestHandler._backoff_factor, user_agent: str=RequestHandler._user_agent, proxies: dict=RequestHandler._proxies, + enable_alt_client: bool=_enable_alt_client, json: Optional[dict]=None): """ Start a request session and parse meta data from for this `id`. """ if id_ and not json: self.__id = id_ - super().__init__(timeout, total, status_forcelist, backoff_factor, user_agent, proxies) - self.__handler = RequestHandler(self.timeout, self.total, self.status_forcelist, self.backoff_factor, self.user_agent, self.proxies) + super().__init__(timeout, total, status_forcelist, backoff_factor, user_agent, proxies, enable_alt_client) + self.__handler = RequestHandler(self.timeout, self.total, self.status_forcelist, self.backoff_factor, self.user_agent, self.proxies, enable_alt_client) self.__url = urljoin(Hentai._URL, str(self.id)) self.__api = urljoin(Hentai._API, str(self.id)) self.__response = self.handler.get(self.api)