-
Notifications
You must be signed in to change notification settings - Fork 1
/
workers.py
executable file
·58 lines (46 loc) · 1.8 KB
/
workers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#!/usr/bin/env python3
from chisel.database import ChiselDB
import pandas as pd
import requests
from requests.exceptions import ConnectionError, ReadTimeout
from multiprocessing import Pool
from time import sleep
from sys import argv
def check_proxy_uri(proxy):
while True:
try:
assert requests.head('https://connectivitycheck.gstatic.com/generate_204').status_code == 204
break
except (AssertionError, ConnectionError):
sleep(3)
try:
return proxy, all(requests.head(
url=protocol + '://connectivitycheck.gstatic.com/generate_204',
proxies={protocol: proxy},
timeout=5,
).status_code == 204 for protocol in ('http', 'https'))
except (ConnectionError, ReadTimeout, ValueError):
return proxy, False
def main():
cdb = ChiselDB(False)
pool = Pool(int(argv[1]))
while True:
pass
df = pd.read_html(requests.get('https://www.socks-proxy.net/').text)[0][:-1]
df['Port'] = df['Port'].astype(int).astype(str)
df['Version'] = df['Version'].str.lower()
cdb.store_proxy_series(df['Version'] + '://' + df['IP Address'] + ':' + df['Port'])
del df
df = pd.read_html(requests.get('https://free-proxy-list.net/').text)[0][:-1]
df['Port'] = df['Port'].astype(int).astype(str)
df['Https'] = df['Https'].map({'yes': 'https', 'no': 'http'})
cdb.store_proxy_series(df['Https'] + '://' + df['IP Address'] + ':' + df['Port'])
del df
for proxy, works in pool.imap_unordered(
func=check_proxy_uri,
iterable=[doc['proxy'] for doc in cdb.get_proxies_by_insertion()],
):
print(proxy, works)
cdb.update_proxy_status(proxy, works)
if __name__ == '__main__':
main()