bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #507 from Suwmlee/master

Browse Source 
Refine Proxy
This commit is contained in:
Yoshiko2
2021-06-04 14:10:04 +08:00
committed by GitHub
parent f97c6f8748 7fe5b69ad2
commit e913925279
3 changed files with 72 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
ADC_function.py
View File

@@ -9,8 +9,6 @@ from lxml import etree
import re
import config
SUPPORT_PROXY_TYPE = ("http", "socks5", "socks5h")
def get_data_state(data: dict) -> bool: # 元数据获取失败检测
if "title" not in data or "number" not in data:
@@ -31,25 +29,10 @@ def getXpathSingle(htmlcode, xpath):
return result1
def get_proxy(proxy: str, proxytype: str = None) -> dict:
''' 获得代理参数，默认http代理
'''
if proxy:
if proxytype in SUPPORT_PROXY_TYPE:
proxies = {"http": proxytype + "://" + proxy, "https": proxytype + "://" + proxy}
else:
proxies = {"http": "http://" + proxy, "https": "https://" + proxy}
else:
proxies = {}
return proxies
# 网页请求核心
def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None):
verify = config.Config().cacert_file()
switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
proxies = get_proxy(proxy, proxytype)
configProxy = config.Config().proxy()
errors = ""
if ua is None:
@@ -58,13 +41,14 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None)
else:
headers = {"User-Agent": ua}
for i in range(retry_count):
for i in range(configProxy.retry):
try:
if switch == '1' or switch == 1:
result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, verify=verify,
if configProxy.enable:
proxies = configProxy.proxies()
result = requests.get(str(url), headers=headers, timeout=configProxy.timeout, proxies=proxies, verify=verify,
cookies=cookies)
else:
result = requests.get(str(url), headers=headers, timeout=timeout, cookies=cookies)
result = requests.get(str(url), headers=headers, timeout=configProxy.timeout, cookies=cookies)
result.encoding = "utf-8"
@@ -78,15 +62,14 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None)
print("[-]Proxy error! Please check your Proxy")
return
except Exception as e:
print("[-]Connect retry {}/{}".format(i + 1, retry_count))
print("[-]Connect retry {}/{}".format(i + 1, configProxy.retry))
errors = str(e)
print('[-]Connect Failed! Please check your Proxy or Network!')
print("[-]" + errors)
def post_html(url: str, query: dict, headers: dict = None) -> requests.Response:
switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
proxies = get_proxy(proxy, proxytype)
configProxy = config.Config().proxy()
errors = ""
headers_ua = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"}
@@ -95,15 +78,16 @@ def post_html(url: str, query: dict, headers: dict = None) -> requests.Response:
else:
headers.update(headers_ua)
for i in range(retry_count):
for i in range(configProxy.retry):
try:
if switch == 1 or switch == '1':
result = requests.post(url, data=query, proxies=proxies, headers=headers, timeout=timeout)
if configProxy.enable:
proxies = configProxy.proxies()
result = requests.post(url, data=query, proxies=proxies, headers=headers, timeout=configProxy.timeout)
else:
result = requests.post(url, data=query, headers=headers, timeout=timeout)
result = requests.post(url, data=query, headers=headers, timeout=configProxy.timeout)
return result
except Exception as e:
print("[-]Connect retry {}/{}".format(i + 1, retry_count))
print("[-]Connect retry {}/{}".format(i + 1, configProxy.retry))
errors = str(e)
print("[-]Connect Failed! Please check your Proxy or Network!")
print("[-]" + errors)

45
config.py
View File

@@ -92,7 +92,7 @@ class Config:
return self.conf.getint("transalte","delay")
def transalte_values(self) -> str:
return self.conf.get("transalte", "values")
def proxy(self) -> [str, int, int, str]:
def proxy(self):
try:
sec = "proxy"
switch = self.conf.get(sec, "switch")
@@ -100,7 +100,8 @@ class Config:
timeout = self.conf.getint(sec, "timeout")
retry = self.conf.getint(sec, "retry")
proxytype = self.conf.get(sec, "type")
return switch, proxy, timeout, retry, proxytype
iniProxy = IniProxy(switch, proxy, timeout, retry, proxytype)
return iniProxy
except ValueError:
self._exit("common")
@@ -235,6 +236,43 @@ class Config:
return conf
class IniProxy():
""" Proxy Config from .ini
"""
SUPPORT_PROXY_TYPE = ("http", "socks5", "socks5h")
enable = False
address = ""
timeout = 5
retry = 3
proxytype = "socks5"
def __init__(self, switch, address, timeout, retry, proxytype) -> None:
""" Initial Proxy from .ini
"""
if switch == '1' or switch == 1:
self.enable = True
self.address = address
self.timeout = timeout
self.retry = retry
self.proxytype = proxytype
def proxies(self):
''' 获得代理参数，默认http代理
'''
if self.address:
if self.proxytype in self.SUPPORT_PROXY_TYPE:
proxies = {"http": self.proxytype + "://" + self.address, "https": self.proxytype + "://" + self.address}
else:
proxies = {"http": "http://" + self.address, "https": "https://" + self.address}
else:
proxies = {}
return proxies
if __name__ == "__main__":
config = Config()
print(config.main_mode())
@@ -243,7 +281,8 @@ if __name__ == "__main__":
print(config.soft_link())
print(config.failed_move())
print(config.auto_exit())
print(config.proxy())
print(config.proxy().enable)
print(config.proxy().retry)
print(config.naming_rule())
print(config.location_rule())
print(config.update_check())

32
core.py
View File

@@ -351,17 +351,17 @@ def trimblank(s: str):
# path = examle:photo , video.in the Project Folder!
def download_file_with_filename(url, filename, path, conf: config.Config, filepath, failed_folder):
switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
configProxy = conf.proxy()
for i in range(retry_count):
for i in range(configProxy.retry):
try:
if switch == 1 or switch == '1':
if configProxy.enable:
if not os.path.exists(path):
os.makedirs(path)
proxies = get_proxy(proxy, proxytype)
proxies = configProxy.proxies()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
r = requests.get(url, headers=headers, timeout=timeout, proxies=proxies)
r = requests.get(url, headers=headers, timeout=configProxy.timeout, proxies=proxies)
if r == '':
print('[-]Movie Data not found!')
return
@@ -373,7 +373,7 @@ def download_file_with_filename(url, filename, path, conf: config.Config, filepa
os.makedirs(path)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
r = requests.get(url, timeout=timeout, headers=headers)
r = requests.get(url, timeout=configProxy.timeout, headers=headers)
if r == '':
print('[-]Movie Data not found!')
return
@@ -382,16 +382,16 @@ def download_file_with_filename(url, filename, path, conf: config.Config, filepa
return
except requests.exceptions.RequestException:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(retry_count))
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ConnectionError:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(retry_count))
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ProxyError:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(retry_count))
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ConnectTimeout:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(retry_count))
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
print('[-]Connect Failed! Please check your Proxy or Network!')
moveFailedFolder(filepath, failed_folder)
return
@@ -399,8 +399,8 @@ def download_file_with_filename(url, filename, path, conf: config.Config, filepa
def trailer_download(trailer, leak_word, c_word, number, path, filepath, conf: config.Config, failed_folder):
if download_file_with_filename(trailer, number + leak_word + c_word + '-trailer.mp4', path, conf, filepath, failed_folder) == 'failed':
return
switch, _proxy, _timeout, retry, _proxytype = conf.proxy()
for i in range(retry):
configProxy = conf.proxy()
for i in range(configProxy.retry):
if os.path.getsize(path+'/' + number + leak_word + c_word + '-trailer.mp4') == 0:
print('[!]Video Download Failed! Trying again. [{}/3]', i + 1)
download_file_with_filename(trailer, number + leak_word + c_word + '-trailer.mp4', path, conf, filepath, failed_folder)
@@ -419,8 +419,8 @@ def extrafanart_download(data, path, conf: config.Config, filepath, failed_folde
if download_file_with_filename(url, '/extrafanart-' + str(j)+'.jpg', path, conf, filepath, failed_folder) == 'failed':
moveFailedFolder(filepath, failed_folder)
return
switch, _proxy, _timeout, retry, _proxytype = conf.proxy()
for i in range(retry):
configProxy = conf.proxy()
for i in range(configProxy.retry):
if os.path.getsize(path + '/extrafanart-' + str(j) + '.jpg') == 0:
print('[!]Image Download Failed! Trying again. [{}/3]', i + 1)
download_file_with_filename(url, '/extrafanart-' + str(j)+'.jpg', path, conf, filepath,
@@ -441,8 +441,8 @@ def image_download(cover, number, leak_word, c_word, path, conf: config.Config,
moveFailedFolder(filepath, failed_folder)
return
switch, _proxy, _timeout, retry, _proxytype = conf.proxy()
for i in range(retry):
configProxy = conf.proxy()
for i in range(configProxy.retry):
if os.path.getsize(path + '/' + number + leak_word + c_word + '-fanart.jpg') == 0:
print('[!]Image Download Failed! Trying again. [{}/3]', i + 1)
download_file_with_filename(cover, number + leak_word + c_word + '-fanart.jpg', path, conf, filepath, failed_folder)