Fix get_html() & post_html()

This commit is contained in:
yoshiko2
2021-04-29 09:59:26 +08:00
parent 6e8ebb97bb
commit 043789dd45

View File

@@ -10,6 +10,7 @@ import config
SUPPORT_PROXY_TYPE = ("http", "socks5", "socks5h") SUPPORT_PROXY_TYPE = ("http", "socks5", "socks5h")
def get_data_state(data: dict) -> bool: # 元数据获取失败检测 def get_data_state(data: dict) -> bool: # 元数据获取失败检测
if "title" not in data or "number" not in data: if "title" not in data or "number" not in data:
return False return False
@@ -48,16 +49,19 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None)
verify = config.Config().cacert_file() verify = config.Config().cacert_file()
switch, proxy, timeout, retry_count, proxytype = config.Config().proxy() switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
proxies = get_proxy(proxy, proxytype) proxies = get_proxy(proxy, proxytype)
errors = ""
if ua is None: if ua is None:
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"} # noqa headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"} # noqa
else: else:
headers = {"User-Agent": ua} headers = {"User-Agent": ua}
for i in range(retry_count): for i in range(retry_count):
try: try:
if switch == '1' or switch == 1: if switch == '1' or switch == 1:
result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, verify=verify, cookies=cookies) result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, verify=verify,
cookies=cookies)
else: else:
result = requests.get(str(url), headers=headers, timeout=timeout, cookies=cookies) result = requests.get(str(url), headers=headers, timeout=timeout, cookies=cookies)
@@ -74,13 +78,15 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None)
return return
except Exception as e: except Exception as e:
print("[-]Connect retry {}/{}".format(i + 1, retry_count)) print("[-]Connect retry {}/{}".format(i + 1, retry_count))
print("[-]" + str(e)) errors = str(e)
print('[-]Connect Failed! Please check your Proxy or Network!') print('[-]Connect Failed! Please check your Proxy or Network!')
print("[-]" + errors)
def post_html(url: str, query: dict, headers: dict = None) -> requests.Response: def post_html(url: str, query: dict, headers: dict = None) -> requests.Response:
switch, proxy, timeout, retry_count, proxytype = config.Config().proxy() switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
proxies = get_proxy(proxy, proxytype) proxies = get_proxy(proxy, proxytype)
errors = ""
headers_ua = { headers_ua = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"} "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"}
if headers is None: if headers is None:
@@ -95,9 +101,11 @@ def post_html(url: str, query: dict, headers: dict = None) -> requests.Response:
else: else:
result = requests.post(url, data=query, headers=headers, timeout=timeout) result = requests.post(url, data=query, headers=headers, timeout=timeout)
return result return result
except requests.exceptions.ProxyError: except Exception as e:
print("[-]Connect retry {}/{}".format(i + 1, retry_count)) print("[-]Connect retry {}/{}".format(i + 1, retry_count))
errors = str(e)
print("[-]Connect Failed! Please check your Proxy or Network!") print("[-]Connect Failed! Please check your Proxy or Network!")
print("[-]" + errors)
def get_javlib_cookie() -> [dict, str]: def get_javlib_cookie() -> [dict, str]:
@@ -127,6 +135,7 @@ def get_javlib_cookie() -> [dict, str]:
return raw_cookie, user_agent return raw_cookie, user_agent
def translateTag_to_sc(tag): def translateTag_to_sc(tag):
tranlate_to_sc = config.Config().transalte_to_sc() tranlate_to_sc = config.Config().transalte_to_sc()
if tranlate_to_sc: if tranlate_to_sc:
@@ -445,7 +454,6 @@ def translateTag_to_sc(tag):
'处男', 'ドキュメンタリー': '记录片', 'ドラッグ・媚薬': '药局', 'ドラマ': '电视剧', 'ニューハーフ': '变性人', '处男', 'ドキュメンタリー': '记录片', 'ドラッグ・媚薬': '药局', 'ドラマ': '电视剧', 'ニューハーフ': '变性人',
'ニーソックス': '过膝袜', '妊婦': '孕妇', '寝取り・寝取られ': '睡下', 'HowTo': 'HowTo', 'ニーソックス': '过膝袜', '妊婦': '孕妇', '寝取り・寝取られ': '睡下', 'HowTo': 'HowTo',
# fc2 # fc2
'美人': '美女', 'ハメ撮り': '拍鸽子', 'フェチ': '恋物癖', '美人': '美女', 'ハメ撮り': '拍鸽子', 'フェチ': '恋物癖',
'コスプレ・制服': 'COSPLAY制服', '自分撮り': '自拍', 'その他': '其他', 'OL・お姉さん': 'OL姐姐', 'ゲイ': '同性恋', 'コスプレ・制服': 'COSPLAY制服', '自分撮り': '自拍', 'その他': '其他', 'OL・お姉さん': 'OL姐姐', 'ゲイ': '同性恋',
@@ -461,6 +469,7 @@ def translateTag_to_sc(tag):
else: else:
return tag return tag
def translate( def translate(
src: str, src: str,
target_language: str = "zh_cn", target_language: str = "zh_cn",
@@ -521,6 +530,7 @@ def translate(
time.sleep(delay) time.sleep(delay)
return trans_result return trans_result
# ========================================================================是否为无码 # ========================================================================是否为无码
def is_uncensored(number): def is_uncensored(number):
if re.match('^\d{4,}', number) or re.match('n\d{4}', number) or 'HEYZO' in number.upper(): if re.match('^\d{4,}', number) or re.match('n\d{4}', number) or 'HEYZO' in number.upper():