bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将元数据的来源网站记入日志以便进行评估

Browse Source
This commit is contained in:
lededev
2021-10-18 10:51:32 +08:00
parent c9b96f65ab
commit 24b4f9f5e2
4 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ADC_function.py
View File

@@ -1,6 +1,6 @@
from os import replace from os import replace
import requests import requests
import hashlib #import hashlib
from pathlib import Path from pathlib import Path
import secrets import secrets
import os.path import os.path
@@ -20,7 +20,7 @@ def getXpathSingle(htmlcode, xpath):
return result1 return result1
G_USER_AGENT = r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36' G_USER_AGENT = r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36'
# 网页请求核心 # 网页请求核心
def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None): def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None):

2
WebCrawler/__init__.py
View File

@@ -115,6 +115,7 @@ def get_data_from_json(file_number): # 从JSON返回元数据
json_data = json.loads(pool.apply_async(func_mapping[source], (file_number,)).get()) json_data = json.loads(pool.apply_async(func_mapping[source], (file_number,)).get())
# if any service return a valid return, break # if any service return a valid return, break
if get_data_state(json_data): if get_data_state(json_data):
print(f"[+]Find movie [{file_number}] metadata on website '{source}'")
break break
pool.close() pool.close()
pool.terminate() pool.terminate()
@@ -126,6 +127,7 @@ def get_data_from_json(file_number): # 从JSON返回元数据
json_data = json.loads(func_mapping[source](file_number)) json_data = json.loads(func_mapping[source](file_number))
# if any service return a valid return, break # if any service return a valid return, break
if get_data_state(json_data): if get_data_state(json_data):
print(f"[+]Find movie [{file_number}] metadata on website '{source}'")
break break
except: except:
break break

2
WebCrawler/storyline.py
View File

@@ -34,7 +34,7 @@ def getStoryline(number, title):
for i in range(cnt): for i in range(cnt):
sl = len(result[i])if isinstance(result[i], str) else 0 sl = len(result[i])if isinstance(result[i], str) else 0
if sl and first: if sl and first:
s += f'，[选中结果{apply_sites[i]}字数:{sl}]' s += f'，[选中{apply_sites[i]}字数:{sl}]'
first = False first = False
sel = result[i] sel = result[i]
elif sl: elif sl:

4
config.py
View File

@@ -394,10 +394,10 @@ if __name__ == "__main__":
code = compile(evstr, "<string>", "eval") code = compile(evstr, "<string>", "eval")
print('{}: "{}"'.format(evstr, eval(code))) print('{}: "{}"'.format(evstr, eval(code)))
config = Config() config = Config()
mfilter = ('conf', 'proxy', '_exit', '_default_config', 'getboolean_override', 'getint_override', 'get_override', 'ini_path') mfilter = {'conf', 'proxy', '_exit', '_default_config', 'getboolean_override', 'getint_override', 'get_override', 'ini_path'}
for _m in [m for m in dir(config) if not m.startswith('__') and m not in mfilter]: for _m in [m for m in dir(config) if not m.startswith('__') and m not in mfilter]:
evprint(f'config.{_m}()') evprint(f'config.{_m}()')
pfilter = ('proxies', 'SUPPORT_PROXY_TYPE') pfilter = {'proxies', 'SUPPORT_PROXY_TYPE'}
# test getInstance() # test getInstance()
assert(getInstance() == config) assert(getInstance() == config)
for _p in [p for p in dir(getInstance().proxy()) if not p.startswith('__') and p not in pfilter]: for _p in [p for p in dir(getInstance().proxy()) if not p.startswith('__') and p not in pfilter]: