bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

The crawler result can be Dict or Str loaded by JSON

Browse Source
This commit is contained in:
yoshiko2
2022-04-19 00:00:28 +08:00
parent 022c0d30eb
commit 9be375de9e
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
WebCrawler/__init__.py
View File

@@ -134,7 +134,10 @@ def get_data_from_json(file_number, oCC):
for source in sources: for source in sources:
if conf.debug() == True: if conf.debug() == True:
print('[+]select', source) print('[+]select', source)
try:
json_data = json.loads(pool.apply_async(func_mapping[source], (file_number,)).get()) json_data = json.loads(pool.apply_async(func_mapping[source], (file_number,)).get())
except:
json_data = pool.apply_async(func_mapping[source], (file_number,)).get()
# if any service return a valid return, break # if any service return a valid return, break
if get_data_state(json_data): if get_data_state(json_data):
print(f"[+]Find movie [{file_number}] metadata on website '{source}'") print(f"[+]Find movie [{file_number}] metadata on website '{source}'")
@@ -146,7 +149,10 @@ def get_data_from_json(file_number, oCC):
try: try:
if conf.debug() == True: if conf.debug() == True:
print('[+]select', source) print('[+]select', source)
try:
json_data = json.loads(func_mapping[source](file_number)) json_data = json.loads(func_mapping[source](file_number))
except:
json_data = func_mapping[source](file_number)
# if any service return a valid return, break # if any service return a valid return, break
if get_data_state(json_data): if get_data_state(json_data):
print(f"[+]Find movie [{file_number}] metadata on website '{source}'") print(f"[+]Find movie [{file_number}] metadata on website '{source}'")