bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix crash caused by custom site

Browse Source
This commit is contained in:
yoshiko2
2021-07-10 15:01:56 +08:00
parent 80b973ef59
commit 5a80f13bec
1 changed files with 25 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
core.py
View File

@@ -71,35 +71,35 @@ def get_data_from_json(file_number, filepath, conf: config.Config): # 从JSON
# default fetch order list, from the beginning to the end
sources = conf.sources().split(',')
# if the input file name matches certain rules,
# move some web service to the beginning of the list
lo_file_number = file_number.lower()
if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number)
):
sources.insert(0, sources.pop(sources.index("carib")))
elif "avsox" in sources and (re.match(r"^\d{5,}", file_number) or
"heyzo" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("javdb")))
sources.insert(1, sources.pop(sources.index("avsox")))
elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or
"siro" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("mgstage")))
elif "fc2" in sources and ("fc2" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("javdb")))
sources.insert(1, sources.pop(sources.index("fc2")))
elif "dlsite" in sources and (
"rj" in lo_file_number or "vj" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("dlsite")))
if not len(conf.sources()) > 60:
# if the input file name matches certain rules,
# move some web service to the beginning of the list
lo_file_number = file_number.lower()
if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number)
):
sources.insert(0, sources.pop(sources.index("carib")))
elif "avsox" in sources and (re.match(r"^\d{5,}", file_number) or
"heyzo" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("javdb")))
sources.insert(1, sources.pop(sources.index("avsox")))
elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or
"siro" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("mgstage")))
elif "fc2" in sources and ("fc2" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("javdb")))
sources.insert(1, sources.pop(sources.index("fc2")))
elif "dlsite" in sources and (
"rj" in lo_file_number or "vj" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("dlsite")))
json_data = {}
if conf.multi_threading():
pool = ThreadPool(processes=11)
pool = ThreadPool(processes=len(conf.sources().split(',')))
# Set the priority of multi-thread crawling and join the multi-thread queue
for source in sources: