bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #560 from Suwmlee/master

Browse Source 
Check sources
This commit is contained in:
Yoshiko2
2021-08-10 02:48:27 +08:00
committed by GitHub
parent e572a55907 1204076366
commit a1ca4b9cad
1 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
WebCrawler/__init__.py
View File

@@ -62,25 +62,37 @@ def get_data_from_json(file_number, conf: config.Config): # 从JSON返回元数
if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number) if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number)
): ):
sources.insert(0, sources.pop(sources.index("carib"))) sources.insert(0, sources.pop(sources.index("carib")))
elif "avsox" in sources and (re.match(r"^\d{5,}", file_number) or elif re.match(r"^\d{5,}", file_number) or "heyzo" in lo_file_number:
"heyzo" in lo_file_number if "javdb" in sources:
): sources.insert(0, sources.pop(sources.index("javdb")))
sources.insert(0, sources.pop(sources.index("avsox"))) if "avsox" in sources:
sources.insert(1, sources.pop(sources.index("javdb"))) sources.insert(0, sources.pop(sources.index("avsox")))
elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or
"siro" in lo_file_number "siro" in lo_file_number
): ):
sources.insert(0, sources.pop(sources.index("mgstage"))) sources.insert(0, sources.pop(sources.index("mgstage")))
elif "fc2" in sources and ("fc2" in lo_file_number elif "fc2" in lo_file_number:
): if "javdb" in sources:
sources.insert(0, sources.pop(sources.index("fc2club"))) sources.insert(0, sources.pop(sources.index("javdb")))
sources.insert(1, sources.pop(sources.index("fc2"))) if "fc2" in sources:
sources.insert(2, sources.pop(sources.index("javdb"))) sources.insert(0, sources.pop(sources.index("fc2")))
if "fc2club" in sources:
sources.insert(0, sources.pop(sources.index("fc2club")))
elif "dlsite" in sources and ( elif "dlsite" in sources and (
"rj" in lo_file_number or "vj" in lo_file_number "rj" in lo_file_number or "vj" in lo_file_number
): ):
sources.insert(0, sources.pop(sources.index("dlsite"))) sources.insert(0, sources.pop(sources.index("dlsite")))
# check sources in func_mapping
todel = []
for s in sources:
if not s in func_mapping:
print('[!] Source Not Exist : ' + s)
todel.append(s)
for d in todel:
print('[!] Remove Source : ' + s)
sources.remove(d)
json_data = {} json_data = {}
if conf.multi_threading(): if conf.multi_threading():