bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #591 from lededev/xcity-f1

Browse Source 
xcity.py: get detail page by form query
This commit is contained in:
Yoshiko2
2021-09-27 22:00:43 +08:00
committed by GitHub
parent 875c5dc3a1 4ffc34a5cf
commit 30bc6a59c6
7 changed files with 158 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
WebCrawler/__init__.py
View File

@@ -55,7 +55,7 @@ def get_data_from_json(file_number, conf: config.Config): # 从JSON返回元数
# default fetch order list, from the beginning to the end
sources = conf.sources().split(',')
if not len(conf.sources()) > 60:
if not len(conf.sources()) > 80:
# if the input file name matches certain rules,
# move some web service to the beginning of the list
lo_file_number = file_number.lower()
@@ -82,6 +82,11 @@ def get_data_from_json(file_number, conf: config.Config): # 从JSON返回元数
"rj" in lo_file_number or "vj" in lo_file_number
):
sources.insert(0, sources.pop(sources.index("dlsite")))
elif re.match(r"^[a-z0-9]{3,}$", lo_file_number):
if "javdb" in sources:
sources.insert(0, sources.pop(sources.index("javdb")))
if "xcity" in sources:
sources.insert(0, sources.pop(sources.index("xcity")))
# check sources in func_mapping
todel = []