bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix order of sources

Browse Source
This commit is contained in:
yoshiko2
2022-05-08 03:26:48 +08:00
parent e3029ef8cd
commit d991fecb73
1 changed files with 18 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
WebCrawler/__init__.py
View File

@@ -72,46 +72,44 @@ def get_data_from_json(file_number, oCC):
conf = config.getInstance() conf = config.getInstance()
# default fetch order list, from the beginning to the end # default fetch order list, from the beginning to the end
sources = conf.sources().split(',') sources = conf.sources().split(',')
def insert(sources,source):
if source in sources:
sources.insert(0, sources.pop(sources.index(source)))
return sources
if len(sources) <= len(func_mapping): if len(sources) <= len(func_mapping):
# if the input file name matches certain rules, # if the input file name matches certain rules,
# move some web service to the beginning of the list # move some web service to the beginning of the list
lo_file_number = file_number.lower() lo_file_number = file_number.lower()
if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number) if "carib" in sources and (re.match(r"^\d{6}-\d{3}", file_number)
): ):
sources.insert(0, sources.pop(sources.index("carib"))) sources = insert(sources,"carib")
elif "item" in file_number:
sources = insert(sources,"getchu")
elif re.match(r"^\d{5,}", file_number) or "heyzo" in lo_file_number: elif re.match(r"^\d{5,}", file_number) or "heyzo" in lo_file_number:
if "javdb" in sources:
sources.insert(0, sources.pop(sources.index("javdb")))
if "avsox" in sources: if "avsox" in sources:
sources.insert(0, sources.pop(sources.index("avsox"))) sources = insert(sources,"avsox")
elif "mgstage" in sources and (re.match(r"\d+\D+", file_number) or elif "mgstage" in sources and \
"siro" in lo_file_number (re.match(r"\d+\D+", file_number) or "siro" in lo_file_number):
): sources = insert(sources,"mgstage")
sources.insert(0, sources.pop(sources.index("mgstage")))
elif "fc2" in lo_file_number: elif "fc2" in lo_file_number:
if "javdb" in sources:
sources.insert(0, sources.pop(sources.index("javdb")))
if "fc2" in sources: if "fc2" in sources:
sources.insert(0, sources.pop(sources.index("fc2"))) sources = insert(sources,"fc2")
if "fc2club" in sources:
sources.insert(0, sources.pop(sources.index("fc2club")))
elif "gcolle" in sources and (re.search("\d{6}", file_number)): elif "gcolle" in sources and (re.search("\d{6}", file_number)):
sources.insert(0, sources.pop(sources.index("gcolle"))) sources = insert(sources,"gcolle")
elif "dlsite" in sources and ( elif "dlsite" in sources and (
"rj" in lo_file_number or "vj" in lo_file_number "rj" in lo_file_number or "vj" in lo_file_number
): ):
sources.insert(0, sources.pop(sources.index("dlsite"))) sources = insert(sources,"dlsite")
elif re.match(r"^[a-z0-9]{3,}$", lo_file_number): elif re.match(r"^[a-z0-9]{3,}$", lo_file_number):
if "javdb" in sources:
sources.insert(0, sources.pop(sources.index("javdb")))
if "xcity" in sources: if "xcity" in sources:
sources.insert(0, sources.pop(sources.index("xcity"))) sources = insert(sources,"xcity")
if "madou" in sources: if "madou" in sources:
sources.insert(0, sources.pop(sources.index("madou"))) sources = insert(sources,"madou")
elif "madou" in sources and ( elif "madou" in sources and (
re.match(r"^[a-z0-9]{3,}-[0-9]{1,}$", lo_file_number) re.match(r"^[a-z0-9]{3,}-[0-9]{1,}$", lo_file_number)
): ):
sources.insert(0, sources.pop(sources.index("madou"))) sources = insert(sources,"madou")
# check sources in func_mapping # check sources in func_mapping
todel = [] todel = []