Fix crash caused by custom site
This commit is contained in:
4
core.py
4
core.py
@@ -71,7 +71,7 @@ def get_data_from_json(file_number, filepath, conf: config.Config): # 从JSON
|
||||
|
||||
# default fetch order list, from the beginning to the end
|
||||
sources = conf.sources().split(',')
|
||||
|
||||
if not len(conf.sources()) > 60:
|
||||
# if the input file name matches certain rules,
|
||||
# move some web service to the beginning of the list
|
||||
lo_file_number = file_number.lower()
|
||||
@@ -99,7 +99,7 @@ def get_data_from_json(file_number, filepath, conf: config.Config): # 从JSON
|
||||
json_data = {}
|
||||
|
||||
if conf.multi_threading():
|
||||
pool = ThreadPool(processes=11)
|
||||
pool = ThreadPool(processes=len(conf.sources().split(',')))
|
||||
|
||||
# Set the priority of multi-thread crawling and join the multi-thread queue
|
||||
for source in sources:
|
||||
|
||||
Reference in New Issue
Block a user