bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

取消storyline进程池模式以提升兼容性

Browse Source
This commit is contained in:
lededev
2022-04-18 01:07:58 +08:00
parent 5d00dd29e4
commit 3224f8c1ab
2 changed files with 5 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Movie_Data_Capture.py
View File

@@ -668,7 +668,6 @@ def period(delta, pattern):
if __name__ == '__main__': if __name__ == '__main__':
version = '6.0.3' version = '6.0.3'
multiprocessing.freeze_support()
urllib3.disable_warnings() # Ignore http proxy warning urllib3.disable_warnings() # Ignore http proxy warning
app_start = time.time() app_start = time.time()

13
WebCrawler/storyline.py
View File

@@ -5,7 +5,6 @@ import json
import builtins import builtins
from ADC_function import * from ADC_function import *
from lxml.html import fromstring from lxml.html import fromstring
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool from multiprocessing.dummy import Pool as ThreadPool
from difflib import SequenceMatcher from difflib import SequenceMatcher
from unicodedata import category from unicodedata import category
@@ -13,7 +12,7 @@ from number_parser import is_uncensored
G_registered_storyline_site = {"airavwiki", "airav", "avno1", "xcity", "amazon", "58avgo"} G_registered_storyline_site = {"airavwiki", "airav", "avno1", "xcity", "amazon", "58avgo"}
G_mode_txt = ('顺序执行','线程池','进程池') G_mode_txt = ('顺序执行','线程池')
class noThread(object): class noThread(object):
def map(self, fn, param): def map(self, fn, param):
@@ -50,9 +49,8 @@ def getStoryline(number, title, sites: list=None, 无码=None):
cores = min(len(apply_sites), os.cpu_count()) cores = min(len(apply_sites), os.cpu_count())
if cores == 0: if cores == 0:
return '' return ''
run_mode = conf.storyline_mode() run_mode = 1 if conf.storyline_mode() > 0 else 0
assert run_mode in (0,1,2) with ThreadPool(cores) if run_mode > 0 else noThread() as pool:
with ThreadPool(cores) if run_mode == 1 else Pool(cores) if run_mode == 2 else noThread() as pool:
results = pool.map(getStoryline_mp, mp_args) results = pool.map(getStoryline_mp, mp_args)
sel = '' sel = ''
if not debug and conf.storyline_show() == 0: if not debug and conf.storyline_show() == 0:
@@ -63,7 +61,7 @@ def getStoryline(number, title, sites: list=None, 无码=None):
if not len(sel): if not len(sel):
sel = value sel = value
return sel return sel
# 以下debug结果输出会写入日志，进程池中的则不会，只在标准输出中显示 # 以下debug结果输出会写入日志
s = f'[!]Storyline{G_mode_txt[run_mode]}模式运行{len(apply_sites)}个任务共耗时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}' s = f'[!]Storyline{G_mode_txt[run_mode]}模式运行{len(apply_sites)}个任务共耗时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}'
sel_site = '' sel_site = ''
for site, desc in zip(apply_sites, results): for site, desc in zip(apply_sites, results):
@@ -101,8 +99,7 @@ def getStoryline_mp(args):
storyline = getStoryline_58avgo(number, debug) storyline = getStoryline_58avgo(number, debug)
if not debug: if not debug:
return storyline return storyline
# 进程池模式的子进程getStoryline_*()的print()不会写入日志中，线程池和顺序执行不受影响 print("[!]MP 线程[{}]运行{:.3f}秒，结束于{}返回结果: {}".format(
print("[!]MP 进程[{}]运行{:.3f}秒，结束于{}返回结果: {}".format(
site, site,
time.time() - start_time, time.time() - start_time,
time.strftime("%H:%M:%S"), time.strftime("%H:%M:%S"),