剧情简介新增运行模式run_mode, 0:顺序执行 1:线程池 2:进程池

WebCrawler/storyline.py

@@ -2,13 +2,25 @@ import sys
 sys.path.append('../')
 import re
 import json
+import builtins
 from ADC_function import *
 from multiprocessing import Pool
+from multiprocessing.dummy import Pool as ThreadPool
 from difflib import SequenceMatcher
 from unicodedata import category
 
 G_registered_storyline_site = {"airav", "avno1", "xcity", "amazon"}
 
+G_mode_txt = ('顺序执行','线程池','进程池')
+
+class noThread(object):
+    def map(self, fn, param):
+        return builtins.map(fn, param)
+    def __enter__(self):
+        return self
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
 
 # 获取剧情介绍 从列表中的站点同时查，取值优先级从前到后
 def getStoryline(number, title):
@@ -18,9 +30,12 @@ def getStoryline(number, title):
     storyine_sites = conf.storyline_site().split(',')
     apply_sites = [ s for s in storyine_sites if s in G_registered_storyline_site]
     mp_args = ((site, number, title, debug) for site in apply_sites)
-    # choose process pool not thread pool because https://www.python.org/dev/peps/pep-0371/
+    cores = min(len(apply_sites), os.cpu_count())
-    with Pool() as proc_pool:
+    run_mode = conf.storyline_mode()
-        result = proc_pool.map(getStoryline_mp, mp_args)
+    assert run_mode in (0,1,2)
+    with ThreadPool(cores) if run_mode == 1 else Pool(cores) if run_mode == 2 else noThread() as pool:
+        result = pool.map(getStoryline_mp, mp_args)
+    result = list(result) if run_mode == 0 else result
     if not debug and conf.storyline_show() == 0:
         for value in result:
             if isinstance(value, str) and len(value):
@@ -28,7 +43,7 @@ def getStoryline(number, title):
         return ''
     # 以下debug结果输出会写入日志，进程池中的则不会，只在标准输出中显示
     cnt = len(apply_sites)
-    s = f'[!]MP Storyline 运行{cnt}个进程总用时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}'
+    s = f'[!]Storyline{G_mode_txt[run_mode]}模式运行{cnt}个进程总用时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}'
     first = True
     sel = ''
     for i in range(cnt):

config.ini

@@ -92,5 +92,8 @@ extrafanart_folder=extrafanart
 ; 于amazon商城没有番号信息，选中对应DVD的准确率仅99.6%。如果列表为空则不查询，设置成不查询可大幅提高刮削速度。
 ; site=
 site=airav,avno1,xcity,amazon
+; 运行模式：0:顺序执行(最慢) 1:线程池(默认值) 2:进程池(启动开销比线程池大，并发站点越多越快)
+run_mode=1
 ; show_result剧情简介调试信息 0关闭 1简略 2详细(详细部分不记入日志)，剧情简介失效时可打开2查看原因
 show_result=0
+

config.py

@@ -252,6 +252,12 @@ class Config:
         except:
             return 0
 
+    def storyline_mode(self) -> int:
+        try:
+            v = self.conf.getint("storyline", "run_mode")
+            return v if v in (0,1,2) else 2 if v > 2 else 0
+        except:
+            return 1
 
     @staticmethod
     def _exit(sec: str) -> None:
@@ -350,6 +356,7 @@ class Config:
         conf.add_section(sec14)
         conf.set(sec14, "site", "airav,avno1,xcity,amazon")
         conf.set(sec14, "show_result", 0)
+        conf.set(sec14, "run_mode", 1)
 
         return conf