From 5ef16e3a6de5c3bb9381a6ceb9fe4b71619bcd81 Mon Sep 17 00:00:00 2001
From: lededev <lededev@noreplay.github.com>
Date: Mon, 18 Oct 2021 18:09:36 +0800
Subject: [PATCH] =?UTF-8?q?=E5=89=A7=E6=83=85=E7=AE=80=E4=BB=8B=E6=96=B0?=
 =?UTF-8?q?=E5=A2=9E=E8=BF=90=E8=A1=8C=E6=A8=A1=E5=BC=8Frun=5Fmode,=200:?=
 =?UTF-8?q?=E9=A1=BA=E5=BA=8F=E6=89=A7=E8=A1=8C=201:=E7=BA=BF=E7=A8=8B?=
 =?UTF-8?q?=E6=B1=A0=202:=E8=BF=9B=E7=A8=8B=E6=B1=A0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 WebCrawler/storyline.py | 23 +++++++++++++++++++----
 config.ini              |  3 +++
 config.py               |  7 +++++++
 3 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/WebCrawler/storyline.py b/WebCrawler/storyline.py
index 5ad4fd7..d9da869 100644
--- a/WebCrawler/storyline.py
+++ b/WebCrawler/storyline.py
@@ -2,13 +2,25 @@ import sys
 sys.path.append('../')
 import re
 import json
+import builtins
 from ADC_function import *
 from multiprocessing import Pool
+from multiprocessing.dummy import Pool as ThreadPool
 from difflib import SequenceMatcher
 from unicodedata import category
 
 G_registered_storyline_site = {"airav", "avno1", "xcity", "amazon"}
 
+G_mode_txt = ('顺序执行','线程池','进程池')
+
+class noThread(object):
+    def map(self, fn, param):
+        return builtins.map(fn, param)
+    def __enter__(self):
+        return self
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
 
 # 获取剧情介绍 从列表中的站点同时查，取值优先级从前到后
 def getStoryline(number, title):
@@ -18,9 +30,12 @@ def getStoryline(number, title):
     storyine_sites = conf.storyline_site().split(',')
     apply_sites = [ s for s in storyine_sites if s in G_registered_storyline_site]
     mp_args = ((site, number, title, debug) for site in apply_sites)
-    # choose process pool not thread pool because https://www.python.org/dev/peps/pep-0371/
-    with Pool() as proc_pool:
-        result = proc_pool.map(getStoryline_mp, mp_args)
+    cores = min(len(apply_sites), os.cpu_count())
+    run_mode = conf.storyline_mode()
+    assert run_mode in (0,1,2)
+    with ThreadPool(cores) if run_mode == 1 else Pool(cores) if run_mode == 2 else noThread() as pool:
+        result = pool.map(getStoryline_mp, mp_args)
+    result = list(result) if run_mode == 0 else result
     if not debug and conf.storyline_show() == 0:
         for value in result:
             if isinstance(value, str) and len(value):
@@ -28,7 +43,7 @@ def getStoryline(number, title):
         return ''
     # 以下debug结果输出会写入日志，进程池中的则不会，只在标准输出中显示
     cnt = len(apply_sites)
-    s = f'[!]MP Storyline 运行{cnt}个进程总用时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}'
+    s = f'[!]Storyline{G_mode_txt[run_mode]}模式运行{cnt}个进程总用时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}'
     first = True
     sel = ''
     for i in range(cnt):
diff --git a/config.ini b/config.ini
index 5125ad3..700fa95 100755
--- a/config.ini
+++ b/config.ini
@@ -92,5 +92,8 @@ extrafanart_folder=extrafanart
 ; 于amazon商城没有番号信息，选中对应DVD的准确率仅99.6%。如果列表为空则不查询，设置成不查询可大幅提高刮削速度。
 ; site=
 site=airav,avno1,xcity,amazon
+; 运行模式：0:顺序执行(最慢) 1:线程池(默认值) 2:进程池(启动开销比线程池大，并发站点越多越快)
+run_mode=1
 ; show_result剧情简介调试信息 0关闭 1简略 2详细(详细部分不记入日志)，剧情简介失效时可打开2查看原因
 show_result=0
+
diff --git a/config.py b/config.py
index abe030e..83a36bc 100644
--- a/config.py
+++ b/config.py
@@ -252,6 +252,12 @@ class Config:
         except:
             return 0
 
+    def storyline_mode(self) -> int:
+        try:
+            v = self.conf.getint("storyline", "run_mode")
+            return v if v in (0,1,2) else 2 if v > 2 else 0
+        except:
+            return 1
 
     @staticmethod
     def _exit(sec: str) -> None:
@@ -350,6 +356,7 @@ class Config:
         conf.add_section(sec14)
         conf.set(sec14, "site", "airav,avno1,xcity,amazon")
         conf.set(sec14, "show_result", 0)
+        conf.set(sec14, "run_mode", 1)
 
         return conf