From 3224f8c1abf7b13c3f964861b87915af1efd283d Mon Sep 17 00:00:00 2001 From: lededev Date: Mon, 18 Apr 2022 01:07:58 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8F=96=E6=B6=88storyline=E8=BF=9B=E7=A8=8B?= =?UTF-8?q?=E6=B1=A0=E6=A8=A1=E5=BC=8F=E4=BB=A5=E6=8F=90=E5=8D=87=E5=85=BC?= =?UTF-8?q?=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Movie_Data_Capture.py | 1 - WebCrawler/storyline.py | 13 +++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Movie_Data_Capture.py b/Movie_Data_Capture.py index c2ef207..672671d 100644 --- a/Movie_Data_Capture.py +++ b/Movie_Data_Capture.py @@ -668,7 +668,6 @@ def period(delta, pattern): if __name__ == '__main__': version = '6.0.3' - multiprocessing.freeze_support() urllib3.disable_warnings() # Ignore http proxy warning app_start = time.time() diff --git a/WebCrawler/storyline.py b/WebCrawler/storyline.py index 022853c..40117b5 100644 --- a/WebCrawler/storyline.py +++ b/WebCrawler/storyline.py @@ -5,7 +5,6 @@ import json import builtins from ADC_function import * from lxml.html import fromstring -from multiprocessing import Pool from multiprocessing.dummy import Pool as ThreadPool from difflib import SequenceMatcher from unicodedata import category @@ -13,7 +12,7 @@ from number_parser import is_uncensored G_registered_storyline_site = {"airavwiki", "airav", "avno1", "xcity", "amazon", "58avgo"} -G_mode_txt = ('顺序执行','线程池','进程池') +G_mode_txt = ('顺序执行','线程池') class noThread(object): def map(self, fn, param): @@ -50,9 +49,8 @@ def getStoryline(number, title, sites: list=None, 无码=None): cores = min(len(apply_sites), os.cpu_count()) if cores == 0: return '' - run_mode = conf.storyline_mode() - assert run_mode in (0,1,2) - with ThreadPool(cores) if run_mode == 1 else Pool(cores) if run_mode == 2 else noThread() as pool: + run_mode = 1 if conf.storyline_mode() > 0 else 0 + with ThreadPool(cores) if run_mode > 0 else noThread() as pool: results = pool.map(getStoryline_mp, mp_args) sel = '' if not debug and conf.storyline_show() == 0: @@ -63,7 +61,7 @@ def getStoryline(number, title, sites: list=None, 无码=None): if not len(sel): sel = value return sel - # 以下debug结果输出会写入日志,进程池中的则不会,只在标准输出中显示 + # 以下debug结果输出会写入日志 s = f'[!]Storyline{G_mode_txt[run_mode]}模式运行{len(apply_sites)}个任务共耗时(含启动开销){time.time() - start_time:.3f}秒,结束于{time.strftime("%H:%M:%S")}' sel_site = '' for site, desc in zip(apply_sites, results): @@ -101,8 +99,7 @@ def getStoryline_mp(args): storyline = getStoryline_58avgo(number, debug) if not debug: return storyline - # 进程池模式的子进程getStoryline_*()的print()不会写入日志中,线程池和顺序执行不受影响 - print("[!]MP 进程[{}]运行{:.3f}秒,结束于{}返回结果: {}".format( + print("[!]MP 线程[{}]运行{:.3f}秒,结束于{}返回结果: {}".format( site, time.time() - start_time, time.strftime("%H:%M:%S"),