From 0622b8bda35a42cc62196f6f232bfd0a0af5ca37 Mon Sep 17 00:00:00 2001 From: lededev Date: Wed, 3 Nov 2021 20:14:06 +0800 Subject: [PATCH] map function call local function --- WebCrawler/storyline.py | 56 ++++++++++++++++++++--------------------- core.py | 14 +++++------ 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/WebCrawler/storyline.py b/WebCrawler/storyline.py index a175c03..9444473 100644 --- a/WebCrawler/storyline.py +++ b/WebCrawler/storyline.py @@ -77,36 +77,34 @@ def getStoryline(number, title, sites: list=None): def getStoryline_mp(args): - return _getStoryline_mp(*args) - - -# 注:新进程的print()不会写入日志中,将来调试修复失效数据源需直接查看标准输出,issue信息需截图屏幕 -def _getStoryline_mp(site, number, title, debug): - start_time = time.time() - storyline = None - if not isinstance(site, str): + def _inner(site, number, title, debug): + start_time = time.time() + storyline = None + if not isinstance(site, str): + return storyline + elif site == "airavwiki": + storyline = getStoryline_airavwiki(number, debug) + elif site == "airav": + storyline = getStoryline_airav(number, debug) + elif site == "avno1": + storyline = getStoryline_avno1(number, debug) + elif site == "xcity": + storyline = getStoryline_xcity(number, debug) + elif site == "amazon": + storyline = getStoryline_amazon(title, number, debug) + elif site == "58avgo": + storyline = getStoryline_58avgo(number, debug) + if not debug: + return storyline + # 进程池模式的子进程getStoryline_*()的print()不会写入日志中,线程池和顺序执行不受影响 + print("[!]MP 进程[{}]运行{:.3f}秒,结束于{}返回结果: {}".format( + site, + time.time() - start_time, + time.strftime("%H:%M:%S"), + storyline if isinstance(storyline, str) and len(storyline) else '[空]') + ) return storyline - elif site == "airavwiki": - storyline = getStoryline_airavwiki(number, debug) - elif site == "airav": - storyline = getStoryline_airav(number, debug) - elif site == "avno1": - storyline = getStoryline_avno1(number, debug) - elif site == "xcity": - storyline = getStoryline_xcity(number, debug) - elif site == "amazon": - storyline = getStoryline_amazon(title, number, debug) - elif site == "58avgo": - storyline = getStoryline_58avgo(number, debug) - if not debug: - return storyline - print("[!]MP 进程[{}]运行{:.3f}秒,结束于{}返回结果: {}".format( - site, - time.time() - start_time, - time.strftime("%H:%M:%S"), - storyline if isinstance(storyline, str) and len(storyline) else '[空]') - ) - return storyline + return _inner(*args) def getStoryline_airav(number, debug): diff --git a/core.py b/core.py index 3e17fed..069c327 100755 --- a/core.py +++ b/core.py @@ -217,14 +217,12 @@ def extrafanart_download_one_by_one(data, path, filepath): print(f'[!]Extrafanart download one by one mode runtime {time.perf_counter() - tm_start:.3f}s') def download_one_file(args): - return _download_one_file(*args) - -def _download_one_file(url: str, save_path: Path): - filebytes = get_html(url, return_type='content') - if isinstance(filebytes, bytes) and len(filebytes): - if len(filebytes) == save_path.open('wb').write(filebytes): - return str(save_path) - return None + def _inner(url: str, save_path: Path): + filebytes = get_html(url, return_type='content') + if isinstance(filebytes, bytes) and len(filebytes): + if len(filebytes) == save_path.open('wb').write(filebytes): + return str(save_path) + return _inner(*args) def extrafanart_download_threadpool(url_list, save_dir, number): tm_start = time.perf_counter()