From 0e0b92a9fa437ef9bf64a8b98f26233a4607e3a9 Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 16 Apr 2022 19:31:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E-C=E5=91=BD=E4=BB=A4=E5=8F=AF?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=AF=B9=E4=BB=BB=E4=BD=95=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=8F=82=E6=95=B0=E8=BF=9B=E8=A1=8C=E8=A6=86?= =?UTF-8?q?=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Movie_Data_Capture.py | 52 +++++++------- WebCrawler/dlsite.py | 2 +- WebCrawler/javbus.py | 2 +- WebCrawler/javdb.py | 2 +- WebCrawler/madou.py | 13 ++-- config.py | 156 +++++++++++++++++++++++------------------- 6 files changed, 125 insertions(+), 102 deletions(-) diff --git a/Movie_Data_Capture.py b/Movie_Data_Capture.py index 6291f87..85175db 100644 --- a/Movie_Data_Capture.py +++ b/Movie_Data_Capture.py @@ -76,6 +76,8 @@ def argparse_function(ver: str) -> typing.Tuple[str, str, str, str, bool, bool]: help="Override [priority]website= in config.") parser.add_argument("-D", "--download-images", dest='dnimg', action="store_true", help="Override [common]download_only_missing_images=0 force invoke image downloading.") + parser.add_argument("-C", "--config-override", dest='cfgcmd', default='', nargs='?', + help="Common use config override. grammar: section:key=value[;section:key=value] eg. 'de:s=1' or 'debug_mode:switch=1' override[debug_mode]switch=1") parser.add_argument("-z", "--zero-operation", dest='zero_op', action="store_true", help="""Only show job list of files and numbers, and **NO** actual operation is performed. It may help you correct wrong numbers before real job.""") @@ -83,35 +85,38 @@ is performed. It may help you correct wrong numbers before real job.""") args = parser.parse_args() - def get_natural_number_or_none(value): - return int(value) if isinstance(value, str) and value.isnumeric() and int(value) >= 0 else None + def set_natural_number_or_none(sk, value): + if isinstance(value, str) and value.isnumeric() and int(value) >= 0: + conf.set_override(f'{sk}={value}') - def get_str_or_none(value): - return value if isinstance(value, str) and len(value) else None + def set_str_or_none(sk, value): + if isinstance(value, str) and len(value): + conf.set_override(f'{sk}={value}') - def get_bool_or_none(value): - return True if isinstance(value, bool) and value else None + def set_bool_or_none(sk, value): + if isinstance(value, bool) and value: + conf.set_override(f'{sk}=1') - config.G_conf_override["common:main_mode"] = get_natural_number_or_none(args.main_mode) - config.G_conf_override["common:link_mode"] = get_natural_number_or_none(args.link_mode) - config.G_conf_override["common:source_folder"] = get_str_or_none(args.path) - config.G_conf_override["common:auto_exit"] = get_bool_or_none(args.auto_exit) - config.G_conf_override["common:nfo_skip_days"] = get_natural_number_or_none(args.days) - config.G_conf_override["common:stop_counter"] = get_natural_number_or_none(args.cnt) - config.G_conf_override["common:ignore_failed_list"] = get_bool_or_none(args.ignore_failed_list) - config.G_conf_override["debug_mode:switch"] = get_bool_or_none(args.debug) - config.G_conf_override["common:rerun_delay"] = get_str_or_none(args.delaytm) - config.G_conf_override["priority:website"] = get_str_or_none(args.site) - if get_bool_or_none(args.dnimg): - config.G_conf_override["common:download_only_missing_images"] = False + set_natural_number_or_none("common:main_mode", args.main_mode) + set_natural_number_or_none("common:link_mode", args.link_mode) + set_str_or_none("common:source_folder", args.path) + set_bool_or_none("common:auto_exit", args.auto_exit) + set_natural_number_or_none("common:nfo_skip_days", args.days) + set_natural_number_or_none("common:stop_counter", args.cnt) + set_bool_or_none("common:ignore_failed_list", args.ignore_failed_list) + set_str_or_none("common:rerun_delay", args.delaytm) + set_str_or_none("priority:website", args.site) + if isinstance(args.dnimg, bool) and args.dnimg: + conf.set_override("common:download_only_missing_images=0") + set_bool_or_none("debug_mode:switch", args.debug) + if isinstance(args.cfgcmd, str) and len(args.cfgcmd.strip()): + conf.set_override(args.cfgcmd.strip()) no_net_op = False if conf.main_mode() == 3: no_net_op = args.no_network_operation if no_net_op: - config.G_conf_override["common:stop_counter"] = 0 - config.G_conf_override["common:rerun_delay"] = '0s' - config.G_conf_override["face:aways_imagecut"] = True + conf.set_override("common:stop_counter=0;common:rerun_delay=0s;face:aways_imagecut=1") return args.file, args.number, args.logdir, args.regexstr, args.zero_op, no_net_op @@ -303,8 +308,9 @@ def signal_handler(*args): def sigdebug_handler(*args): - config.G_conf_override["debug_mode:switch"] = not config.G_conf_override["debug_mode:switch"] - print('[!]Debug {}'.format('On' if config.getInstance().debug() else 'oFF')) + conf = config.getInstance() + conf.set_override(f"debug_mode:switch={int(not conf.debug())}") + print(f"[!]Debug {('oFF', 'On')[int(conf.debug())]}") # 新增失败文件列表跳过处理，及.nfo修改天数跳过处理，提示跳过视频总数，调试模式(-g)下详细被跳过文件，跳过小广告 diff --git a/WebCrawler/dlsite.py b/WebCrawler/dlsite.py index bae2791..19ab291 100644 --- a/WebCrawler/dlsite.py +++ b/WebCrawler/dlsite.py @@ -157,6 +157,6 @@ def main(number): # main('DV-1562') # input("[+][+]Press enter key exit, you can check the error messge before you exit.\n[+][+]按回车键结束，你可以在结束之前查看和错误信息。") if __name__ == "__main__": - config.G_conf_override["debug_mode:switch"] = True + config.getInstance().set_override("debug_mode:switch=1") print(main('VJ013178')) print(main('RJ329607')) diff --git a/WebCrawler/javbus.py b/WebCrawler/javbus.py index 5218016..d565652 100644 --- a/WebCrawler/javbus.py +++ b/WebCrawler/javbus.py @@ -174,7 +174,7 @@ def main(number): return js if __name__ == "__main__" : - config.G_conf_override['debug_mode:switch'] = True + config.getInstance().set_override("debug_mode:switch=1") # print(main('ABP-888')) # print(main('ABP-960')) # print(main('ADV-R0624')) # 404 diff --git a/WebCrawler/javdb.py b/WebCrawler/javdb.py index 3dfff16..c65a0dd 100755 --- a/WebCrawler/javdb.py +++ b/WebCrawler/javdb.py @@ -328,7 +328,7 @@ def main(number): # main('DV-1562') # input("[+][+]Press enter key exit, you can check the error messge before you exit.\n[+][+]按回车键结束，你可以在结束之前查看和错误信息。") if __name__ == "__main__": - config.G_conf_override['debug_mode:switch'] = True + config.getInstance().set_override("debug_mode:switch=1") # print(main('blacked.20.05.30')) # print(main('AGAV-042')) # print(main('BANK-022')) diff --git a/WebCrawler/madou.py b/WebCrawler/madou.py index eb1f365..c04d9a2 100644 --- a/WebCrawler/madou.py +++ b/WebCrawler/madou.py @@ -165,10 +165,11 @@ def main(number): if __name__ == '__main__': - config.G_conf_override['debug_mode:switch'] = True - print(main('TM0002')) - print(main('MD0222')) - print(main('MD0140-2')) - print(main('MAD039')) - print(main('JDMY027')) + config.getInstance().set_override("debug_mode:switch=1") + print(main('MD0129')) + # print(main('TM0002')) + # print(main('MD0222')) + # print(main('MD0140-2')) + # print(main('MAD039')) + # print(main('JDMY027')) diff --git a/config.py b/config.py index 6f129f5..4b105b8 100644 --- a/config.py +++ b/config.py @@ -10,18 +10,7 @@ G_conf_override = { # index 0 save Config() first instance for quick access by using getInstance() 0: None, # register override config items - "common:main_mode": None, - "common:link_mode": None, - "common:source_folder": None, - "common:auto_exit": None, - "common:nfo_skip_days": None, - "common:stop_counter": None, - "common:ignore_failed_list": None, - "common:rerun_delay": None, - "debug_mode:switch": None, - "face:aways_imagecut": None, - "priority:website": None, - "common:download_only_missing_images": None + # no need anymore } @@ -104,32 +93,83 @@ class Config: # sys.exit(3) # #self.conf = self._default_config() - def getboolean_override(self, section, item, fallback=None) -> bool: - if G_conf_override[f"{section}:{item}"] is not None: - return bool(G_conf_override[f"{section}:{item}"]) - if fallback is not None: - return self.conf.getboolean(section, item, fallback=fallback) - return self.conf.getboolean(section, item) + def set_override(self, option_cmd: str): + """ + 通用的参数覆盖选项 -C 配置覆盖串 + 配置覆盖串语法：小节名:键名=值[;[小节名:]键名=值][;[小节名:]键名+=值] 多个键用分号分隔 名称可省略部分尾部字符 + 或 小节名:键名+=值[;[小节名:]键名=值][;[小节名:]键名+=值] 在已有值的末尾追加内容，多个键的=和+=可以交叉出现 + 例子: face:aspect_ratio=2;aways_imagecut=1;priority:website=javdb + 小节名必须出现在开头至少一次，分号后可只出现键名=值，不再出现小节名，如果后续全部键名都属于同一个小节 + 例如配置文件存在两个小节[proxy][priority]，那么pro可指代proxy，pri可指代priority + [face] ;face小节下方有4个键名locations_model= uncensored_only= aways_imagecut= aspect_ratio= + l,lo,loc,loca,locat,locati...直到locations_model完整名称都可以用来指代locations_model=键名 + u,un,unc...直到uncensored_only完整名称都可以用来指代uncensored_only=键名 + aw,awa...直到aways_imagecut完整名称都可以用来指代aways_imagecut=键名 + as,asp...aspect_ratio完整名称都可以用来指代aspect_ratio=键名 + a则因为二义性，不是合法的省略键名 + """ + def err_exit(str): + print(str) + sys.exit(2) - def getint_override(self, section, item, fallback=None) -> int: - if G_conf_override[f"{section}:{item}"] is not None: - return int(G_conf_override[f"{section}:{item}"]) - if fallback is not None: - return self.conf.getint(section, item, fallback=fallback) - return self.conf.getint(section, item) - - def get_override(self, section, item) -> str: - return self.conf.get(section, item) if G_conf_override[f"{section}:{item}"] is None else str( - G_conf_override[f"{section}:{item}"]) + sections = self.conf.sections() + sec_name = None + for cmd in option_cmd.split(';'): + syntax_err = True + rex = re.findall(r'^(.*?):(.*?)(=|\+=)(.*)$', cmd, re.U) + if len(rex) and len(rex[0]) == 4: + (sec, key, assign, val) = rex[0] + sec_lo = sec.lower().strip() + key_lo = key.lower().strip() + syntax_err = False + elif sec_name: # 已经出现过一次小节名，属于同一个小节的后续键名可以省略小节名 + rex = re.findall(r'^(.*?)(=|\+=)(.*)$', cmd, re.U) + if len(rex) and len(rex[0]) == 3: + (key, assign, val) = rex[0] + sec_lo = sec_name.lower() + key_lo = key.lower().strip() + syntax_err = False + if syntax_err: + err_exit(f"[-]Config override syntax incorrect. example: 'd:s=1' or 'debug_mode:switch=1'. cmd='{cmd}' all='{option_cmd}'") + if not len(sec_lo): + err_exit(f"[-]Config override Section name '{sec}' is empty! cmd='{cmd}'") + if not len(key_lo): + err_exit(f"[-]Config override Key name '{key}' is empty! cmd='{cmd}'") + if not len(val.strip()): + print(f"[!]Conig overide value '{val}' is empty! cmd='{cmd}'") + sec_name = None + for s in sections: + if not s.lower().startswith(sec_lo): + continue + if sec_name: + err_exit(f"[-]Conig overide Section short name '{sec_lo}' is not unique! dup1='{sec_name}' dup2='{s}' cmd='{cmd}'") + sec_name = s + if sec_name is None: + err_exit(f"[-]Conig overide Section name '{sec}' not found! cmd='{cmd}'") + key_name = None + keys = self.conf[sec_name] + for k in keys: + if not k.lower().startswith(key_lo): + continue + if key_name: + err_exit(f"[-]Conig overide Key short name '{key_lo}' is not unique! dup1='{key_name}' dup2='{k}' cmd='{cmd}'") + key_name = k + if key_name is None: + err_exit(f"[-]Conig overide Key name '{key}' not found! cmd='{cmd}'") + if assign == "+=": + val = keys[key_name] + val + if self.debug(): + print(f"[!]Set config override [{sec_name}]{key_name}={val} by cmd='{cmd}'") + self.conf.set(sec_name, key_name, val) def main_mode(self) -> int: try: - return self.getint_override("common", "main_mode") + return self.conf.getint("common", "main_mode") except ValueError: self._exit("common:main_mode") def source_folder(self) -> str: - return self.get_override("common", "source_folder") + return self.conf.get("common", "source_folder") def failed_folder(self) -> str: return self.conf.get("common", "failed_output_folder") @@ -141,7 +181,7 @@ class Config: return self.conf.get("common", "actor_gender") def link_mode(self) -> int: - return self.getint_override("common", "link_mode") + return self.conf.getint("common", "link_mode") def scan_hardlink(self) -> bool: return self.conf.getboolean("common", "scan_hardlink", fallback=False)#未找到配置选项,默认不刮削 @@ -150,7 +190,7 @@ class Config: return self.conf.getboolean("common", "failed_move") def auto_exit(self) -> bool: - return self.getboolean_override("common", "auto_exit") + return self.conf.getboolean("common", "auto_exit") def translate_to_sc(self) -> bool: return self.conf.getboolean("common", "translate_to_sc") @@ -162,22 +202,22 @@ class Config: return self.conf.getboolean("common", "del_empty_folder") def nfo_skip_days(self) -> int: - return self.getint_override("common", "nfo_skip_days", fallback=30) + return self.conf.getint("common", "nfo_skip_days", fallback=30) def stop_counter(self) -> int: - return self.getint_override("common", "stop_counter", fallback=0) + return self.conf.getint("common", "stop_counter", fallback=0) def ignore_failed_list(self) -> bool: - return self.getboolean_override("common", "ignore_failed_list") + return self.conf.getboolean("common", "ignore_failed_list") def download_only_missing_images(self) -> bool: - return self.getboolean_override("common", "download_only_missing_images") + return self.conf.getboolean("common", "download_only_missing_images") def mapping_table_validity(self) -> int: return self.conf.getint("common", "mapping_table_validity") def rerun_delay(self) -> int: - value = self.get_override("common", "rerun_delay") + value = self.conf.get("common", "rerun_delay") if not (isinstance(value, str) and re.match(r'^[\dsmh]+$', value, re.I)): return 0 # not match '1h30m45s' or '30' or '1s2m1h4s5m' if value.isnumeric() and int(value) >= 0: @@ -289,7 +329,7 @@ class Config: self._exit("update:update_check") def sources(self) -> str: - return self.get_override("priority", "website") + return self.conf.get("priority", "website") def escape_literals(self) -> str: return self.conf.get("escape", "literals") @@ -298,7 +338,7 @@ class Config: return self.conf.get("escape", "folders") def debug(self) -> bool: - return self.getboolean_override("debug_mode", "switch") + return self.conf.getboolean("debug_mode", "switch") def is_storyline(self) -> bool: try: @@ -359,7 +399,7 @@ class Config: return self.conf.getboolean("face", "uncensored_only", fallback=True) def face_aways_imagecut(self) -> bool: - return self.getboolean_override("face", "aways_imagecut", fallback=False) + return self.conf.getboolean("face", "aways_imagecut", fallback=False) def face_aspect_ratio(self) -> float: return self.conf.getfloat("face", "aspect_ratio", fallback=2.12) @@ -527,8 +567,7 @@ if __name__ == "__main__": config = Config() - mfilter = {'conf', 'proxy', '_exit', '_default_config', 'getboolean_override', 'getint_override', 'get_override', - 'ini_path'} + mfilter = {'conf', 'proxy', '_exit', '_default_config', 'ini_path', 'set_override'} for _m in [m for m in dir(config) if not m.startswith('__') and m not in mfilter]: evprint(f'config.{_m}()') pfilter = {'proxies', 'SUPPORT_PROXY_TYPE'} @@ -537,36 +576,13 @@ if __name__ == "__main__": for _p in [p for p in dir(getInstance().proxy()) if not p.startswith('__') and p not in pfilter]: evprint(f'getInstance().proxy().{_p}') - # Override Test - G_conf_override["common:nfo_skip_days"] = 4321 - G_conf_override["common:stop_counter"] = 1234 - assert config.nfo_skip_days() == 4321 - assert getInstance().stop_counter() == 1234 - # remove override - G_conf_override["common:stop_counter"] = None - G_conf_override["common:nfo_skip_days"] = None - assert config.nfo_skip_days() != 4321 - assert config.stop_counter() != 1234 # Create new instance conf2 = Config() assert getInstance() != conf2 assert getInstance() == config - G_conf_override["common:main_mode"] = 9 - G_conf_override["common:source_folder"] = "A:/b/c" - # Override effect to all instances - assert config.main_mode() == 9 - assert conf2.main_mode() == 9 - assert getInstance().main_mode() == 9 - assert conf2.source_folder() == "A:/b/c" - print("### Override Test ###".center(36)) - evprint('getInstance().main_mode()') - evprint('config.source_folder()') - G_conf_override["common:main_mode"] = None - evprint('conf2.main_mode()') - evprint('config.main_mode()') - # unregister key acess will raise except - try: - print(G_conf_override["common:actor_gender"]) - except KeyError as ke: - print(f'Catched KeyError: {ke} is not a register key of G_conf_override dict.', file=sys.stderr) + + conf2.set_override("d:s=1;face:asp=2;f:aw=0;pri:w=javdb;f:l=") + assert conf2.face_aspect_ratio() == 2 + assert conf2.face_aways_imagecut() == False + assert conf2.sources() == "javdb" print(f"Load Config file '{conf2.ini_path}'.")