bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #764 from lededev/unce-1

Browse Source 
修复cnn人脸检测
This commit is contained in:
Yoshiko2
2022-04-18 18:45:30 +08:00
committed by GitHub
parent 0e3ea062b6 6794677006
commit c27a142938
11 changed files with 51 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ADC_function.py
View File

@@ -467,7 +467,7 @@ def download_file_with_filename(url: str, filename: str, path: str) -> None:
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
sys.exit(0)
os._exit(0)
proxies = configProxy.proxies()
headers = {
'User-Agent': G_USER_AGENT}
@@ -484,7 +484,7 @@ def download_file_with_filename(url: str, filename: str, path: str) -> None:
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
sys.exit(0)
os._exit(0)
headers = {
'User-Agent': G_USER_AGENT}
r = requests.get(url, timeout=configProxy.timeout, headers=headers)

6
ImageProcessing/__init__.py
View File

@@ -1,3 +1,6 @@
import sys
sys.path.append('../')
import logging
import os
import config
@@ -104,4 +107,5 @@ def face_center(filename, model):
return (0, 0)
if __name__ == '__main__':
cutImage(1,'H:\\test\\','12.jpg','test.jpg')
cutImage(1,'z:/t/','p.jpg','o.jpg')
#cutImage(1,'H:\\test\\','12.jpg','test.jpg')

8
ImageProcessing/cnn.py
View File

@@ -1,4 +1,8 @@
import hog
import sys
sys.path.append('../')
from ImageProcessing.hog import face_center as hog_face_center
def face_center(filename, model):
return hog.face_center(filename, model)
return hog_face_center(filename, model)

11
Movie_Data_Capture.py
View File

@@ -77,7 +77,7 @@ def argparse_function(ver: str) -> typing.Tuple[str, str, str, str, bool, bool]:
parser.add_argument("-D", "--download-images", dest='dnimg', action="store_true",
help="Override [common]download_only_missing_images=0 force invoke image downloading.")
parser.add_argument("-C", "--config-override", dest='cfgcmd', default='', nargs='?',
help="Common use config override. grammar: section:key=value[;section:key=value] eg. 'de:s=1' or 'debug_mode:switch=1' override[debug_mode]switch=1")
help="Common use config override. grammar: section:key=value[;[section:]key=value] eg. 'de:s=1' or 'debug_mode:switch=1' override[debug_mode]switch=1")
parser.add_argument("-z", "--zero-operation", dest='zero_op', action="store_true",
help="""Only show job list of files and numbers, and **NO** actual operation
is performed. It may help you correct wrong numbers before real job.""")
@@ -116,7 +116,7 @@ is performed. It may help you correct wrong numbers before real job.""")
if conf.main_mode() == 3:
no_net_op = args.no_network_operation
if no_net_op:
conf.set_override("common:stop_counter=0;common:rerun_delay=0s;face:aways_imagecut=1")
conf.set_override("common:stop_counter=0;rerun_delay=0s;face:aways_imagecut=1")
return args.file, args.number, args.logdir, args.regexstr, args.zero_op, no_net_op
@@ -304,7 +304,7 @@ def close_logfile(logdir: str):
def signal_handler(*args):
print('[!]Ctrl+C detected, Exit.')
sys.exit(9)
os._exit(9)
def sigdebug_handler(*args):
@@ -426,7 +426,7 @@ def create_failed_folder(failed_folder: str):
os.makedirs(failed_folder)
except:
print(f"[-]Fatal error! Can not make folder '{failed_folder}'")
sys.exit(0)
os._exit(0)
def rm_empty_folder(path):
@@ -517,7 +517,7 @@ def main(args: tuple) -> Path:
folder_path = ""
if main_mode not in (1, 2, 3):
print(f"[-]Main mode must be 1 or 2 or 3! You can run '{os.path.basename(sys.argv[0])} --help' for more help.")
sys.exit(4)
os._exit(4)
signal.signal(signal.SIGINT, signal_handler)
if sys.platform == 'win32':
@@ -668,7 +668,6 @@ def period(delta, pattern):
if __name__ == '__main__':
version = '6.0.3'
multiprocessing.freeze_support()
urllib3.disable_warnings() # Ignore http proxy warning
app_start = time.time()

2
WebCrawler/carib.py
View File

@@ -60,7 +60,7 @@ def get_year(lx: html.HtmlElement) -> str:
def get_outline(lx: html.HtmlElement, number: str, title: str) -> str:
o = lx.xpath("//div[@class='movie-info section']/p[@itemprop='description']/text()")[0].strip()
g = getStoryline(number, title)
g = getStoryline(number, title, 无码=True)
if len(g):
return g
return o

8
WebCrawler/javbus.py
View File

@@ -60,10 +60,10 @@ def getCID(html):
string = html.xpath("//a[contains(@class,'sample-box')][1]/@href")[0].replace('https://pics.dmm.co.jp/digital/video/','')
result = re.sub('/.*?.jpg','',string)
return result
def getOutline(number, title): #获取剧情介绍 多进程并发查询
def getOutline(number, title, uncensored): #获取剧情介绍 多进程并发查询
if any(caller for caller in inspect.stack() if os.path.basename(caller.filename) == 'airav.py'):
return '' # 从airav.py过来的调用不计算outline直接返回，避免重复抓取数据拖慢处理速度
return getStoryline(number,title)
return getStoryline(number,title, 无码=uncensored)
def getSeriseJa(html):
x = html.xpath('//span[contains(text(),"シリーズ:")]/../a/text()')
return str(x[0]) if len(x) else ''
@@ -98,7 +98,7 @@ def main_uncensored(number):
'title': title,
'studio': getStudioJa(lx),
'year': getYear(lx),
'outline': getOutline(w_number, title),
'outline': getOutline(w_number, title, True),
'runtime': getRuntime(lx),
'director': getDirectorJa(lx),
'actor': getActor(lx),
@@ -141,7 +141,7 @@ def main(number):
'title': title,
'studio': getStudio(lx),
'year': getYear(lx),
'outline': getOutline(number, title),
'outline': getOutline(number, title, getUncensored(lx)),
'runtime': getRuntime(lx),
'director': getDirector(lx),
'actor': getActor(lx),

6
WebCrawler/javdb.py
View File

@@ -166,8 +166,8 @@ def getDirector(html):
result1 = str(html.xpath('//strong[contains(text(),"導演")]/../span/text()')).strip(" ['']")
result2 = str(html.xpath('//strong[contains(text(),"導演")]/../span/a/text()')).strip(" ['']")
return str(result1 + result2).strip('+').replace("', '", '').replace('"', '')
def getOutline(number, title): #获取剧情介绍 多进程并发查询
return getStoryline(number,title)
def getOutline(number, title, uncensored): #获取剧情介绍 多进程并发查询
return getStoryline(number, title, 无码=uncensored)
def getSeries(html):
result1 = str(html.xpath('//strong[contains(text(),"系列")]/../span/text()')).strip(" ['']")
result2 = str(html.xpath('//strong[contains(text(),"系列")]/../span/a/text()')).strip(" ['']")
@@ -287,7 +287,7 @@ def main(number):
'actor': getActor(lx),
'title': title,
'studio': getStudio(detail_page, lx),
'outline': getOutline(number, title),
'outline': getOutline(number, title, getUncensored(lx)),
'runtime': getRuntime(lx),
'director': getDirector(lx),
'release': getRelease(detail_page),

18
WebCrawler/storyline.py
View File

@@ -5,7 +5,6 @@ import json
import builtins
from ADC_function import *
from lxml.html import fromstring
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool
from difflib import SequenceMatcher
from unicodedata import category
@@ -13,7 +12,7 @@ from number_parser import is_uncensored
G_registered_storyline_site = {"airavwiki", "airav", "avno1", "xcity", "amazon", "58avgo"}
G_mode_txt = ('顺序执行','线程池','进程池')
G_mode_txt = ('顺序执行','线程池')
class noThread(object):
def map(self, fn, param):
@@ -25,14 +24,15 @@ class noThread(object):
# 获取剧情介绍 从列表中的站点同时查，取值优先级从前到后
def getStoryline(number, title, sites: list=None):
def getStoryline(number, title, sites: list=None, 无码=None):
start_time = time.time()
conf = config.getInstance()
if not conf.is_storyline():
return ''
debug = conf.debug() or conf.storyline_show() == 2
storyine_sites = conf.storyline_site().split(',') if sites is None else sites
if is_uncensored(number):
unc = 无码 if isinstance(无码, bool) else is_uncensored(number)
if unc:
storyine_sites += conf.storyline_uncensored_site().split(',')
else:
storyine_sites += conf.storyline_censored_site().split(',')
@@ -49,9 +49,8 @@ def getStoryline(number, title, sites: list=None):
cores = min(len(apply_sites), os.cpu_count())
if cores == 0:
return ''
run_mode = conf.storyline_mode()
assert run_mode in (0,1,2)
with ThreadPool(cores) if run_mode == 1 else Pool(cores) if run_mode == 2 else noThread() as pool:
run_mode = 1 if conf.storyline_mode() > 0 else 0
with ThreadPool(cores) if run_mode > 0 else noThread() as pool:
results = pool.map(getStoryline_mp, mp_args)
sel = ''
if not debug and conf.storyline_show() == 0:
@@ -62,7 +61,7 @@ def getStoryline(number, title, sites: list=None):
if not len(sel):
sel = value
return sel
# 以下debug结果输出会写入日志，进程池中的则不会，只在标准输出中显示
# 以下debug结果输出会写入日志
s = f'[!]Storyline{G_mode_txt[run_mode]}模式运行{len(apply_sites)}个任务共耗时(含启动开销){time.time() - start_time:.3f}秒，结束于{time.strftime("%H:%M:%S")}'
sel_site = ''
for site, desc in zip(apply_sites, results):
@@ -100,8 +99,7 @@ def getStoryline_mp(args):
storyline = getStoryline_58avgo(number, debug)
if not debug:
return storyline
# 进程池模式的子进程getStoryline_*()的print()不会写入日志中，线程池和顺序执行不受影响
print("[!]MP 进程[{}]运行{:.3f}秒，结束于{}返回结果: {}".format(
print("[!]MP 线程[{}]运行{:.3f}秒，结束于{}返回结果: {}".format(
site,
time.time() - start_time,
time.strftime("%H:%M:%S"),

2
WebCrawler/xcity.py
View File

@@ -128,7 +128,7 @@ def getOutline(html, number, title):
a = set(storyline_site) & {'airav', 'avno1'} # 只要中文的简介文字
if len(a):
site = [n for n in storyline_site if n in a]
g = getStoryline(number, title, site)
g = getStoryline(number, title, site, 无码=False)
if len(g):
return g
try:

26
config.py
View File

@@ -69,17 +69,17 @@ class Config:
elif (Path(__file__).resolve().parent / 'config.ini').is_file():
res_path = Path(__file__).resolve().parent / 'config.ini'
if res_path is None:
sys.exit(2)
os._exit(2)
ins = input("Or, Do you want me create a config file for you? (Yes/No)[Y]:")
if re.search('n', ins, re.I):
sys.exit(2)
os._exit(2)
# 用户目录才确定具有写权限，因此选择 ~/mdc.ini 作为配置文件生成路径，而不是有可能并没有写权限的
# 当前目录。目前版本也不再鼓励使用当前路径放置配置文件了，只是作为多配置文件的切换技巧保留。
write_path = path_search_order[2] # Path.home() / "mdc.ini"
write_path.write_text(res_path.read_text(encoding='utf-8'), encoding='utf-8')
print("Config file '{}' created.".format(write_path.resolve()))
input("Press Enter key exit...")
sys.exit(0)
os._exit(0)
# self.conf = self._default_config()
# try:
# self.conf = configparser.ConfigParser()
@@ -90,7 +90,7 @@ class Config:
# except Exception as e:
# print("[-]Config file not found! Use the default settings")
# print("[-]",e)
# sys.exit(3)
# os._exit(3)
# #self.conf = self._default_config()
def set_override(self, option_cmd: str):
@@ -110,7 +110,7 @@ class Config:
"""
def err_exit(str):
print(str)
sys.exit(2)
os._exit(2)
sections = self.conf.sections()
sec_name = None
@@ -365,25 +365,15 @@ class Config:
return "3:58avgo"
def storyline_show(self) -> int:
try:
v = self.conf.getint("storyline", "show_result")
v = self.conf.getint("storyline", "show_result", fallback=0)
return v if v in (0, 1, 2) else 2 if v > 2 else 0
except:
return 0
def storyline_mode(self) -> int:
try:
v = self.conf.getint("storyline", "run_mode")
return v if v in (0, 1, 2) else 2 if v > 2 else 0
except:
return 1
return 1 if self.conf.getint("storyline", "run_mode", fallback=1) > 0 else 0
def cc_convert_mode(self) -> int:
try:
v = self.conf.getint("cc_convert", "mode")
v = self.conf.getint("cc_convert", "mode", fallback=1)
return v if v in (0, 1, 2) else 2 if v > 2 else 0
except:
return 1
def cc_convert_vars(self) -> str:
return self.conf.get("cc_convert", "vars",

13
core.py
View File

@@ -1,5 +1,6 @@
import json
import os.path
import os
import pathlib
import re
import shutil
@@ -104,7 +105,7 @@ def create_folder(json_data): # 创建文件夹
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
sys.exit(0)
os._exit(0)
return os.path.normpath(path)
@@ -124,7 +125,7 @@ def download_file_with_filename(url, filename, path, filepath):
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
sys.exit(0)
os._exit(0)
proxies = configProxy.proxies()
headers = {
'User-Agent': G_USER_AGENT}
@@ -141,7 +142,7 @@ def download_file_with_filename(url, filename, path, filepath):
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
sys.exit(0)
os._exit(0)
headers = {
'User-Agent': G_USER_AGENT}
r = requests.get(url, timeout=configProxy.timeout, headers=headers)
@@ -292,7 +293,7 @@ def print_files(path, leak_word, c_word, naming_rule, part, cn_sub, json_data, f
os.makedirs(path)
except:
print(f"[-]Fatal error! can not make folder '{path}'")
sys.exit(0)
os._exit(0)
old_nfo = None
try:
@@ -724,10 +725,8 @@ def core_main(movie_path, number_th, oCC):
c_word = '-C' # 中文字幕影片后缀
# 判断是否无码
uncensored = 1 if is_uncensored(number) else 0
unce = json_data.get('无码')
if type(unce) is bool:
uncensored = 1 if unce else 0
uncensored = int(unce) if isinstance(unce, bool) else int(is_uncensored(number))
if '流出' in movie_path or 'uncensored' in movie_path.lower():
liuchu = '流出'