bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into dev

Browse Source
This commit is contained in:
Mathhew
2022-06-13 09:04:26 +08:00
parent f11378186d 8a342af11b
commit 8348fa167b
4 changed files with 108 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
ADC_function.py
View File

@@ -470,15 +470,12 @@ def download_file_with_filename(url: str, filename: str, path: str) -> None:
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
proxies = configProxy.proxies()
headers = {
'User-Agent': G_USER_AGENT}
r = requests.get(url, headers=headers, timeout=configProxy.timeout, proxies=proxies)
r = get_html(url=url, return_type='content')
if r == '':
print('[-]Movie Download Data not found!')
return
with open(os.path.join(path, filename), "wb") as code:
code.write(r.content)
code.write(r)
return
else:
if not os.path.exists(path):
@@ -487,14 +484,12 @@ def download_file_with_filename(url: str, filename: str, path: str) -> None:
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
headers = {
'User-Agent': G_USER_AGENT}
r = requests.get(url, timeout=configProxy.timeout, headers=headers)
r = get_html(url=url, return_type='content')
if r == '':
print('[-]Movie Download Data not found!')
return
with open(os.path.join(path, filename), "wb") as code:
code.write(r.content)
code.write(r)
return
except requests.exceptions.RequestException:
i += 1
@@ -522,15 +517,18 @@ def download_one_file(args) -> str:
wrapped for map function
"""
(url, save_path, json_data) = args
filebytes = get_html(url, return_type='content', json_headers=json_data['headers'])
(url, save_path, json_headers) = args
if json_headers != None:
filebytes = get_html(url, return_type='content', json_headers=json_headers['headers'])
else:
filebytes = get_html(url, return_type='content')
if isinstance(filebytes, bytes) and len(filebytes):
with save_path.open('wb') as fpbyte:
if len(filebytes) == fpbyte.write(filebytes):
return str(save_path)
def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel: int = 0, json_data=None):
def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel: int = 0, json_headers=None):
"""
download files in parallel 多线程下载文件
@@ -549,7 +547,7 @@ def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel:
and fullpath and isinstance(fullpath, (str, Path)) and len(str(fullpath)):
fullpath = Path(fullpath)
fullpath.parent.mkdir(parents=True, exist_ok=True)
mp_args.append((url, fullpath, json_data))
mp_args.append((url, fullpath, json_headers))
if not len(mp_args):
return []
if not isinstance(parallel, int) or parallel not in range(1, 200):

3
Movie_Data_Capture.py
View File

@@ -591,6 +591,7 @@ def main(args: tuple) -> Path:
print("[!] " + "Mapping Table Download FAILED".center(47))
print("[!] " + "无法连接github".center(47))
print("[!] " + "请过几小时再试试".center(47))
print("[!]", e)
print("[-] " + "------ AUTO EXIT AFTER 30s !!! ------ ".center(47))
time.sleep(30)
os._exit(-1)
@@ -677,7 +678,7 @@ def period(delta, pattern):
if __name__ == '__main__':
version = '6.2.1'
version = '6.2.2'
urllib3.disable_warnings() # Ignore http proxy warning
app_start = time.time()

96
core.py
View File

@@ -72,11 +72,14 @@ def get_info(json_data): # 返回json里的数据
return title, studio, year, outline, runtime, director, actor_photo, release, number, cover, trailer, website, series, label
def small_cover_check(path, filename, cover_small, movie_path, json_data=None):
def small_cover_check(path, filename, cover_small, movie_path, json_headers=None):
full_filepath = Path(path) / filename
if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(str(full_filepath)):
return
download_file_with_filename(cover_small, filename, path, movie_path, json_data)
if json_headers != None:
download_file_with_filename(cover_small, filename, path, movie_path, json_headers['headers'])
else:
download_file_with_filename(cover_small, filename, path, movie_path)
print('[+]Image Downloaded! ' + full_filepath.name)
@@ -120,57 +123,28 @@ def download_file_with_filename(url, filename, path, filepath, json_headers=None
for i in range(configProxy.retry):
try:
if configProxy.enable:
if not os.path.exists(path):
try:
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
proxies = configProxy.proxies()
headers = {'User-Agent': G_USER_AGENT}
if json_headers != None:
headers.update(json_headers)
r = requests.get(url, headers=headers, timeout=configProxy.timeout, proxies=proxies)
r = get_html(url=url,return_type='content',json_headers=json_headers)
if r == '':
print('[-]Movie Download Data not found!')
return
with open(os.path.join(path, filename), "wb") as code:
code.write(r.content)
code.write(r)
return
else:
if not os.path.exists(path):
try:
os.makedirs(path)
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
headers = {'User-Agent': G_USER_AGENT}
if json_headers != None:
headers.update(json_headers)
r = requests.get(url, timeout=configProxy.timeout, headers=headers)
if r == '':
print('[-]Movie Download Data not found!')
return
with open(os.path.join(path, filename), "wb") as code:
code.write(r.content)
return
except requests.exceptions.RequestException:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ConnectionError:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ProxyError:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ConnectTimeout:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except IOError:
print(f"[-]Create Directory '{path}' failed!")
moveFailedFolder(filepath)
return
print('[-]Image Download : Proxy error ' + str(i) + '/' + str(configProxy.retry))
# except IOError:
# print(f"[-]Create Directory '{path}' failed!")
# moveFailedFolder(filepath)
# return
except Exception as e:
print('[-]Image Download :Error',e)
print('[-]Connect Failed! Please check your Proxy or Network!')
moveFailedFolder(filepath)
return
@@ -303,12 +277,12 @@ def image_ext(url):
# 封面是否下载成功，否则移动到failed
def image_download(cover, fanart_path, thumb_path, path, filepath, json_data):
def image_download(cover, fanart_path, thumb_path, path, filepath, json_headers=None):
full_filepath = os.path.join(path, fanart_path)
if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(full_filepath):
return
if "headers" in json_data:
if download_file_with_filename(cover, fanart_path, path, filepath, json_data['headers']) == 'failed':
if json_headers != None:
if download_file_with_filename(cover, fanart_path, path, filepath, json_headers['headers']) == 'failed':
moveFailedFolder(filepath)
return
else:
@@ -320,8 +294,8 @@ def image_download(cover, fanart_path, thumb_path, path, filepath, json_data):
for i in range(configProxy.retry):
if file_not_exist_or_empty(full_filepath):
print('[!]Image Download Failed! Trying again. [{}/3]', i + 1)
if "headers" in json_data:
download_file_with_filename(cover, fanart_path, path, filepath, json_data['headers'])
if json_headers != None:
download_file_with_filename(cover, fanart_path, path, filepath, json_headers['headers'])
else:
download_file_with_filename(cover, fanart_path, path, filepath)
continue
@@ -364,8 +338,10 @@ def print_files(path, leak_word, c_word, naming_rule, part, cn_sub, json_data, f
print(" <sorttitle><![CDATA[" + naming_rule + "]]></sorttitle>", file=code)
print(" <customrating>JP-18+</customrating>", file=code)
print(" <mpaa>JP-18+</mpaa>", file=code)
print(" <set>", file=code)
print(" </set>", file=code)
try:
print(" <set>" + series + "</set>", file=code)
except:
print(" <set></set>", file=code)
print(" <studio>" + studio + "</studio>", file=code)
print(" <year>" + year + "</year>", file=code)
print(" <outline><![CDATA[" + outline + "]]></outline>", file=code)
@@ -399,7 +375,7 @@ def print_files(path, leak_word, c_word, naming_rule, part, cn_sub, json_data, f
try:
for i in tag:
print(" <tag>" + i + "</tag>", file=code)
print(" <tag>" + series + "</tag>", file=code)
# print(" <tag>" + series + "</tag>", file=code)
except:
aaaaa = ''
if cn_sub == '1':
@@ -413,7 +389,7 @@ def print_files(path, leak_word, c_word, naming_rule, part, cn_sub, json_data, f
try:
for i in tag:
print(" <genre>" + i + "</genre>", file=code)
print(" <genre>" + series + "</genre>", file=code)
# print(" <genre>" + series + "</genre>", file=code)
except:
aaaaaaaa = ''
print(" <num>" + number + "</num>", file=code)
@@ -864,10 +840,16 @@ def core_main(movie_path, number_th, oCC):
# 检查小封面, 如果image cut为3，则下载小封面
if imagecut == 3:
if 'headers' in json_data:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data)
else:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path)
# creatFolder会返回番号路径
image_download( cover, fanart_path,thumb_path, path, movie_path, json_data)
if 'headers' in json_data:
image_download(cover, fanart_path, thumb_path, path, movie_path, json_data)
else:
image_download(cover, fanart_path, thumb_path, path, movie_path)
if not multi_part or part.lower() == '-cd1':
try:
@@ -877,7 +859,10 @@ def core_main(movie_path, number_th, oCC):
# 下载剧照 data, path, filepath
if conf.is_extrafanart() and json_data.get('extrafanart'):
if 'headers' in json_data:
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path, json_data)
else:
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path)
# 下载演员头像 KODI .actors 目录位置
if conf.download_actor_photo_for_kodi():
@@ -918,10 +903,16 @@ def core_main(movie_path, number_th, oCC):
# 检查小封面, 如果image cut为3，则下载小封面
if imagecut == 3:
if 'headers' in json_data:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data)
else:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path)
# creatFolder会返回番号路径
image_download( cover, fanart_path, thumb_path, path, movie_path)
if 'headers' in json_data:
image_download(cover, fanart_path, thumb_path, path, movie_path, json_data)
else:
image_download(cover, fanart_path, thumb_path, path, movie_path)
if not multi_part or part.lower() == '-cd1':
try:
@@ -931,7 +922,10 @@ def core_main(movie_path, number_th, oCC):
# 下载剧照 data, path, filepath
if conf.is_extrafanart() and json_data.get('extrafanart'):
if 'headers' in json_data:
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path, json_data)
else:
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path)
# 下载演员头像 KODI .actors 目录位置
if conf.download_actor_photo_for_kodi():

37
number_parser.py
View File

@@ -80,6 +80,32 @@ def get_number(debug: bool, file_path: str) -> str:
print(f'[-]Number Parser exception: {e} [{file_path}]')
return None
# modou提取number
def md(filename):
m = re.search(r'(md[a-z]{0,2}-?)(\d{2,})(-ep\d*)*', filename, re.I)
return f'{m.group(1).replace("-","").upper()}{m.group(2).zfill(4)}{m.group(3) or ""}'
def mmz(filename):
m = re.search(r'(mmz-?)(\d{2,})(-ep\d*)*', filename, re.I)
return f'{m.group(1).replace("-","").upper()}{m.group(2).zfill(3)}{m.group(3) or ""}'
def msd(filename):
m = re.search(r'(msd-?)(\d{2,})(-ep\d*)*', filename, re.I)
return f'{m.group(1).replace("-","").upper()}{m.group(2).zfill(3)}{m.group(3) or ""}'
def mky(filename):
m = re.search(r'(mky-[a-z]{2,2}-?)(\d{2,})(-ep\d*)*', filename, re.I)
return f'{m.group(1).replace("-","").upper()}{m.group(2).zfill(3)}{m.group(3) or ""}'
def yk(filename):
m = re.search(r'(yk-?)(\d{2,})(-ep\d*)*', filename, re.I)
return f'{m.group(1).replace("-","").upper()}{m.group(2).zfill(3)}{m.group(3) or ""}'
def pm(filename):
m = re.search(r'(pm[a-z]?-?)(\d{2,})(-ep\d*)*', filename, re.I)
return f'{m.group(1).replace("-","").upper()}{m.group(2).zfill(3)}{m.group(3) or ""}'
# 按javdb数据源的命名规范提取number
G_TAKE_NUM_RULES = {
@@ -90,7 +116,13 @@ G_TAKE_NUM_RULES = {
'x-art': lambda x: str(re.search(r'x-art\.\d{2}\.\d{2}\.\d{2}', x, re.I).group()),
'xxx-av': lambda x: ''.join(['xxx-av-', re.findall(r'xxx-av[^\d]*(\d{3,5})[^\d]*', x, re.I)[0]]),
'heydouga': lambda x: 'heydouga-' + '-'.join(re.findall(r'(\d{4})[\-_](\d{3,4})[^\d]*', x, re.I)[0]),
'heyzo': lambda x: 'HEYZO-' + re.findall(r'heyzo[^\d]*(\d{4})', x, re.I)[0]
'heyzo': lambda x: 'HEYZO-' + re.findall(r'heyzo[^\d]*(\d{4})', x, re.I)[0],
r'\bmd[a-z]{0,2}-\d{2,}': md,
r'\bmmz-\d{2,}':mmz,
r'\bmsd-\d{2,}':msd,
r'\bmky-[a-z]{2,2}-\d{2,}':mky,
r'\byk-\d{2,3}': yk,
r'\bpm[a-z]?-?\d{2,}':pm
}
@@ -176,6 +208,9 @@ if __name__ == "__main__":
"rctd-461CH-CD2.mp4", # ch后可加CDn
"rctd-461-Cd3-C.mp4", # CDn后可加-C
"rctd-461-C-cD4.mp4", # cD1 Cd1 cd1 CD1 最终生成.nfo时统一为大写CD1
"MD-123.ts",
"MDSR-0001-ep2.ts",
"MKY-NS-001.mp4"
)