bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix image download failed

Browse Source
This commit is contained in:
yoshiko2
2022-05-27 23:03:20 +08:00
parent bb6ff56ce5
commit 5ab121f996
2 changed files with 55 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ADC_function.py
View File

@@ -522,9 +522,9 @@ def download_one_file(args) -> str:
wrapped for map function wrapped for map function
""" """
(url, save_path, json_data) = args (url, save_path, json_headers) = args
if json_data != None: if json_headers != None:
filebytes = get_html(url, return_type='content', json_headers=json_data['headers']) filebytes = get_html(url, return_type='content', json_headers=json_headers['headers'])
else: else:
filebytes = get_html(url, return_type='content') filebytes = get_html(url, return_type='content')
if isinstance(filebytes, bytes) and len(filebytes): if isinstance(filebytes, bytes) and len(filebytes):
@@ -533,7 +533,7 @@ def download_one_file(args) -> str:
return str(save_path) return str(save_path)
def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel: int = 0, json_data=None): def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel: int = 0, json_headers=None):
""" """
download files in parallel 多线程下载文件 download files in parallel 多线程下载文件
@@ -552,7 +552,7 @@ def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel:
and fullpath and isinstance(fullpath, (str, Path)) and len(str(fullpath)): and fullpath and isinstance(fullpath, (str, Path)) and len(str(fullpath)):
fullpath = Path(fullpath) fullpath = Path(fullpath)
fullpath.parent.mkdir(parents=True, exist_ok=True) fullpath.parent.mkdir(parents=True, exist_ok=True)
mp_args.append((url, fullpath, json_data)) mp_args.append((url, fullpath, json_headers))
if not len(mp_args): if not len(mp_args):
return [] return []
if not isinstance(parallel, int) or parallel not in range(1, 200): if not isinstance(parallel, int) or parallel not in range(1, 200):

106
core.py
View File

@@ -71,11 +71,14 @@ def get_info(json_data): # 返回json里的数据
return title, studio, year, outline, runtime, director, actor_photo, release, number, cover, trailer, website, series, label return title, studio, year, outline, runtime, director, actor_photo, release, number, cover, trailer, website, series, label
def small_cover_check(path, filename, cover_small, movie_path, json_data=None): def small_cover_check(path, filename, cover_small, movie_path, json_headers=None):
full_filepath = Path(path) / filename full_filepath = Path(path) / filename
if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(str(full_filepath)): if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(str(full_filepath)):
return return
download_file_with_filename(cover_small, filename, path, movie_path, json_data) if json_headers != None:
download_file_with_filename(cover_small, filename, path, movie_path, json_headers['headers'])
else:
download_file_with_filename(cover_small, filename, path, movie_path)
print('[+]Image Downloaded! ' + full_filepath.name) print('[+]Image Downloaded! ' + full_filepath.name)
@@ -119,57 +122,36 @@ def download_file_with_filename(url, filename, path, filepath, json_headers=None
for i in range(configProxy.retry): for i in range(configProxy.retry):
try: try:
if configProxy.enable: if not os.path.exists(path):
if not os.path.exists(path): try:
try: os.makedirs(path)
os.makedirs(path) except:
except: print(f"[-]Fatal error! Can not make folder '{path}'")
print(f"[-]Fatal error! Can not make folder '{path}'") os._exit(0)
os._exit(0) headers = {'User-Agent': G_USER_AGENT}
if not json_headers == None:
if 'headers' in json_headers:
headers.update(json_headers)
if configProxy:
proxies = configProxy.proxies() proxies = configProxy.proxies()
headers = {'User-Agent': G_USER_AGENT}
if json_headers != None:
headers.update(json_headers)
r = requests.get(url, headers=headers, timeout=configProxy.timeout, proxies=proxies) r = requests.get(url, headers=headers, timeout=configProxy.timeout, proxies=proxies)
if r == '':
print('[-]Movie Download Data not found!')
return
with open(os.path.join(path, filename), "wb") as code:
code.write(r.content)
return
else: else:
if not os.path.exists(path): r = requests.get(url, headers=headers, timeout=configProxy.timeout)
try: if r == '':
os.makedirs(path) print('[-]Movie Download Data not found!')
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
headers = {'User-Agent': G_USER_AGENT}
if json_headers != None:
headers.update(json_headers)
r = requests.get(url, timeout=configProxy.timeout, headers=headers)
if r == '':
print('[-]Movie Download Data not found!')
return
with open(os.path.join(path, filename), "wb") as code:
code.write(r.content)
return return
except requests.exceptions.RequestException: with open(os.path.join(path, filename), "wb") as code:
i += 1 code.write(r.content)
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry)) return
except requests.exceptions.ConnectionError:
i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ProxyError: except requests.exceptions.ProxyError:
i += 1 i += 1
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry)) print('[-]Image Download : Proxy error ' + str(i) + '/' + str(configProxy.retry))
except requests.exceptions.ConnectTimeout: # except IOError:
i += 1 # print(f"[-]Create Directory '{path}' failed!")
print('[-]Image Download : Connect retry ' + str(i) + '/' + str(configProxy.retry)) # moveFailedFolder(filepath)
except IOError: # return
print(f"[-]Create Directory '{path}' failed!") except Exception as e:
moveFailedFolder(filepath) print('[-]Image Download :Error',e)
return
print('[-]Connect Failed! Please check your Proxy or Network!') print('[-]Connect Failed! Please check your Proxy or Network!')
moveFailedFolder(filepath) moveFailedFolder(filepath)
return return
@@ -302,12 +284,12 @@ def image_ext(url):
# 封面是否下载成功，否则移动到failed # 封面是否下载成功，否则移动到failed
def image_download(cover, fanart_path, thumb_path, path, filepath, json_data): def image_download(cover, fanart_path, thumb_path, path, filepath, json_headers=None):
full_filepath = os.path.join(path, fanart_path) full_filepath = os.path.join(path, fanart_path)
if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(full_filepath): if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(full_filepath):
return return
if "headers" in json_data: if json_headers != None:
if download_file_with_filename(cover, fanart_path, path, filepath, json_data['headers']) == 'failed': if download_file_with_filename(cover, fanart_path, path, filepath, json_headers['headers']) == 'failed':
moveFailedFolder(filepath) moveFailedFolder(filepath)
return return
else: else:
@@ -319,8 +301,8 @@ def image_download(cover, fanart_path, thumb_path, path, filepath, json_data):
for i in range(configProxy.retry): for i in range(configProxy.retry):
if file_not_exist_or_empty(full_filepath): if file_not_exist_or_empty(full_filepath):
print('[!]Image Download Failed! Trying again. [{}/3]', i + 1) print('[!]Image Download Failed! Trying again. [{}/3]', i + 1)
if "headers" in json_data: if json_headers != None:
download_file_with_filename(cover, fanart_path, path, filepath, json_data['headers']) download_file_with_filename(cover, fanart_path, path, filepath, json_headers['headers'])
else: else:
download_file_with_filename(cover, fanart_path, path, filepath) download_file_with_filename(cover, fanart_path, path, filepath)
continue continue
@@ -863,10 +845,16 @@ def core_main(movie_path, number_th, oCC):
# 检查小封面, 如果image cut为3，则下载小封面 # 检查小封面, 如果image cut为3，则下载小封面
if imagecut == 3: if imagecut == 3:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data) if 'headers' in json_data:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data['headers'])
else:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path)
# creatFolder会返回番号路径 # creatFolder会返回番号路径
image_download( cover, fanart_path,thumb_path, path, movie_path, json_data) if 'headers' in json_data:
image_download(cover, fanart_path, thumb_path, path, movie_path, json_data['headers'])
else:
image_download(cover, fanart_path, thumb_path, path, movie_path)
if not multi_part or part.lower() == '-cd1': if not multi_part or part.lower() == '-cd1':
try: try:
@@ -917,10 +905,16 @@ def core_main(movie_path, number_th, oCC):
# 检查小封面, 如果image cut为3，则下载小封面 # 检查小封面, 如果image cut为3，则下载小封面
if imagecut == 3: if imagecut == 3:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data) if 'headers' in json_data:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data['headers'])
else:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path)
# creatFolder会返回番号路径 # creatFolder会返回番号路径
image_download( cover, fanart_path, thumb_path, path, movie_path) if 'headers' in json_data:
image_download(cover, fanart_path, thumb_path, path, movie_path, json_data['headers'])
else:
image_download(cover, fanart_path, thumb_path, path, movie_path)
if not multi_part or part.lower() == '-cd1': if not multi_part or part.lower() == '-cd1':
try: try: