diff --git a/WebCrawler/__init__.py b/WebCrawler/__init__.py index 8c3516f..cc4c73c 100644 --- a/WebCrawler/__init__.py +++ b/WebCrawler/__init__.py @@ -86,6 +86,8 @@ def get_data_from_json(file_number, oCC): sources = insert(sources,"carib") elif "item" in file_number: sources = insert(sources,"getchu") + elif "rj" in lo_file_number or "vj" in lo_file_number or re.match(r"[\u4e00-\u9fa5]", file_number): + sources = insert(sources,"dlsite") elif re.match(r"^\d{5,}", file_number) or "heyzo" in lo_file_number: if "avsox" in sources: sources = insert(sources,"avsox") @@ -97,10 +99,6 @@ def get_data_from_json(file_number, oCC): sources = insert(sources,"fc2") elif "gcolle" in sources and (re.search("\d{6}", file_number)): sources = insert(sources,"gcolle") - elif "dlsite" in sources and ( - "rj" in lo_file_number or "vj" in lo_file_number - ): - sources = insert(sources,"dlsite") elif re.match(r"^[a-z0-9]{3,}$", lo_file_number): if "xcity" in sources: sources = insert(sources,"xcity") @@ -170,8 +168,12 @@ def get_data_from_json(file_number, oCC): # 然而也可以跟进关注其它命名规则例如airav.wiki Domain Creation Date: 2019-08-28T07:18:42.0Z # 如果将来javdb.com命名规则下不同Studio出现同名碰撞导致无法区分,可考虑更换规则,更新相应的number分析和抓取代码。 if str(json_data.get('number')).upper() != file_number.upper(): - print('[-]Movie number has changed! [{}]->[{}]'.format(file_number, str(json_data.get('number')))) - return None + try: + if json_data.get('allow_number_change'): + pass + except: + print('[-]Movie number has changed! [{}]->[{}]'.format(file_number, str(json_data.get('number')))) + return None # ================================================网站规则添加结束================================================ diff --git a/WebCrawler/dlsite.py b/WebCrawler/dlsite.py index dc2c570..9340465 100644 --- a/WebCrawler/dlsite.py +++ b/WebCrawler/dlsite.py @@ -26,7 +26,7 @@ def getActorPhoto(actor): #//*[@id="star_qdt"]/li/a/img def getStudio(html): try: try: - result = html.xpath('//th[contains(text(),"系列名")]/../td/span[1]/a/text()')[0] + result = html.xpath('//th[contains(text(),"商标名")]/../td/span[1]/a/text()')[0] except: result = html.xpath('//th[contains(text(),"社团名")]/../td/span[1]/a/text()')[0] except: @@ -101,14 +101,29 @@ def getSeries(html): except: result = '' return result +# +def getExtrafanart(html): + try: + result = [] + for i in html.xpath('//*[@id="work_left"]/div/div/div[1]/div/@data-src'): + result.append("https:" + i) + except: + result = '' + return result def main(number): try: - number = number.upper() - htmlcode = get_html('https://www.dlsite.com/maniax/work/=/product_id/' + number + '.html/?locale=zh_CN', - cookies={'locale': 'zh-cn'}) - html = etree.fromstring(htmlcode, etree.HTMLParser()) + if "RJ" in number or "VJ" in number: + number = number.upper() + htmlcode = get_html('https://www.dlsite.com/maniax/work/=/product_id/' + number + '.html/?locale=zh_CN',cookies={'locale': 'zh-cn'}) + html = etree.fromstring(htmlcode, etree.HTMLParser()) + else: + htmlcode = get_html(f'https://www.dlsite.com/maniax/fsr/=/language/jp/sex_category/male/keyword/{number}/order/trend/work_type_category/movie',cookies={'locale': 'zh-cn'}) + html = etree.HTML(htmlcode) + a = html.xpath('//*[@id="search_result_img_box"]/li[1]/dl/dd[2]/div[2]/a/@href')[0] + html = etree.HTML(get_html(a,cookies={'locale': 'zh-cn'})) + number = str(re.findall("\wJ\w+",a)).strip(" [']") dic = { - 'actor': getActor(html), + 'actor': getStudio(html), 'title': getTitle(html), 'studio': getStudio(html), 'outline': getOutline(html), @@ -118,7 +133,7 @@ def main(number): 'number': number, 'cover': 'https:' + getCover(html), 'cover_small': '', - 'imagecut': 0, + 'imagecut': 1, 'tag': getTag(html), 'label': getLabel(html), 'year': getYear(getRelease(html)), # str(re.search('\d{4}',getRelease(a)).group()), @@ -126,6 +141,8 @@ def main(number): 'website': 'https://www.dlsite.com/maniax/work/=/product_id/' + number + '.html', 'source': 'dlsite.py', 'series': getSeries(html), + 'extrafanart':getExtrafanart(html), + 'allow_number_change':True, } js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), ) # .encode('UTF-8') return js @@ -144,5 +161,5 @@ def main(number): # input("[+][+]Press enter key exit, you can check the error messge before you exit.\n[+][+]按回车键结束,你可以在结束之前查看和错误信息。") if __name__ == "__main__": config.getInstance().set_override("debug_mode:switch=1") - print(main('VJ013178')) + print(main('牝教師4~穢された教壇~ 「生意気ドジっ娘女教師・美結~高飛車ハメ堕ち2濁金」')) print(main('RJ329607')) diff --git a/core.py b/core.py index e64c2f5..a11cef2 100644 --- a/core.py +++ b/core.py @@ -698,14 +698,14 @@ def debug_print(data: json): print("[+] ------- DEBUG INFO -------") for i, v in data.items(): if i == 'outline': - print('[+] -', "%-14s" % i, ':', len(v), 'characters') + print('[+] -', "%-19s" % i, ':', len(v), 'characters') continue if i == 'actor_photo' or i == 'year': continue if i == 'extrafanart': - print('[+] -', "%-14s" % i, ':', len(v), 'links') + print('[+] -', "%-19s" % i, ':', len(v), 'links') continue - print(f'[+] - {i:<{cnspace(i,14)}} : {v}') + print(f'[+] - {i:<{cnspace(i,19)}} : {v}') print("[+] ------- DEBUG INFO -------") except: diff --git a/number_parser.py b/number_parser.py index b51e89c..976d1c4 100755 --- a/number_parser.py +++ b/number_parser.py @@ -18,6 +18,8 @@ def get_number(debug: bool, file_path: str) -> str: 'snis-829' >>> get_number(False, "/Users/Guest/AV_Data_Capture/snis-829-C.mp4") 'snis-829' + >>> get_number(False, "/Users/Guest/AV_Data_Capture/[脸肿字幕组][PoRO]牝教師4~穢された教壇~ 「生意気ドジっ娘女教師・美結~高飛車ハメ堕ち2濁金」[720p][x264_aac].mp4") + '牝教師4~穢された教壇~ 「生意気ドジっ娘女教師・美結~高飛車ハメ堕ち2濁金」' >>> get_number(False, "C:¥Users¥Guest¥snis-829.mp4") 'snis-829' >>> get_number(False, "C:¥Users¥Guest¥snis-829-C.mp4") @@ -41,6 +43,11 @@ def get_number(debug: bool, file_path: str) -> str: file_number = get_number_by_dict(filepath) if file_number: return file_number + elif '字幕组' in filepath or 'SUB' in filepath.upper(): + filepath = G_spat.sub("", filepath) + filepath = re.sub("\[.*?\]","",filepath) + file_number = str(re.findall(r'(.+?)\.', filepath)).strip(" [']") + return file_number elif '-' in filepath or '_' in filepath: # 普通提取番号 主要处理包含减号-和_的番号 filepath = G_spat.sub("", filepath) filename = str(re.sub("\[\d{4}-\d{1,2}-\d{1,2}\] - ", "", filepath)) # 去除文件名中时间