bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

javbus.py: 优化，修理无码片的导演、系列等字段

Browse Source
This commit is contained in:
lededev
2021-10-19 00:14:26 +08:00
parent dd106453f7
commit d80b2eeb7d
1 changed files with 80 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
WebCrawler/javbus.py
View File

@@ -3,81 +3,61 @@ sys.path.append('../')
import re import re
from pyquery import PyQuery as pq#need install from pyquery import PyQuery as pq#need install
from lxml import etree#need install from lxml import etree#need install
from bs4 import BeautifulSoup#need install
import json import json
from ADC_function import * from ADC_function import *
from WebCrawler.storyline import getStoryline from WebCrawler.storyline import getStoryline
import inspect import inspect
def getActorPhoto(htmlcode): #//*[@id="star_qdt"]/li/a/img def getActorPhoto(doc): #//*[@id="star_qdt"]/li/a/img
soup = BeautifulSoup(htmlcode, 'lxml') actors = doc('div.star-name a').items()
a = soup.find_all(attrs={'class': 'star-name'})
d={} d={}
for i in a: for i in actors:
l=i.a['href'] url=i.attr.href
t=i.get_text() t=i.attr.title
html = etree.fromstring(get_html(l), etree.HTMLParser()) html = etree.fromstring(get_html(url), etree.HTMLParser())
p=urljoin("https://www.javbus.com", p=urljoin("https://www.javbus.com",
str(html.xpath('//*[@id="waterfall"]/div[1]/div/div[1]/img/@src')).strip(" ['']")) str(html.xpath('//*[@id="waterfall"]/div[1]/div/div[1]/img/@src')).strip(" ['']"))
p2={t:p} p2={t:p}
d.update(p2) d.update(p2)
return d return d
def getTitle(htmlcode): #获取标题 def getTitle(html): #获取标题
doc = pq(htmlcode) title = str(html.xpath('/html/head/title/text()')[0])
title=str(doc('div.container h3').text()).replace(' ','-') title = str(re.findall('^.+?\s+(.*) - JavBus$', title)[0]).strip()
try: return title
title2 = re.sub('n\d+-','',title) def getStudioJa(html):
return title2 x = html.xpath('//span[contains(text(),"メーカー:")]/../a/text()')
except: return str(x[0]) if len(x) else ''
return title def getStudio(html): #获取厂商
def getStudio(htmlcode): #获取厂商 已修改 x = html.xpath('//span[contains(text(),"製作商:")]/../a/text()')
html = etree.fromstring(htmlcode,etree.HTMLParser()) return str(x[0]) if len(x) else ''
# 如果记录中冇导演，厂商排在第4位 def getYear(html): #获取年份
if '製作商:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/span/text()')).strip(" ['']"): result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[2]/text()')).strip(" ['']").strip()
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/a/text()')).strip(" ['']") return result[:4] if len(result)>=len('2000-01-01') else ''
# 如果记录中有导演，厂商排在第5位 def getCover(doc): #获取封面链接
elif '製作商:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[5]/span/text()')).strip(" ['']"):
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[5]/a/text()')).strip(" ['']")
else:
result = ''
return result
def getYear(htmlcode): #获取年份
html = etree.fromstring(htmlcode,etree.HTMLParser())
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[2]/text()')).strip(" ['']")
return result
def getCover(htmlcode): #获取封面链接
doc = pq(htmlcode)
image = doc('a.bigImage') image = doc('a.bigImage')
return urljoin("https://www.javbus.com", image.attr('href')) return urljoin("https://www.javbus.com", image.attr('href'))
def getRelease(htmlcode): #获取出版日期 def getRelease(html): #获取出版日期
html = etree.fromstring(htmlcode, etree.HTMLParser())
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[2]/text()')).strip(" ['']") result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[2]/text()')).strip(" ['']")
return result return result
def getRuntime(htmlcode): #获取分钟 已修改 def getRuntime(html): #获取分钟 已修改
html = etree.fromstring(htmlcode, etree.HTMLParser())
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[3]/text()')).strip(" ['']分鐘") result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[3]/text()')).strip(" ['']分鐘")
return result return result
def getActor(htmlcode): #获取女优 def getActor(doc): #获取女优
b=[] b=[]
soup=BeautifulSoup(htmlcode,'lxml') actors = doc('div.star-name a').items()
a=soup.find_all(attrs={'class':'star-name'}) for i in actors:
for i in a: b.append(i.attr.title)
b.append(i.get_text())
return b return b
def getNum(htmlcode): #获取番号 def getNum(html): #获取番号
html = etree.fromstring(htmlcode, etree.HTMLParser()) kwdlist = html.xpath('/html/head/meta[@name="keywords"]/@content')[0].split(',')
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[1]/span[2]/text()')).strip(" ['']") return kwdlist[0]
return result def getDirectorJa(html):
def getDirector(htmlcode): #获取导演 已修改 x = html.xpath('//span[contains(text(),"監督:")]/../a/text()')
html = etree.fromstring(htmlcode, etree.HTMLParser()) return str(x[0]) if len(x) else ''
if '導演:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/span/text()')).strip(" ['']"): def getDirector(html): #获取导演
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/a/text()')).strip(" ['']") x = html.xpath('//span[contains(text(),"導演:")]/../a/text()')
else: return str(x[0]) if len(x) else ''
result = '' # 记录中有可能没有导演数据 def getCID(html):
return result
def getCID(htmlcode):
html = etree.fromstring(htmlcode, etree.HTMLParser())
#print(htmlcode)
string = html.xpath("//a[contains(@class,'sample-box')][1]/@href")[0].replace('https://pics.dmm.co.jp/digital/video/','') string = html.xpath("//a[contains(@class,'sample-box')][1]/@href")[0].replace('https://pics.dmm.co.jp/digital/video/','')
result = re.sub('/.*?.jpg','',string) result = re.sub('/.*?.jpg','',string)
return result return result
@@ -94,27 +74,16 @@ def getOutline0(number): #获取剧情介绍 airav.wiki站点404，函数暂时
return '' return ''
def getOutline(number, title): #获取剧情介绍 多进程并发查询 def getOutline(number, title): #获取剧情介绍 多进程并发查询
return getStoryline(number,title) return getStoryline(number,title)
def getSerise(htmlcode): #获取系列 已修改 def getSeriseJa(html):
html = etree.fromstring(htmlcode, etree.HTMLParser()) x = html.xpath('//span[contains(text(),"シリーズ:")]/../a/text()')
# 如果记录中冇导演，系列排在第6位 return str(x[0]) if len(x) else ''
if '系列:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[6]/span/text()')).strip(" ['']"): def getSerise(html): #获取系列
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[6]/a/text()')).strip(" ['']") x = html.xpath('//span[contains(text(),"系列:")]/../a/text()')
# 如果记录中有导演，系列排在第7位 return str(x[0]) if len(x) else ''
elif '系列:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[7]/span/text()')).strip(" ['']"): def getTag(html): # 获取标签
result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[7]/a/text()')).strip(" ['']") klist = html.xpath('/html/head/meta[@name="keywords"]/@content')[0].split(',')
else: taglist = [translateTag_to_sc(v) for v in klist[1:]]
result = '' return taglist
return result
def getTag(htmlcode): # 获取标签
tag = []
soup = BeautifulSoup(htmlcode, 'lxml')
a = soup.find_all(attrs={'class': 'genre'})
for i in a:
if 'onmouseout' in str(i) or '多選提交' in str(i):
continue
tag.append(translateTag_to_sc(i.get_text()))
return tag
def getExtrafanart(htmlcode): # 获取剧照 def getExtrafanart(htmlcode): # 获取剧照
html_pather = re.compile(r'<div id=\"sample-waterfall\">[\s\S]*?</div></a>\s*?</div>') html_pather = re.compile(r'<div id=\"sample-waterfall\">[\s\S]*?</div></a>\s*?</div>')
html = html_pather.search(htmlcode) html = html_pather.search(htmlcode)
@@ -128,30 +97,30 @@ def getExtrafanart(htmlcode): # 获取剧照
def main_uncensored(number): def main_uncensored(number):
htmlcode = get_html('https://www.javbus.com/ja/' + number) htmlcode = get_html('https://www.javbus.com/ja/' + number)
if getTitle(htmlcode) == '':
htmlcode = get_html('https://www.javbus.com/ja/' + number.replace('-','_'))
if "<title>404 Page Not Found" in htmlcode: if "<title>404 Page Not Found" in htmlcode:
raise Exception('404 page not found') raise Exception('404 page not found')
title = str(re.sub('\w+-\d+-','',getTitle(htmlcode))).replace(getNum(htmlcode)+'-','') doc = pq(htmlcode)
lx = etree.fromstring(htmlcode, etree.HTMLParser())
title = getTitle(lx)
dic = { dic = {
'title': title, 'title': title,
'studio': getStudio(htmlcode), 'studio': getStudioJa(lx),
'year': getYear(htmlcode), 'year': getYear(lx),
'outline': getOutline(number, title), 'outline': getOutline(number, title),
'runtime': getRuntime(htmlcode), 'runtime': getRuntime(lx),
'director': getDirector(htmlcode), 'director': getDirectorJa(lx),
'actor': getActor(htmlcode), 'actor': getActor(doc),
'release': getRelease(htmlcode), 'release': getRelease(lx),
'number': getNum(htmlcode), 'number': getNum(lx),
'cover': getCover(htmlcode), 'cover': getCover(doc),
'tag': getTag(htmlcode), 'tag': getTag(lx),
'extrafanart': getExtrafanart(htmlcode), 'extrafanart': getExtrafanart(htmlcode),
'label': getSerise(htmlcode), 'label': getSeriseJa(lx),
'imagecut': 0, 'imagecut': 0,
'actor_photo': '', # 'actor_photo': '',
'website': 'https://www.javbus.com/ja/' + number, 'website': 'https://www.javbus.com/ja/' + number,
'source': 'javbus.py', 'source': 'javbus.py',
'series': getSerise(htmlcode), 'series': getSeriseJa(lx),
} }
js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), ) # .encode('UTF-8') js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), ) # .encode('UTF-8')
return js return js
@@ -166,26 +135,28 @@ def main(number):
htmlcode = get_html('https://www.javbus.com/' + number) htmlcode = get_html('https://www.javbus.com/' + number)
if "<title>404 Page Not Found" in htmlcode: if "<title>404 Page Not Found" in htmlcode:
raise Exception('404 page not found') raise Exception('404 page not found')
title = str(re.sub('\w+-\d+-', '', getTitle(htmlcode))) doc = pq(htmlcode)
lx = etree.fromstring(htmlcode,etree.HTMLParser())
title = getTitle(lx)
dic = { dic = {
'title': title, 'title': title,
'studio': getStudio(htmlcode), 'studio': getStudio(lx),
'year': str(re.search('\d{4}', getYear(htmlcode)).group()), 'year': getYear(lx),
'outline': getOutline(number, title), 'outline': getOutline(number, title),
'runtime': getRuntime(htmlcode), 'runtime': getRuntime(lx),
'director': getDirector(htmlcode), 'director': getDirector(lx),
'actor': getActor(htmlcode), 'actor': getActor(doc),
'release': getRelease(htmlcode), 'release': getRelease(lx),
'number': getNum(htmlcode), 'number': getNum(lx),
'cover': getCover(htmlcode), 'cover': getCover(doc),
'imagecut': 1, 'imagecut': 1,
'tag': getTag(htmlcode), 'tag': getTag(lx),
'extrafanart': getExtrafanart(htmlcode), 'extrafanart': getExtrafanart(htmlcode),
'label': getSerise(htmlcode), 'label': getSerise(lx),
# 'actor_photo': getActorPhoto(htmlcode), # 'actor_photo': getActorPhoto(doc),
'website': 'https://www.javbus.com/' + number, 'website': 'https://www.javbus.com/' + number,
'source': 'javbus.py', 'source': 'javbus.py',
'series': getSerise(htmlcode), 'series': getSerise(lx),
} }
js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4,separators=(',', ':'), ) # .encode('UTF-8') js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4,separators=(',', ':'), ) # .encode('UTF-8')
return js return js
@@ -206,8 +177,10 @@ if __name__ == "__main__" :
config.G_conf_override['debug_mode:switch'] = True config.G_conf_override['debug_mode:switch'] = True
print(main('ABP-888')) print(main('ABP-888'))
print(main('ABP-960')) print(main('ABP-960'))
# print(main('ADV-R0624')) # 404 print(main('ADV-R0624')) # 404
# print(main('MMNT-010')) print(main('MMNT-010'))
print(main('ipx-292')) print(main('ipx-292'))
print(main('CEMD-011')) print(main('CEMD-011'))
print(main('CJOD-278')) print(main('CJOD-278'))
print(main('100221_001'))
print(main('AVSW-061'))