bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scrapinglib

Browse Source 
- 优化提取extrafanart，trailer等，直接使用xpath expr，不需要正则匹配
- 优化 getchu 获取cover方法,直接使用og标签信息
- 优化 www.getchu 识别 getchu-id 的资源
- 统一获取 tag 方法,返回值 list
This commit is contained in:
Mathhew
2022-06-15 14:23:49 +08:00
parent eed33408a8
commit 0dda035057
16 changed files with 107 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scrapinglib/airav.py
View File

@@ -104,9 +104,6 @@ class Airav(Parser):
return result
return super().getCover(htmltree)
def getTags(self, htmltree):
return self.getTreeAll(htmltree, self.expr_tags)
def getSeries(self, htmltree):
result = self.javbus.get('series')
if isinstance(result, str) and len(result):

4
scrapinglib/avsox.py
View File

@@ -53,7 +53,7 @@ class Avsox(Parser):
return self.getTreeElement(self.searchtree, self.expr_smallcover)
def getTags(self, htmltree):
tags = super().getTags(htmltree).split(',')
tags = self.getTreeElement(htmltree).split(',')
return [i.strip() for i in tags[2:]] if len(tags) > 2 else []
def getOutline(self, htmltree):
@@ -70,7 +70,7 @@ class Avsox(Parser):
return d
def getActorPhoto(self, htmltree):
a = super().getActorPhoto(htmltree)
a = self.getTreeAll(htmltree, self.expr_actorphoto)
d = {}
for i in a:
l = i.find('.//img').attrib['src']

4
scrapinglib/carib.py
View File

@@ -47,9 +47,6 @@ class Carib(Parser):
def getCover(self, htmltree):
return f'https://www.caribbeancom.com/moviepages/{self.number}/images/l_l.jpg'
def getTags(self, htmltree):
return self.getTreeAll(htmltree, self.expr_tags)
def getExtrafanart(self, htmltree):
r = []
genres = self.getTreeAll(htmltree, self.expr_extrafanart)
@@ -62,7 +59,6 @@ class Carib(Parser):
return r
def getActorPhoto(self, htmltree):
# return super().getActorPhoto(htmltree)
htmla = htmltree.xpath("//*[@id='moviepages']/div[@class='container']/div[@class='inner-container']/div[@class='movie-info section']/ul/li[@class='movie-spec']/span[@class='spec-content']/a[@itemprop='actor']")
names = htmltree.xpath("//*[@id='moviepages']/div[@class='container']/div[@class='inner-container']/div[@class='movie-info section']/ul/li[@class='movie-spec']/span[@class='spec-content']/a[@itemprop='actor']/span[@itemprop='name']/text()")
t = {}

8
scrapinglib/dlsite.py
View File

@@ -74,9 +74,8 @@ class Dlsite(Parser):
def getOutline(self, htmltree):
total = []
result = self.getTreeAll(htmltree, self.expr_outline)
for i in result:
total.append(i.strip('\r\n'))
return str(total).strip(" ['']").replace("', '', '",r'\n').replace("', '",r'\n').strip(", '', '")
total = [ x.strip() for x in result if x.strip()]
return '\n'.join(total)
def getRelease(self, htmltree):
return super().getRelease(htmltree).replace('年','-').replace('月','-').replace('日','')
@@ -84,9 +83,6 @@ class Dlsite(Parser):
def getCover(self, htmltree):
return 'https:' + super().getCover(htmltree).replace('.webp', '.jpg')
def getTags(self, htmltree):
return self.getTreeAll(htmltree, self.expr_tags)
def getExtrafanart(self, htmltree):
try:
result = []

4
scrapinglib/fanza.py
View File

@@ -106,13 +106,13 @@ class Fanza(Parser):
return self.getFanzaStrings('ジャンル：')
def getLabel(self, htmltree):
ret = self.getFanzaStrings('レーベル')
ret = self.getFanzaString('レーベル')
if ret == "----":
return ''
return ret
def getSeries(self, htmltree):
ret = self.getFanzaStrings('シリーズ：')
ret = self.getFanzaString('シリーズ：')
if ret == "----":
return ''
return ret

12
scrapinglib/fc2.py
View File

@@ -18,6 +18,7 @@ class Fc2(Parser):
expr_director = '//*[@id="top"]/div[1]/section[1]/div/section/div[2]/ul/li[3]/a/text()'
expr_actor = '//*[@id="top"]/div[1]/section[1]/div/section/div[2]/ul/li[3]/a/text()'
expr_cover = "//div[@class='items_article_MainitemThumb']/span/img/@src"
expr_extrafanart = '//ul[@class="items_article_SampleImagesArea"]/li/a/@href'
expr_tags = "//a[@class='tag tagTag']/text()"
def search(self, number):
@@ -45,17 +46,6 @@ class Fc2(Parser):
def getCover(self, htmltree):
return urljoin('https://adult.contents.fc2.com', super().getCover(htmltree))
def getExtrafanart(self, htmltree):
html_pather = re.compile(r'<ul class=\"items_article_SampleImagesArea\"[\s\S]*?</ul>')
html = html_pather.search(self.htmlcode)
if html:
html = html.group()
extrafanart_pather = re.compile(r'<a href=\"(.*?)\"')
extrafanart_imgs = extrafanart_pather.findall(html)
if extrafanart_imgs:
return extrafanart_imgs
return ''
def getTrailer(self, htmltree):
video_pather = re.compile(r'\'[a-zA-Z0-9]{32}\'')
video = video_pather.findall(self.htmlcode)

3
scrapinglib/gcolle.py
View File

@@ -58,9 +58,6 @@ class Gcolle(Parser):
def getCover(self, htmltree):
return "https:" + super().getCover(htmltree)
def getTags(self, htmltree):
return self.getTreeAll(htmltree, self.expr_tags)
def getExtrafanart(self, htmltree):
extrafanart = self.getTreeAll(htmltree, self.expr_extrafanart)
if len(extrafanart) == 0:

19
scrapinglib/getchu.py
View File

@@ -35,7 +35,7 @@ class wwwGetchu(Parser):
GETCHU_WWW_SEARCH_URL = 'http://www.getchu.com/php/search.phtml?genre=anime_dvd&search_keyword=_WORD_&check_key_dtl=1&submit='
expr_title = '//*[@id="soft-title"]/text()'
expr_cover = "/html/body/div[1]/table[2]/tr[1]/td/a/@href"
expr_cover = '//head/meta[@property="og:image"]'
expr_director = "//td[contains(text(),'ブランド')]/following-sibling::td/a[1]/text()"
expr_studio = "//td[contains(text(),'ブランド')]/following-sibling::td/a[1]/text()"
expr_actor = "//td[contains(text(),'ブランド')]/following-sibling::td/a[1]/text()"
@@ -47,8 +47,12 @@ class wwwGetchu(Parser):
expr_series = "//td[contains(text(),'ジャンル：')]/following-sibling::td/text()"
def queryNumberUrl(self, number):
self.number = quote(number, encoding="euc_jp")
queryUrl = self.GETCHU_WWW_SEARCH_URL.replace("_WORD_", self.number)
if 'GETCHU' in number.upper():
idn = re.findall('\d+',number)[0]
return "http://www.getchu.com/soft.phtml?id=" + idn
else:
self.number = quote(number, encoding="euc_jp")
queryUrl = self.GETCHU_WWW_SEARCH_URL.replace("_WORD_", self.number)
# NOTE dont know why will try 2 times
retry = 2
for i in range(retry):
@@ -64,14 +68,11 @@ class wwwGetchu(Parser):
return 'GETCHU-' + re.findall('\d+', self.detailurl.replace("http://www.getchu.com/soft.phtml?id=", ""))[0]
def getCover(self, htmltree):
return "http://www.getchu.com" + super().getCover(htmltree).replace("./", '/')
return self.getTreeElement(htmltree, self.expr_cover).get('content')
def getActors(self, htmltree):
return super().getDirector(htmltree)
def getTags(self, htmltree):
return self.getTreeAll(htmltree, self.expr_tags)
def getOutline(self, htmltree):
outline = ''
_list = self.getTreeAll(htmltree, self.expr_outline)
@@ -109,7 +110,6 @@ class dlGetchu(wwwGetchu):
GETCHU_DL_URL = 'https://dl.getchu.com/i/item_WORD_'
expr_title = "//div[contains(@style,'color: #333333; padding: 3px 0px 0px 5px;')]/text()"
expr_cover = "//td[contains(@bgcolor,'#ffffff')]/img/@src"
expr_director = "//td[contains(text(),'作者')]/following-sibling::td/text()"
expr_studio = "//td[contains(text(),'サークル')]/following-sibling::td/a/text()"
expr_label = "//td[contains(text(),'サークル')]/following-sibling::td/a/text()"
@@ -135,9 +135,6 @@ class dlGetchu(wwwGetchu):
def getNum(self, htmltree):
return 'GETCHU-' + re.findall('\d+', self.number)[0]
def getCover(self, htmltree):
return "https://dl.getchu.com" + super().getCover(htmltree)
def extradict(self, dic: dict):
return dic

43
scrapinglib/jav321.py
View File

@@ -12,15 +12,15 @@ class Jav321(Parser):
expr_title = "/html/body/div[2]/div[1]/div[1]/div[1]/h3/text()"
expr_cover = "/html/body/div[2]/div[2]/div[1]/p/a/img/@src"
expr_outline = "/html/body/div[2]/div[1]/div[1]/div[2]/div[3]/div/text()"
# NOTE: 统一使用 xpath
expr_number = '//b[contains(text(),"品番")]/following-sibling::node()'
expr_actor = '//b[contains(text(),"出演者")]/following-sibling::a[starts-with(@href,"/star")]'
expr_label = '//b[contains(text(),"メーカー")]/following-sibling::a[starts-with(@href,"/company")]'
expr_tags = '//b[contains(text(),"ジャンル")]/following-sibling::a[starts-with(@href,"/genre")]'
expr_studio = '//b[contains(text(),"メーカー")]/following-sibling::a[starts-with(@href,"/company")]'
expr_actor = '//b[contains(text(),"出演者")]/following-sibling::a[starts-with(@href,"/star")]/text()'
expr_label = '//b[contains(text(),"メーカー")]/following-sibling::a[starts-with(@href,"/company")]/text()'
expr_tags = '//b[contains(text(),"ジャンル")]/following-sibling::a[starts-with(@href,"/genre")]/text()'
expr_studio = '//b[contains(text(),"メーカー")]/following-sibling::a[starts-with(@href,"/company")]/text()'
expr_release = '//b[contains(text(),"配信開始日")]/following-sibling::node()'
expr_runtime = '//b[contains(text(),"収録時間")]/following-sibling::node()'
# expr_series = '//b[contains(text(),"シリーズ")]'
expr_series = '//b[contains(text(),"シリーズ")]/following-sibling::node()'
expr_extrafanart = '//div[@class="col-md-3"]/div[@class="col-xs-12 col-md-12"]/p/a/img/@src'
def queryNumberUrl(self, number):
return 'https://www.jav321.com/search'
@@ -45,39 +45,8 @@ class Jav321(Parser):
else:
return ''
def getExtrafanart(self, htmltree):
html_pather = re.compile(r'<div class=\"col\-md\-3\"><div class=\"col\-xs\-12 col\-md\-12\">[\s\S]*?</script><script async src=\"\/\/adserver\.juicyads\.com/js/jads\.js\">')
html = html_pather.search(self.detailhtml)
if html:
html = html.group()
extrafanart_pather = re.compile(r'<img.*?src=\"(.*?)\"')
extrafanart_imgs = extrafanart_pather.findall(html)
if extrafanart_imgs:
return extrafanart_imgs
return ''
def getRelease(self, htmltree):
return super().getRelease(htmltree).split(": ")[1]
def getRuntime(self, htmltree):
return super().getRuntime(htmltree).split(": ")[1]
def parseElement(self, all):
if all:
ret = []
for si in all:
ret.append(si.text)
return ",".join(ret)
return ''
def getActors(self, htmltree):
return self.parseElement(super().getActors(htmltree))
def getLabel(self, htmltree):
return self.parseElement(self.getTreeAll(htmltree, self.expr_label))
def getTags(self, htmltree):
return self.parseElement(self.getTreeAll(htmltree, self.expr_tags))
def getStudio(self, htmltree):
return self.parseElement(self.getTreeAll(htmltree, self.expr_studio))

19
scrapinglib/javbus.py
View File

@@ -26,6 +26,7 @@ class Javbus(Parser):
expr_runtime = '/html/body/div[5]/div[1]/div[2]/p[3]/text()'
expr_actor = '//div[@class="star-name"]/a'
expr_actorphoto = '//div[@class="star-name"]/../a/img'
expr_extrafanart = '//div[@id="sample-waterfall"]/a/@href'
expr_tags = '/html/head/meta[@name="keywords"]/@content'
expr_uncensored = '//*[@id="navbar"]/ul[1]/li[@class="active"]/a[contains(@href,"uncensored")]'
@@ -85,9 +86,6 @@ class Javbus(Parser):
def getCover(self, htmltree):
return urljoin("https://www.javbus.com", super().getCover(htmltree))
def getRelease(self, htmltree):
return super().getRelease(htmltree).strip(" ['']")
def getRuntime(self, htmltree):
return super().getRuntime(htmltree).strip(" ['']分鐘")
@@ -99,7 +97,7 @@ class Javbus(Parser):
return b
def getActorPhoto(self, htmltree):
actors = super().getActorPhoto(htmltree)
actors = self.getTreeAll(htmltree, self.expr_actorphoto)
d = {}
for i in actors:
p = i.attrib['src']
@@ -122,20 +120,9 @@ class Javbus(Parser):
return self.getTreeElement(htmltree, self.expr_series)
def getTags(self, htmltree):
tags = super().getTags(htmltree).split(',')
tags = self.getTreeElement(htmltree, self.expr_tags).split(',')
return tags[1:]
def getExtrafanart(self, htmltree):
html_pather = re.compile(r'<div id=\"sample-waterfall\">[\s\S]*?</div></a>\s*?</div>')
html = html_pather.search(self.htmlcode)
if html:
html = html.group()
extrafanart_pather = re.compile(r'<a class=\"sample-box\" href=\"(.*?)\"')
extrafanart_imgs = extrafanart_pather.findall(html)
if extrafanart_imgs:
return [urljoin('https://www.javbus.com',img) for img in extrafanart_imgs]
return ''
def getOutline(self, htmltree):
if self.morestoryline:
if any(caller for caller in inspect.stack() if os.path.basename(caller.filename) == 'airav.py'):

60
scrapinglib/javdb.py
View File

@@ -32,6 +32,7 @@ class Javdb(Parser):
expr_cover = "//div[contains(@class, 'column-video-cover')]/a/img/@src"
expr_cover2 = "//div[contains(@class, 'column-video-cover')]/img/@src"
expr_cover_no = '//*[contains(@class,"movie-list")]/div/a/div[contains(@class, "cover")]/img/@src'
expr_trailer = '//span[contains(text(),"預告片")]/../../video/source/@src'
expr_extrafanart = "//article[@class='message video-panel']/div[@class='message-body']/div[@class='tile-images preview-images']/a[contains(@href,'/samples/')]/@href"
expr_tags = '//strong[contains(text(),"類別")]/../span/a/text()'
expr_tags2 = '//strong[contains(text(),"類別")]/../span/text()'
@@ -105,9 +106,10 @@ class Javdb(Parser):
def getNum(self, htmltree):
if self.noauth:
return self.number
result1 = str(self.getTreeAll(htmltree, self.expr_number)).strip(" ['']")
result2 = str(self.getTreeAll(htmltree, self.expr_number2)).strip(" ['']")
dp_number = str(result2 + result1).strip('+')
# 番号被分割开，需要合并后才是完整番号
part1 = self.getTreeElement(htmltree, self.expr_number)
part2 = self.getTreeElement(htmltree, self.expr_number2)
dp_number = part2 + part1
# NOTE 检测匹配与更新 self.number
if dp_number.upper() != self.number.upper():
raise Exception(f'[!] {self.number}: find [{dp_number}] in javdb, not match')
@@ -131,28 +133,20 @@ class Javdb(Parser):
return self.getTreeElement(htmltree, self.expr_release_no, self.queryid).strip()
return super().getRelease(htmltree)
def getRuntime(self, htmltree):
result1 = str(self.getTreeAll(htmltree, self.expr_runtime)).strip(" ['']")
result2 = str(self.getTreeAll(htmltree, self.expr_runtime2)).strip(" ['']")
return str(result1 + result2).strip('+').rstrip('mi')
def getDirector(self, htmltree):
result1 = str(self.getTreeAll(htmltree, self.expr_director)).strip(" ['']")
result2 = str(self.getTreeAll(htmltree, self.expr_director2)).strip(" ['']")
return str(result1 + result2).strip('+').replace("', '", '').replace('"', '')
return self.getTreeElementbyExprs(htmltree, self.expr_director, self.expr_director2)
def getSeries(self, htmltree):
result1 = str(self.getTreeAll(htmltree, self.expr_series)).strip(" ['']")
result2 = str(self.getTreeAll(htmltree, self.expr_series2)).strip(" ['']")
result = str(result1 + result2).strip('+').replace("', '", '').replace('"', '')
# NOTE 不清楚javdb是否有一部影片多个series的情况，暂时保留
results = self.getTreeAllbyExprs(htmltree, self.expr_series, self.expr_series2)
result = ''.join(results)
if not result and self.fixstudio:
result = self.getStudio(htmltree)
return result
def getLabel(self, htmltree):
result1 = str(self.getTreeAll(htmltree, self.expr_label)).strip(" ['']")
result2 = str(self.getTreeAll(htmltree, self.expr_label2)).strip(" ['']")
result = str(result1 + result2).strip('+').replace("', '", '').replace('"', '')
results = self.getTreeAllbyExprs(htmltree, self.expr_label, self.expr_label2)
result = ''.join(results)
if not result and self.fixstudio:
result = self.getStudio(htmltree)
return result
@@ -182,38 +176,20 @@ class Javdb(Parser):
return getStoryline(self.number, self.getUncensored(htmltree))
return ''
def getStudio(self, htmltree):
try:
return self.getTreeAll(htmltree, self.expr_studio).strip(" ['']")
except:
pass
try:
return self.getTreeAll(htmltree, self.expr_studio2).strip(" ['']")
except:
return ''
def getTrailer(self, htmltree):
video_pather = re.compile(r'<video id\=\".*?>\s*?<source src=\"(.*?)\"')
video = video_pather.findall(self.deatilpage)
video = super().getTrailer(htmltree)
# 加上数组判空
if video and video[0] != "":
if not 'https:' in video[0]:
video_url = 'https:' + video[0]
if video:
if not 'https:' in video:
video_url = 'https:' + video
else:
video_url = video[0]
video_url = video
else:
video_url = ''
return video_url
def getTags(self, htmltree):
try:
return self.getTreeAll(htmltree, self.expr_tags)
except:
pass
try:
return self.getTreeAll(htmltree, self.expr_tags2)
except:
return ''
return self.getTreeAllbyExprs(htmltree, self.expr_tags, self.expr_tags2)
def getUserRating(self, htmltree):
try:

16
scrapinglib/mgstage.py
View File

@@ -16,7 +16,7 @@ class Mgstage(Parser):
expr_actor = '//th[contains(text(),"出演：")]/../td/a/text()'
expr_release = '//th[contains(text(),"配信開始日：")]/../td/a/text()'
expr_cover = '//*[@id="EnlargeImage"]/@href'
expr_label = '//th[contains(text(),"シリーズ：")]/../td/a/text()'
expr_label = '//th[contains(text(),"レーベル：")]/../td/a/text()'
expr_tags = '//th[contains(text(),"ジャンル：")]/../td/a/text()'
expr_tags2 = '//th[contains(text(),"ジャンル：")]/../td/text()'
expr_series = '//th[contains(text(),"シリーズ")]/../td/a/text()'
@@ -34,19 +34,17 @@ class Mgstage(Parser):
return super().getTitle(htmltree).replace('/', ',').strip()
def getTags(self, htmltree):
results = self.getTreeAll(htmltree, self.expr_tags)
results2 = self.getTreeAll(htmltree, self.expr_tags2)
return [ x.strip() for x in (results + results2) if x.strip()]
return self.getTreeAllbyExprs(htmltree, self.expr_tags, self.expr_tags2)
def getTreeAll(self, tree, expr):
alls = super().getTreeAll(tree, expr)
return [ x.strip() for x in alls ]
return [ x.strip() for x in alls if x.strip()]
def getTreeElement(self, tree, expr, index=0):
if expr == '':
return ''
result1 = getTreeElement(tree, expr).strip().replace("', '", '').strip(" ['']")
result2 = getTreeElement(tree, expr.replace('td/a/','td/')).strip().replace("', '", '').strip(" ['']")
result1 = ''.join(self.getTreeAll(tree, expr))
result2 = ''.join(self.getTreeAll(tree, expr.replace('td/a/','td/')))
if result1 == result2:
return str(result1).strip('+').replace("', '",'').replace('"','')
return str(result1 + result2).strip('+').replace("', '",'').replace('"','')
return result1
return result1 + result2

3
scrapinglib/mv91.py
View File

@@ -61,9 +61,6 @@ class Mv91(Parser):
def getStudio(self, htmltree):
return '91制片厂'
def getTags(self, htmltree):
return self.getTreeAll(htmltree, self.expr_tags)
def getActors(self, htmltree):
b=[]
for player in self.getTreeAll(htmltree, self.expr_actor):

110
scrapinglib/parser.py
View File

@@ -8,7 +8,8 @@ from . import httprequest
from .utils import getTreeElement, getTreeAll
class Parser:
""" 基础刮削类
"""
source = 'base'
# poster: `0` 复制 `1` 裁剪
imagecut = 1
@@ -139,7 +140,7 @@ class Parser:
except Exception as e:
print(e)
dic = {"title": ""}
js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), )
js = json.dumps(dic, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
return js
def extradict(self, dic:dict):
@@ -155,15 +156,8 @@ class Parser:
def getTitle(self, htmltree):
return self.getTreeElement(htmltree, self.expr_title).strip()
def getStudio(self, htmltree):
try:
return self.getTreeElement(htmltree, self.expr_studio).strip(" ['']")
except:
pass
try:
return self.getTreeElement(htmltree, self.expr_studio2).strip(" ['']")
except:
return ''
def getRelease(self, htmltree):
return self.getTreeElement(htmltree, self.expr_release).strip().replace('/','-')
def getYear(self, htmltree):
""" year基本都是从release中解析的
@@ -175,73 +169,46 @@ class Parser:
return release
def getRuntime(self, htmltree):
try:
return self.getTreeElement(htmltree, self.expr_runtime).strip("\n\t ['']").rstrip('mi')
except:
pass
try:
return self.getTreeElement(htmltree, self.expr_runtime2).strip("\n\t ['']").rstrip('mi')
except:
return ''
def getRelease(self, htmltree):
return self.getTreeElement(htmltree, self.expr_release).strip().replace('/','-')
return self.getTreeElementbyExprs(htmltree, self.expr_runtime, self.expr_runtime2).strip().rstrip('mi')
def getOutline(self, htmltree):
return self.getTreeElement(htmltree, self.expr_outline).strip().replace("\n","")
return self.getTreeElement(htmltree, self.expr_outline).strip()
def getDirector(self, htmltree):
return self.getTreeElement(htmltree, self.expr_director)
return self.getTreeElement(htmltree, self.expr_director).strip()
def getActors(self, htmltree):
def getActors(self, htmltree) -> list:
return self.getTreeAll(htmltree, self.expr_actor)
def getTags(self, htmltree):
return self.getTreeElement(htmltree, self.expr_tags)
def getTags(self, htmltree) -> list:
alls = self.getTreeAll(htmltree, self.expr_tags)
return [ x.strip() for x in alls if x.strip()]
def getStudio(self, htmltree):
return self.getTreeElementbyExprs(htmltree, self.expr_studio, self.expr_studio2)
def getLabel(self, htmltree):
try:
return self.getTreeElement(htmltree, self.expr_label).strip(" ['']")
except:
pass
try:
return self.getTreeElement(htmltree, self.expr_label2).strip(" ['']")
except:
return ''
return self.getTreeElementbyExprs(htmltree, self.expr_label, self.expr_label2)
def getSeries(self, htmltree):
try:
return self.getTreeElement(htmltree, self.expr_series).strip(" ['']")
except:
pass
try:
return self.getTreeElement(htmltree, self.expr_series2).strip(" ['']")
except:
return ''
return self.getTreeElementbyExprs(htmltree, self.expr_series, self.expr_series2)
def getCover(self, htmltree):
try:
return self.getTreeElement(htmltree, self.expr_cover).strip(" ['']")
except:
pass
try:
return self.getTreeElement(htmltree, self.expr_cover2).strip(" ['']")
except:
return ''
return self.getTreeElementbyExprs(htmltree, self.expr_cover, self.expr_cover2)
def getSmallCover(self, htmltree):
return self.getTreeElement(htmltree, self.expr_smallcover)
def getExtrafanart(self, htmltree):
def getExtrafanart(self, htmltree) -> list:
return self.getTreeAll(htmltree, self.expr_extrafanart)
def getTrailer(self, htmltree):
return self.getTreeElement(htmltree, self.expr_trailer)
def getActorPhoto(self, htmltree):
return self.getTreeAll(htmltree, self.expr_actorphoto)
def getActorPhoto(self, htmltree) -> dict:
return {}
def getUncensored(self, htmlree):
def getUncensored(self, htmlree) -> bool:
if self.expr_uncensored:
u = self.getTreeAll(htmlree, self.expr_uncensored)
return bool(u)
@@ -249,10 +216,10 @@ class Parser:
return self.uncensored
def getUserRating(self, htmltree):
return self.getTreeAll(htmltree, self.expr_userrating)
return self.getTreeElement(htmltree, self.expr_userrating)
def getUserVotes(self, htmltree):
return self.getTreeAll(htmltree, self.expr_uservotes)
return self.getTreeElement(htmltree, self.expr_uservotes)
def getTreeElement(self, tree: html.HtmlElement, expr, index=0):
""" 根据表达式从`xmltree`中获取匹配值,默认 index 为 0
@@ -263,3 +230,32 @@ class Parser:
""" 根据表达式从`xmltree`中获取全部匹配值
"""
return getTreeAll(tree, expr)
def getTreeElementbyExprs(self, tree: html.HtmlElement, expr, expr2=''):
""" 多个表达式获取element
使用内部的 getTreeElement 防止继承修改后出现问题
"""
try:
first = self.getTreeElement(tree, expr).strip()
if first:
return first
second = self.getTreeElement(tree, expr2).strip()
if second:
return second
return ''
except:
return ''
def getTreeAllbyExprs(self, tree: html.HtmlElement, expr, expr2=''):
""" 多个表达式获取所有element
合并并剔除重复元素
"""
try:
result1 = self.getTreeAll(tree, expr)
result2 = self.getTreeAll(tree, expr2)
clean = [ x.strip() for x in result1 if x.strip() and x.strip() != ',']
clean2 = [ x.strip() for x in result2 if x.strip() and x.strip() != ',']
result = list(set(clean + clean2))
return result
except:
return []

8
scrapinglib/utils.py
View File

@@ -2,7 +2,7 @@
from lxml.html import HtmlElement
def getTreeElement(tree: HtmlElement, expr, index=0):
def getTreeElement(tree: HtmlElement, expr='', index=0):
""" 根据表达式从`xmltree`中获取匹配值,默认 index 为 0
:param tree (html.HtmlElement)
:param expr
@@ -16,16 +16,16 @@ def getTreeElement(tree: HtmlElement, expr, index=0):
except:
return ''
def getTreeAll(tree: HtmlElement, expr):
def getTreeAll(tree: HtmlElement, expr=''):
""" 根据表达式从`xmltree`中获取全部匹配值
:param tree (html.HtmlElement)
:param expr
:param index
"""
if expr == '':
return ''
return []
result = tree.xpath(expr)
try:
return result
except:
return ''
return []

9
scrapinglib/xcity.py
View File

@@ -28,7 +28,7 @@ class Xcity(Parser):
return super().getStudio(htmltree).strip('+').replace("', '", '').replace('"', '')
def getRuntime(self, htmltree):
return self.getTreeAll(htmltree, self.expr_runtime)[1].strip()
return self.getTreeElement(htmltree, self.expr_runtime, 1).strip()
def getRelease(self, htmltree):
try:
@@ -37,13 +37,6 @@ class Xcity(Parser):
except:
return ''
def getTags(self, htmltree):
result = self.getTreeAll(htmltree, self.expr_tags)
total = []
for i in result:
total.append(i.replace("\n","").replace("\t",""))
return total
def getCover(self, htmltree):
try:
result = super().getCover(htmltree)