bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

speed up by reusing stateful browser

Browse Source
This commit is contained in:
lededev
2021-09-23 07:01:24 +08:00
parent 367d53b09b
commit c32a4a12ac
1 changed files with 11 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
WebCrawler/xcity.py
View File

@@ -171,13 +171,16 @@ def getExtrafanart(htmlcode): # 获取剧照
def main(number): def main(number):
try: try:
query_result = get_html_by_form('https://xcity.jp/about/', query_result, browser = get_html_by_form(
fields = {'q' : number.replace('-','').lower()}) 'https://xcity.jp/about/',
html = etree.fromstring(query_result, etree.HTMLParser()) fields = {'q' : number.replace('-','').lower()},
urls = str(html.xpath('//table[@class="resultList"]/tr[2]/td[1]/a/@href')).strip(" ['']") return_type = 'browser')
if not len(urls): if not query_result or not query_result.ok:
raise ValueError("xcity.py: urls not found") raise ValueError("xcity.py: page not found")
detail_page = get_html(abs_url('https://xcity.jp', urls)) result = browser.follow_link(browser.links('avod\/detail')[0])
if not result.ok:
raise ValueError("xcity.py: detail page not found")
detail_page = str(browser.page)
dic = { dic = {
'actor': getActor(detail_page), 'actor': getActor(detail_page),
'title': getTitle(detail_page), 'title': getTitle(detail_page),
@@ -195,7 +198,7 @@ def main(number):
'label': getLabel(detail_page), 'label': getLabel(detail_page),
'year': getYear(getRelease(detail_page)), # str(re.search('\d{4}',getRelease(a)).group()), 'year': getYear(getRelease(detail_page)), # str(re.search('\d{4}',getRelease(a)).group()),
'actor_photo': getActorPhoto(getActor(detail_page)), 'actor_photo': getActorPhoto(getActor(detail_page)),
'website': 'https://xcity.jp' + urls, 'website': browser.url,
'source': 'xcity.py', 'source': 'xcity.py',
'series': getSeries(detail_page), 'series': getSeries(detail_page),
} }