bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #406 from Ercsion/master

Browse Source 
Add search function of airav
This commit is contained in:
Yoshiko2
2021-01-11 01:23:26 +08:00
committed by GitHub
parent 1ed1717c0d 3198925284
commit e0117a7a16
5 changed files with 379 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ADC_function.py
View File

@@ -40,6 +40,7 @@ def get_proxy(proxy: str, proxytype: str = None) -> dict:
# 网页请求核心
def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None):
verify=config.Config().cacert_file()
switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
proxies = get_proxy(proxy, proxytype)
@@ -51,7 +52,7 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None)
for i in range(retry_count):
try:
if switch == '1' or switch == 1:
result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, cookies=cookies)
result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, verify=verify, cookies=cookies)
else:
result = requests.get(str(url), headers=headers, timeout=timeout, cookies=cookies)

54
WebCrawler/airav.py
View File

@@ -7,7 +7,14 @@ from bs4 import BeautifulSoup#need install
import json
from ADC_function import *
'''
API
注册：https://www.airav.wiki/api/auth/signup
设置：https://www.airav.wiki/api/get_web_settings
搜索：https://www.airav.wiki/api/video/list?lng=zh-CN&search=
搜索：https://www.airav.wiki/api/video/list?lang=zh-TW&lng=zh-TW&search=
'''
host = 'https://www.airav.wiki'
# airav这个网站没有演员图片，所以直接使用javbus的图
def getActorPhoto(htmlcode): #//*[@id="star_qdt"]/li/a/img
@@ -121,6 +128,44 @@ def getExtrafanart(htmlcode): # 获取剧照
return extrafanart_imgs
return ''
def search(keyword): #搜索，返回结果
result = []
page = 1
while page > 0:
# search_result = {"offset": 0,"count": 4,"result": [
# {"vid": "99-07-15076","slug": "Wrop6o","name": "朝ゴミ出しする近所の遊び好きノーブラ奥さん 江波りゅう",
# "url": "","view": 98,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-07-15076.jpg","barcode": "_1pondo_012717_472"},
# {"vid": "99-27-00286","slug": "DlPEua","name": "放課後に、仕込んでください 〜優等生は無言でスカートを捲り上げる〜",
# "url": "","view": 69,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-27-00286.jpg","barcode": "caribbeancom012717-360"},
# {"vid": "99-07-15070","slug": "VLS3WY","name": "放課後に、仕込んでください ～優等生は無言でスカートを捲り上げる～ ももき希",
# "url": "","view": 58,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-07-15070.jpg","barcode": "caribbeancom_012717-360"},
# {"vid": "99-27-00287","slug": "YdMVb3","name": "朝ゴミ出しする近所の遊び好きノーブラ奥さん 江波りゅう",
# "url": "","view": 56,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-27-00287.jpg","barcode": "1pondo_012717_472"}
# ],"status": "ok"}
search_result = get_html(host + '/api/video/list?lang=zh-TW&lng=jp&search=' + keyword + '&page=' + str(page))
try:
json_data = json.loads(search_result)
except json.decoder.JSONDecodeError:
print("[-]Json decoder error!")
return []
result_offset = int(json_data["offset"])
result_count = int(json_data["count"])
result_size = len(json_data["result"])
if result_count <= 0 or result_size <= 0:
return result
elif result_count > result_offset + result_size: #请求下一页内容
result.extend(json_data["result"])
page += 1
elif result_count == result_offset + result_size: #请求最后一页内容
result.extend(json_data["result"])
page = 0
else:
page = 0
return result
def main(number):
try:
try:
@@ -180,5 +225,10 @@ def main(number):
if __name__ == '__main__':
print(main('ADN-188'))
#print(main('ADN-188'))
print(search('ADN-188'))
print(search('012717_472'))
print(search('080719-976'))
print(search('姫川ゆうな'))

1
config.ini
View File

@@ -14,6 +14,7 @@ type=http
proxy=127.0.0.1:1080
timeout=5
retry=3
cacert_file=
[Name_Rule]
location_rule=actor+'/'+number

5
config.py
View File

@@ -88,6 +88,9 @@ class Config:
except ValueError:
self._exit("common")
def cacert_file(self) -> str:
return self.conf.get('proxy', 'cacert_file')
def media_type(self) -> str:
return self.conf.get('media', 'media_type')
@@ -153,6 +156,8 @@ class Config:
conf.set(sec2, "timeout", "5")
conf.set(sec2, "retry", "3")
conf.set(sec2, "type", "socks5")
conf.set(sec2, "cacert_file", "")
sec3 = "Name_Rule"
conf.add_section(sec3)

319
xlog.py Executable file
View File

@@ -0,0 +1,319 @@
import os
import sys
import time
from datetime import datetime
import traceback
import threading
import json
import shutil
import types
CRITICAL = 50
FATAL = CRITICAL
ERROR = 40
WARNING = 30
WARN = WARNING
INFO = 20
DEBUG = 10
NOTSET = 0
class Logger():
def __init__(self, name, buffer_size=0, file_name=None, roll_num=1):
self.err_color = '\033[0m'
self.warn_color = '\033[0m'
self.debug_color = '\033[0m'
self.reset_color = '\033[0m'
self.set_console_color = lambda color: sys.stderr.write(color)
self.name = str(name)
self.file_max_size = 1024 * 1024
self.buffer_lock = threading.Lock()
self.buffer = {} # id => line
self.buffer_size = buffer_size
self.last_no = 0
self.min_level = NOTSET
self.log_fd = None
self.roll_num = roll_num
if file_name:
self.set_file(file_name)
def set_buffer(self, buffer_size):
with self.buffer_lock:
self.buffer_size = buffer_size
buffer_len = len(self.buffer)
if buffer_len > self.buffer_size:
for i in range(self.last_no - buffer_len, self.last_no - self.buffer_size):
try:
del self.buffer[i]
except:
pass
def setLevel(self, level):
if level == "DEBUG":
self.min_level = DEBUG
elif level == "INFO":
self.min_level = INFO
elif level == "WARN":
self.min_level = WARN
elif level == "ERROR":
self.min_level = ERROR
elif level == "FATAL":
self.min_level = FATAL
else:
print(("log level not support:%s", level))
def set_color(self):
self.err_color = None
self.warn_color = None
self.debug_color = None
self.reset_color = None
self.set_console_color = lambda x: None
if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
if os.name == 'nt':
self.err_color = 0x04
self.warn_color = 0x06
self.debug_color = 0x002
self.reset_color = 0x07
import ctypes
SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
GetStdHandle = ctypes.windll.kernel32.GetStdHandle
self.set_console_color = lambda color: SetConsoleTextAttribute(GetStdHandle(-11), color)
elif os.name == 'posix':
self.err_color = '\033[31m'
self.warn_color = '\033[33m'
self.debug_color = '\033[32m'
self.reset_color = '\033[0m'
self.set_console_color = lambda color: sys.stderr.write(color)
def set_file(self, file_name):
self.log_filename = file_name
if os.path.isfile(file_name):
self.file_size = os.path.getsize(file_name)
if self.file_size > self.file_max_size:
self.roll_log()
self.file_size = 0
else:
self.file_size = 0
self.log_fd = open(file_name, "a+")
def roll_log(self):
for i in range(self.roll_num, 1, -1):
new_name = "%s.%d" % (self.log_filename, i)
old_name = "%s.%d" % (self.log_filename, i - 1)
if not os.path.isfile(old_name):
continue
#self.info("roll_log %s -> %s", old_name, new_name)
shutil.move(old_name, new_name)
shutil.move(self.log_filename, self.log_filename + ".1")
def log_console(self, level, console_color, fmt, *args, **kwargs):
try:
console_string = '[%s] %s\n' % (level, fmt % args)
self.set_console_color(console_color)
sys.stderr.write(console_string)
self.set_console_color(self.reset_color)
except:
pass
def log_to_file(self, level, console_color, fmt, *args, **kwargs):
if self.log_fd:
if level == 'e':
string = '%s' % (fmt % args)
else:
time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:23]
string = '%s [%s] [%s] %s\n' % (time_str, self.name, level, fmt % args)
self.log_fd.write(string)
try:
self.log_fd.flush()
except:
pass
self.file_size += len(string)
if self.file_size > self.file_max_size:
self.log_fd.close()
self.log_fd = None
self.roll_log()
self.log_fd = open(self.log_filename, "w")
self.file_size = 0
def log(self, level, console_color, html_color, fmt, *args, **kwargs):
self.buffer_lock.acquire()
try:
self.log_console(level, console_color, fmt, *args, **kwargs)
self.log_to_file(level, console_color, fmt, *args, **kwargs)
if self.buffer_size:
self.last_no += 1
self.buffer[self.last_no] = string
buffer_len = len(self.buffer)
if buffer_len > self.buffer_size:
del self.buffer[self.last_no - self.buffer_size]
except Exception as e:
string = '%s - [%s]LOG_EXCEPT: %s, Except:%s<br> %s' % (time.ctime()[4:-5], level, fmt % args, e, traceback.format_exc())
self.last_no += 1
self.buffer[self.last_no] = string
buffer_len = len(self.buffer)
if buffer_len > self.buffer_size:
del self.buffer[self.last_no - self.buffer_size]
finally:
self.buffer_lock.release()
def debug(self, fmt, *args, **kwargs):
if self.min_level > DEBUG:
return
self.log('-', self.debug_color, '21610b', fmt, *args, **kwargs)
def info(self, fmt, *args, **kwargs):
if self.min_level > INFO:
return
self.log('+', self.reset_color, '000000', fmt, *args)
def warning(self, fmt, *args, **kwargs):
if self.min_level > WARN:
return
self.log('#', self.warn_color, 'FF8000', fmt, *args, **kwargs)
def warn(self, fmt, *args, **kwargs):
self.warning(fmt, *args, **kwargs)
def error(self, fmt, *args, **kwargs):
if self.min_level > ERROR:
return
self.log('!', self.err_color, 'FE2E2E', fmt, *args, **kwargs)
def exception(self, fmt, *args, **kwargs):
self.error(fmt, *args, **kwargs)
string = '%s' % (traceback.format_exc())
self.log_to_file('e', self.err_color, string)
def critical(self, fmt, *args, **kwargs):
if self.min_level > CRITICAL:
return
self.log('!', self.err_color, 'D7DF01', fmt, *args, **kwargs)
def tofile(self, fmt, *args, **kwargs):
self.log_to_file('@', self.warn_color, fmt, *args, **kwargs)
#=================================================================
def set_buffer_size(self, set_size):
self.buffer_lock.acquire()
self.buffer_size = set_size
buffer_len = len(self.buffer)
if buffer_len > self.buffer_size:
for i in range(self.last_no - buffer_len, self.last_no - self.buffer_size):
try:
del self.buffer[i]
except:
pass
self.buffer_lock.release()
def get_last_lines(self, max_lines):
self.buffer_lock.acquire()
buffer_len = len(self.buffer)
if buffer_len > max_lines:
first_no = self.last_no - max_lines
else:
first_no = self.last_no - buffer_len + 1
jd = {}
if buffer_len > 0:
for i in range(first_no, self.last_no + 1):
jd[i] = self.unicode_line(self.buffer[i])
self.buffer_lock.release()
return json.dumps(jd)
def get_new_lines(self, from_no):
self.buffer_lock.acquire()
jd = {}
first_no = self.last_no - len(self.buffer) + 1
if from_no < first_no:
from_no = first_no
if self.last_no >= from_no:
for i in range(from_no, self.last_no + 1):
jd[i] = self.unicode_line(self.buffer[i])
self.buffer_lock.release()
return json.dumps(jd)
def unicode_line(self, line):
try:
if type(line) is str:
return line
else:
return str(line, errors='ignore')
except Exception as e:
print(("unicode err:%r" % e))
print(("line can't decode:%s" % line))
print(("Except stack:%s" % traceback.format_exc()))
return ""
loggerDict = {}
def getLogger(name=None, buffer_size=0, file_name=None, roll_num=1):
global loggerDict, default_log
if name is None:
for n in loggerDict:
name = n
break
if name is None:
name = u"default"
if not isinstance(name, str):
raise TypeError('A logger name must be string or Unicode')
if isinstance(name, bytes):
name = name.encode('utf-8')
if name in loggerDict:
return loggerDict[name]
else:
logger_instance = Logger(name, buffer_size, file_name, roll_num)
loggerDict[name] = logger_instance
default_log = logger_instance
return logger_instance
default_log = getLogger()
def debg(fmt, *args, **kwargs):
default_log.debug(fmt, *args, **kwargs)
def info(fmt, *args, **kwargs):
default_log.info(fmt, *args, **kwargs)
def warn(fmt, *args, **kwargs):
default_log.warning(fmt, *args, **kwargs)
def erro(fmt, *args, **kwargs):
default_log.error(fmt, *args, **kwargs)
def excp(fmt, *args, **kwargs):
default_log.exception(fmt, *args, **kwargs)
def crit(fmt, *args, **kwargs):
default_log.critical(fmt, *args, **kwargs)
def tofile(fmt, *args, **kwargs):
default_log.tofile(fmt, *args, **kwargs)
if __name__ == '__main__':
log_file = os.path.join(os.path.dirname(sys.argv[0]), "test.log")
getLogger().set_file(log_file)
debg("debug")
info("info")
warn("warning")
erro("error")
crit("critical")
tofile("write to file only")
try:
1/0
except Exception as e:
excp("An error has occurred")