Merge pull request #406 from Ercsion/master

Add search function of airav

ADC_function.py

@@ -40,6 +40,7 @@ def get_proxy(proxy: str, proxytype: str = None) -> dict:
 
 # 网页请求核心
 def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None):
+    verify=config.Config().cacert_file()
     switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()
     proxies = get_proxy(proxy, proxytype)
 
@@ -51,7 +52,7 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None)
     for i in range(retry_count):
         try:
             if switch == '1' or switch == 1:
-                result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, cookies=cookies)
+                result = requests.get(str(url), headers=headers, timeout=timeout, proxies=proxies, verify=verify, cookies=cookies)
             else:
                 result = requests.get(str(url), headers=headers, timeout=timeout, cookies=cookies)
 

WebCrawler/airav.py

@@ -7,7 +7,14 @@ from bs4 import BeautifulSoup#need install
 import json
 from ADC_function import *
 
-
+'''
+API
+注册：https://www.airav.wiki/api/auth/signup
+设置：https://www.airav.wiki/api/get_web_settings
+搜索：https://www.airav.wiki/api/video/list?lng=zh-CN&search=
+搜索：https://www.airav.wiki/api/video/list?lang=zh-TW&lng=zh-TW&search=
+'''
+host = 'https://www.airav.wiki'
 
 # airav这个网站没有演员图片，所以直接使用javbus的图
 def getActorPhoto(htmlcode): #//*[@id="star_qdt"]/li/a/img
@@ -121,6 +128,44 @@ def getExtrafanart(htmlcode):  # 获取剧照
             return extrafanart_imgs
     return ''
 
+def search(keyword): #搜索，返回结果
+    result = []
+    page = 1
+    while page > 0:
+        # search_result = {"offset": 0,"count": 4,"result": [
+        #     {"vid": "99-07-15076","slug": "Wrop6o","name": "朝ゴミ出しする近所の遊び好きノーブラ奥さん 江波りゅう",
+        #     "url": "","view": 98,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-07-15076.jpg","barcode": "_1pondo_012717_472"},
+        #     {"vid": "99-27-00286","slug": "DlPEua","name": "放課後に、仕込んでください 〜優等生は無言でスカートを捲り上げる〜",
+        #     "url": "","view": 69,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-27-00286.jpg","barcode": "caribbeancom012717-360"},
+        #     {"vid": "99-07-15070","slug": "VLS3WY","name": "放課後に、仕込んでください ～優等生は無言でスカートを捲り上げる～ ももき希",
+        #     "url": "","view": 58,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-07-15070.jpg","barcode": "caribbeancom_012717-360"},
+        #     {"vid": "99-27-00287","slug": "YdMVb3","name": "朝ゴミ出しする近所の遊び好きノーブラ奥さん 江波りゅう",
+        #     "url": "","view": 56,"img_url": "https://wiki-img.airav.wiki/storage/big_pic/99-27-00287.jpg","barcode": "1pondo_012717_472"}
+        # ],"status": "ok"}
+        search_result = get_html(host + '/api/video/list?lang=zh-TW&lng=jp&search=' + keyword + '&page=' + str(page))
+
+        try:
+            json_data = json.loads(search_result)
+        except json.decoder.JSONDecodeError:
+            print("[-]Json decoder error!")
+            return []
+
+        result_offset = int(json_data["offset"])
+        result_count = int(json_data["count"])
+        result_size = len(json_data["result"])
+        if result_count <= 0 or result_size <= 0:
+            return result
+        elif result_count > result_offset + result_size: #请求下一页内容
+            result.extend(json_data["result"])
+            page += 1
+        elif result_count == result_offset + result_size: #请求最后一页内容
+            result.extend(json_data["result"])
+            page = 0
+        else:
+            page = 0
+
+    return result
+
 def main(number):
     try:
         try:
@@ -180,5 +225,10 @@ def main(number):
 
 
 if __name__ == '__main__':
-    print(main('ADN-188'))
+    #print(main('ADN-188'))
+
+    print(search('ADN-188'))
+    print(search('012717_472'))
+    print(search('080719-976'))
+    print(search('姫川ゆうな'))
 

config.ini

@@ -14,6 +14,7 @@ type=http
 proxy=127.0.0.1:1080
 timeout=5
 retry=3
+cacert_file=
 
 [Name_Rule]
 location_rule=actor+'/'+number

config.py

@@ -88,6 +88,9 @@ class Config:
         except ValueError:
             self._exit("common")
 
+    def cacert_file(self) -> str:
+        return self.conf.get('proxy', 'cacert_file')
+            
     def media_type(self) -> str:
         return self.conf.get('media', 'media_type')
 
@@ -153,6 +156,8 @@ class Config:
         conf.set(sec2, "timeout", "5")
         conf.set(sec2, "retry", "3")
         conf.set(sec2, "type", "socks5")
+        conf.set(sec2, "cacert_file", "")
+
 
         sec3 = "Name_Rule"
         conf.add_section(sec3)

xlog.py

@@ -0,0 +1,319 @@
+
+import os
+import sys
+import time
+from datetime import datetime
+import traceback
+import threading
+import json
+import shutil
+import types
+
+CRITICAL = 50
+FATAL = CRITICAL
+ERROR = 40
+WARNING = 30
+WARN = WARNING
+INFO = 20
+DEBUG = 10
+NOTSET = 0
+
+class Logger():
+    def __init__(self, name, buffer_size=0, file_name=None, roll_num=1):
+        self.err_color = '\033[0m'
+        self.warn_color = '\033[0m'
+        self.debug_color = '\033[0m'
+        self.reset_color = '\033[0m'
+        self.set_console_color = lambda color: sys.stderr.write(color)
+        self.name = str(name)
+        self.file_max_size = 1024 * 1024
+        self.buffer_lock = threading.Lock()
+        self.buffer = {} # id => line
+        self.buffer_size = buffer_size
+        self.last_no = 0
+        self.min_level = NOTSET
+        self.log_fd = None
+        self.roll_num = roll_num
+        if file_name:
+            self.set_file(file_name)
+
+    def set_buffer(self, buffer_size):
+        with self.buffer_lock:
+            self.buffer_size = buffer_size
+            buffer_len = len(self.buffer)
+            if buffer_len > self.buffer_size:
+                for i in range(self.last_no - buffer_len, self.last_no - self.buffer_size):
+                    try:
+                        del self.buffer[i]
+                    except:
+                        pass
+
+    def setLevel(self, level):
+        if level == "DEBUG":
+            self.min_level = DEBUG
+        elif level == "INFO":
+            self.min_level = INFO
+        elif level == "WARN":
+            self.min_level = WARN
+        elif level == "ERROR":
+            self.min_level = ERROR
+        elif level == "FATAL":
+            self.min_level = FATAL
+        else:
+            print(("log level not support:%s", level))
+
+    def set_color(self):
+        self.err_color = None
+        self.warn_color = None
+        self.debug_color = None
+        self.reset_color = None
+        self.set_console_color = lambda x: None
+        if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty():
+            if os.name == 'nt':
+                self.err_color = 0x04
+                self.warn_color = 0x06
+                self.debug_color = 0x002
+                self.reset_color = 0x07
+
+                import ctypes
+                SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
+                GetStdHandle = ctypes.windll.kernel32.GetStdHandle
+                self.set_console_color = lambda color: SetConsoleTextAttribute(GetStdHandle(-11), color)
+
+            elif os.name == 'posix':
+                self.err_color = '\033[31m'
+                self.warn_color = '\033[33m'
+                self.debug_color = '\033[32m'
+                self.reset_color = '\033[0m'
+
+                self.set_console_color = lambda color: sys.stderr.write(color)
+
+    def set_file(self, file_name):
+        self.log_filename = file_name
+        if os.path.isfile(file_name):
+            self.file_size = os.path.getsize(file_name)
+            if self.file_size > self.file_max_size:
+                self.roll_log()
+                self.file_size = 0
+        else:
+            self.file_size = 0
+
+        self.log_fd = open(file_name, "a+")
+
+    def roll_log(self):
+        for i in range(self.roll_num, 1, -1):
+            new_name = "%s.%d" % (self.log_filename, i)
+            old_name = "%s.%d" % (self.log_filename, i - 1)
+            if not os.path.isfile(old_name):
+                continue
+
+            #self.info("roll_log %s -> %s", old_name, new_name)
+            shutil.move(old_name, new_name)
+
+        shutil.move(self.log_filename, self.log_filename + ".1")
+
+    def log_console(self, level, console_color, fmt, *args, **kwargs):
+        try:
+            console_string = '[%s] %s\n' % (level, fmt % args)
+            self.set_console_color(console_color)
+            sys.stderr.write(console_string)
+            self.set_console_color(self.reset_color)
+        except:
+            pass
+
+    def log_to_file(self, level, console_color, fmt, *args, **kwargs):
+        if self.log_fd:
+            if level == 'e':
+                string = '%s' % (fmt % args)
+            else:
+                time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:23]
+                string = '%s [%s] [%s] %s\n' % (time_str, self.name, level, fmt % args)
+
+            self.log_fd.write(string)
+            try:
+                self.log_fd.flush()
+            except:
+                pass
+
+            self.file_size += len(string)
+            if self.file_size > self.file_max_size:
+                self.log_fd.close()
+                self.log_fd = None
+                self.roll_log()
+                self.log_fd = open(self.log_filename, "w")
+                self.file_size = 0
+
+    def log(self, level, console_color, html_color, fmt, *args, **kwargs):
+        self.buffer_lock.acquire()
+        try:
+            self.log_console(level, console_color, fmt, *args, **kwargs)
+
+            self.log_to_file(level, console_color, fmt, *args, **kwargs)
+
+            if self.buffer_size:
+                self.last_no += 1
+                self.buffer[self.last_no] = string
+                buffer_len = len(self.buffer)
+                if buffer_len > self.buffer_size:
+                    del self.buffer[self.last_no - self.buffer_size]
+        except Exception as e:
+            string = '%s - [%s]LOG_EXCEPT: %s, Except:%s<br> %s' % (time.ctime()[4:-5], level, fmt % args, e, traceback.format_exc())
+            self.last_no += 1
+            self.buffer[self.last_no] = string
+            buffer_len = len(self.buffer)
+            if buffer_len > self.buffer_size:
+                del self.buffer[self.last_no - self.buffer_size]
+        finally:
+            self.buffer_lock.release()
+
+    def debug(self, fmt, *args, **kwargs):
+        if self.min_level > DEBUG:
+            return
+        self.log('-', self.debug_color, '21610b', fmt, *args, **kwargs)
+
+    def info(self, fmt, *args, **kwargs):
+        if self.min_level > INFO:
+            return
+        self.log('+', self.reset_color, '000000', fmt, *args)
+
+    def warning(self, fmt, *args, **kwargs):
+        if self.min_level > WARN:
+            return
+        self.log('#', self.warn_color, 'FF8000', fmt, *args, **kwargs)
+
+    def warn(self, fmt, *args, **kwargs):
+        self.warning(fmt, *args, **kwargs)
+
+    def error(self, fmt, *args, **kwargs):
+        if self.min_level > ERROR:
+            return
+        self.log('!', self.err_color, 'FE2E2E', fmt, *args, **kwargs)
+
+    def exception(self, fmt, *args, **kwargs):
+        self.error(fmt, *args, **kwargs)
+        string = '%s' % (traceback.format_exc())
+        self.log_to_file('e', self.err_color, string)
+
+    def critical(self, fmt, *args, **kwargs):
+        if self.min_level > CRITICAL:
+            return
+        self.log('!', self.err_color, 'D7DF01', fmt, *args, **kwargs)
+
+    def tofile(self, fmt, *args, **kwargs):
+        self.log_to_file('@', self.warn_color, fmt, *args, **kwargs)
+
+    #=================================================================
+    def set_buffer_size(self, set_size):
+        self.buffer_lock.acquire()
+        self.buffer_size = set_size
+        buffer_len = len(self.buffer)
+        if buffer_len > self.buffer_size:
+            for i in range(self.last_no - buffer_len, self.last_no - self.buffer_size):
+                try:
+                    del self.buffer[i]
+                except:
+                    pass
+        self.buffer_lock.release()
+
+    def get_last_lines(self, max_lines):
+        self.buffer_lock.acquire()
+        buffer_len = len(self.buffer)
+        if buffer_len > max_lines:
+            first_no = self.last_no - max_lines
+        else:
+            first_no = self.last_no - buffer_len + 1
+
+        jd = {}
+        if buffer_len > 0:
+            for i in range(first_no, self.last_no + 1):
+                jd[i] = self.unicode_line(self.buffer[i])
+        self.buffer_lock.release()
+        return json.dumps(jd)
+
+    def get_new_lines(self, from_no):
+        self.buffer_lock.acquire()
+        jd = {}
+        first_no = self.last_no - len(self.buffer) + 1
+        if from_no < first_no:
+            from_no = first_no
+
+        if self.last_no >= from_no:
+            for i in range(from_no, self.last_no + 1):
+                jd[i] = self.unicode_line(self.buffer[i])
+        self.buffer_lock.release()
+        return json.dumps(jd)
+
+    def unicode_line(self, line):
+        try:
+            if type(line) is str:
+                return line
+            else:
+                return str(line, errors='ignore')
+        except Exception as e:
+            print(("unicode err:%r" % e))
+            print(("line can't decode:%s" % line))
+            print(("Except stack:%s" % traceback.format_exc()))
+            return ""
+
+loggerDict = {}
+
+def getLogger(name=None, buffer_size=0, file_name=None, roll_num=1):
+    global loggerDict, default_log
+    if name is None:
+        for n in loggerDict:
+            name = n
+            break
+    if name is None:
+        name = u"default"
+
+    if not isinstance(name, str):
+        raise TypeError('A logger name must be string or Unicode')
+    if isinstance(name, bytes):
+        name = name.encode('utf-8')
+
+    if name in loggerDict:
+        return loggerDict[name]
+    else:
+        logger_instance = Logger(name, buffer_size, file_name, roll_num)
+        loggerDict[name] = logger_instance
+        default_log = logger_instance
+        return logger_instance
+
+default_log = getLogger()
+
+def debg(fmt, *args, **kwargs):
+    default_log.debug(fmt, *args, **kwargs)
+
+def info(fmt, *args, **kwargs):
+    default_log.info(fmt, *args, **kwargs)
+
+def warn(fmt, *args, **kwargs):
+    default_log.warning(fmt, *args, **kwargs)
+
+def erro(fmt, *args, **kwargs):
+    default_log.error(fmt, *args, **kwargs)
+
+def excp(fmt, *args, **kwargs):
+    default_log.exception(fmt, *args, **kwargs)
+
+def crit(fmt, *args, **kwargs):
+    default_log.critical(fmt, *args, **kwargs)
+
+def tofile(fmt, *args, **kwargs):
+    default_log.tofile(fmt, *args, **kwargs)
+
+if __name__ == '__main__':
+    log_file = os.path.join(os.path.dirname(sys.argv[0]), "test.log")
+    getLogger().set_file(log_file)
+    debg("debug")
+    info("info")
+    warn("warning")
+    erro("error")
+    crit("critical")
+    tofile("write to file only")
+
+    try:
+        1/0
+    except Exception as e:
+        excp("An error has occurred")
+