bacow/AV_Data_Capture
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parall query on storyline data

Browse Source
This commit is contained in:
lededev
2021-10-17 21:59:08 +08:00
parent b006aee34d
commit a546c4e83e
7 changed files with 336 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ADC_function.py
View File

@@ -85,12 +85,11 @@ def post_html(url: str, query: dict, headers: dict = None) -> requests.Response:
def get_html_by_browser(url, cookies: dict = None, ua: str = None, return_type: str = None):
s = None
if isinstance(cookies, dict) and len(cookies):
s = requests.Session()
requests.utils.add_dict_to_cookiejar(s.cookies, cookies)
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua, session=s)
else:
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua)
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua, session=s)
configProxy = config.getInstance().proxy()
if configProxy.enable:
browser.session.proxies = configProxy.proxies()
@@ -109,12 +108,11 @@ def get_html_by_browser(url, cookies: dict = None, ua: str = None, return_type:
def get_html_by_form(url, form_select: str = None, fields: dict = None, cookies: dict = None, ua: str = None, return_type: str = None):
s = None
if isinstance(cookies, dict) and len(cookies):
s = requests.Session()
requests.utils.add_dict_to_cookiejar(s.cookies, cookies)
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua, session=s)
else:
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua)
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua, session=s)
configProxy = config.getInstance().proxy()
if configProxy.enable:
browser.session.proxies = configProxy.proxies()