xcity.py: get detail page by form query

This commit is contained in:
lededev
2021-09-22 06:03:58 +08:00
parent ffd80ba0e4
commit b59b4938d6
3 changed files with 32 additions and 7 deletions

View File

@@ -11,6 +11,7 @@ from lxml import etree
import re
import config
from urllib.parse import urljoin
import mechanicalsoup
def getXpathSingle(htmlcode, xpath):
@@ -83,6 +84,27 @@ def post_html(url: str, query: dict, headers: dict = None) -> requests.Response:
print("[-]" + errors)
def get_html_by_form(url, form_name: str = None, fields: dict = None, cookies: dict = None, ua: str = None, return_type: str = None):
browser = mechanicalsoup.StatefulBrowser(user_agent=G_USER_AGENT if ua is None else ua)
configProxy = config.Config().proxy()
if configProxy.enable:
browser.session.proxies = configProxy.proxies()
result = browser.open(url)
form = browser.select_form() if form_name is None else browser.select_form(form_name)
if isinstance(fields, dict):
for k, v in fields.items():
browser[k] = v
response = browser.submit_selected()
response.encoding = "utf-8"
if return_type == "object":
return response
elif return_type == "content":
return response.content
else:
return response.text
# def get_javlib_cookie() -> [dict, str]:
# import cloudscraper
# switch, proxy, timeout, retry_count, proxytype = config.Config().proxy()