Refactor: Format Code with Ruff and Update Deprecated G2PW Link (#2255)
* ruff check --fix * ruff format --line-length 120 --target-version py39 * Change the link for G2PW Model * update pytorch version and colab
This commit is contained in:
XXXXRT666
File diff suppressed because one or more lines are too long
@@ -21,25 +21,29 @@ from .num import verbalize_digit
|
||||
|
||||
def _time_num2str(num_string: str) -> str:
|
||||
"""A special case for verbalizing number in time."""
|
||||
result = num2str(num_string.lstrip('0'))
|
||||
if num_string.startswith('0'):
|
||||
result = DIGITS['0'] + result
|
||||
result = num2str(num_string.lstrip("0"))
|
||||
if num_string.startswith("0"):
|
||||
result = DIGITS["0"] + result
|
||||
return result
|
||||
|
||||
|
||||
# 时刻表达式
|
||||
RE_TIME = re.compile(r'([0-1]?[0-9]|2[0-3])'
|
||||
r':([0-5][0-9])'
|
||||
r'(:([0-5][0-9]))?')
|
||||
RE_TIME = re.compile(
|
||||
r"([0-1]?[0-9]|2[0-3])"
|
||||
r":([0-5][0-9])"
|
||||
r"(:([0-5][0-9]))?"
|
||||
)
|
||||
|
||||
# 时间范围，如8:30-12:30
|
||||
RE_TIME_RANGE = re.compile(r'([0-1]?[0-9]|2[0-3])'
|
||||
r':([0-5][0-9])'
|
||||
r'(:([0-5][0-9]))?'
|
||||
r'(~|-)'
|
||||
r'([0-1]?[0-9]|2[0-3])'
|
||||
r':([0-5][0-9])'
|
||||
r'(:([0-5][0-9]))?')
|
||||
RE_TIME_RANGE = re.compile(
|
||||
r"([0-1]?[0-9]|2[0-3])"
|
||||
r":([0-5][0-9])"
|
||||
r"(:([0-5][0-9]))?"
|
||||
r"(~|-)"
|
||||
r"([0-1]?[0-9]|2[0-3])"
|
||||
r":([0-5][0-9])"
|
||||
r"(:([0-5][0-9]))?"
|
||||
)
|
||||
|
||||
|
||||
def replace_time(match) -> str:
|
||||
@@ -62,31 +66,33 @@ def replace_time(match) -> str:
|
||||
second_2 = match.group(9)
|
||||
|
||||
result = f"{num2str(hour)}点"
|
||||
if minute.lstrip('0'):
|
||||
if minute.lstrip("0"):
|
||||
if int(minute) == 30:
|
||||
result += "半"
|
||||
else:
|
||||
result += f"{_time_num2str(minute)}分"
|
||||
if second and second.lstrip('0'):
|
||||
if second and second.lstrip("0"):
|
||||
result += f"{_time_num2str(second)}秒"
|
||||
|
||||
if is_range:
|
||||
result += "至"
|
||||
result += f"{num2str(hour_2)}点"
|
||||
if minute_2.lstrip('0'):
|
||||
if minute_2.lstrip("0"):
|
||||
if int(minute) == 30:
|
||||
result += "半"
|
||||
else:
|
||||
result += f"{_time_num2str(minute_2)}分"
|
||||
if second_2 and second_2.lstrip('0'):
|
||||
if second_2 and second_2.lstrip("0"):
|
||||
result += f"{_time_num2str(second_2)}秒"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
RE_DATE = re.compile(r'(\d{4}|\d{2})年'
|
||||
r'((0?[1-9]|1[0-2])月)?'
|
||||
r'(((0?[1-9])|((1|2)[0-9])|30|31)([日号]))?')
|
||||
RE_DATE = re.compile(
|
||||
r"(\d{4}|\d{2})年"
|
||||
r"((0?[1-9]|1[0-2])月)?"
|
||||
r"(((0?[1-9])|((1|2)[0-9])|30|31)([日号]))?"
|
||||
)
|
||||
|
||||
|
||||
def replace_date(match) -> str:
|
||||
@@ -110,8 +116,7 @@ def replace_date(match) -> str:
|
||||
|
||||
|
||||
# 用 / 或者 - 分隔的 YY/MM/DD 或者 YY-MM-DD 日期
|
||||
RE_DATE2 = re.compile(
|
||||
r'(\d{4})([- /.])(0[1-9]|1[012])\2(0[1-9]|[12][0-9]|3[01])')
|
||||
RE_DATE2 = re.compile(r"(\d{4})([- /.])(0[1-9]|1[012])\2(0[1-9]|[12][0-9]|3[01])")
|
||||
|
||||
|
||||
def replace_date2(match) -> str:
|
||||
|
||||
@@ -18,10 +18,7 @@ from pypinyin.constants import SUPPORT_UCS4
|
||||
|
||||
# 全角半角转换
|
||||
# 英文字符全角 -> 半角映射表 (num: 52)
|
||||
F2H_ASCII_LETTERS = {
|
||||
ord(char) + 65248: ord(char)
|
||||
for char in string.ascii_letters
|
||||
}
|
||||
F2H_ASCII_LETTERS = {ord(char) + 65248: ord(char) for char in string.ascii_letters}
|
||||
|
||||
# 英文字符半角 -> 全角映射表
|
||||
H2F_ASCII_LETTERS = {value: key for key, value in F2H_ASCII_LETTERS.items()}
|
||||
@@ -37,26 +34,29 @@ F2H_PUNCTUATIONS = {ord(char) + 65248: ord(char) for char in string.punctuation}
|
||||
H2F_PUNCTUATIONS = {value: key for key, value in F2H_PUNCTUATIONS.items()}
|
||||
|
||||
# 空格 (num: 1)
|
||||
F2H_SPACE = {'\u3000': ' '}
|
||||
H2F_SPACE = {' ': '\u3000'}
|
||||
F2H_SPACE = {"\u3000": " "}
|
||||
H2F_SPACE = {" ": "\u3000"}
|
||||
|
||||
# 非"有拼音的汉字"的字符串，可用于NSW提取
|
||||
if SUPPORT_UCS4:
|
||||
RE_NSW = re.compile(r'(?:[^'
|
||||
r'\u3007' # 〇
|
||||
r'\u3400-\u4dbf' # CJK扩展A:[3400-4DBF]
|
||||
r'\u4e00-\u9fff' # CJK基本:[4E00-9FFF]
|
||||
r'\uf900-\ufaff' # CJK兼容:[F900-FAFF]
|
||||
r'\U00020000-\U0002A6DF' # CJK扩展B:[20000-2A6DF]
|
||||
r'\U0002A703-\U0002B73F' # CJK扩展C:[2A700-2B73F]
|
||||
r'\U0002B740-\U0002B81D' # CJK扩展D:[2B740-2B81D]
|
||||
r'\U0002F80A-\U0002FA1F' # CJK兼容扩展:[2F800-2FA1F]
|
||||
r'])+')
|
||||
RE_NSW = re.compile(
|
||||
r"(?:[^"
|
||||
r"\u3007" # 〇
|
||||
r"\u3400-\u4dbf" # CJK扩展A:[3400-4DBF]
|
||||
r"\u4e00-\u9fff" # CJK基本:[4E00-9FFF]
|
||||
r"\uf900-\ufaff" # CJK兼容:[F900-FAFF]
|
||||
r"\U00020000-\U0002A6DF" # CJK扩展B:[20000-2A6DF]
|
||||
r"\U0002A703-\U0002B73F" # CJK扩展C:[2A700-2B73F]
|
||||
r"\U0002B740-\U0002B81D" # CJK扩展D:[2B740-2B81D]
|
||||
r"\U0002F80A-\U0002FA1F" # CJK兼容扩展:[2F800-2FA1F]
|
||||
r"])+"
|
||||
)
|
||||
else:
|
||||
RE_NSW = re.compile( # pragma: no cover
|
||||
r'(?:[^'
|
||||
r'\u3007' # 〇
|
||||
r'\u3400-\u4dbf' # CJK扩展A:[3400-4DBF]
|
||||
r'\u4e00-\u9fff' # CJK基本:[4E00-9FFF]
|
||||
r'\uf900-\ufaff' # CJK兼容:[F900-FAFF]
|
||||
r'])+')
|
||||
r"(?:[^"
|
||||
r"\u3007" # 〇
|
||||
r"\u3400-\u4dbf" # CJK扩展A:[3400-4DBF]
|
||||
r"\u4e00-\u9fff" # CJK基本:[4E00-9FFF]
|
||||
r"\uf900-\ufaff" # CJK兼容:[F900-FAFF]
|
||||
r"])+"
|
||||
)
|
||||
|
||||
@@ -15,23 +15,26 @@
|
||||
Rules to verbalize numbers into Chinese characters.
|
||||
https://zh.wikipedia.org/wiki/中文数字#現代中文
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
from typing import List
|
||||
|
||||
DIGITS = {str(i): tran for i, tran in enumerate('零一二三四五六七八九')}
|
||||
UNITS = OrderedDict({
|
||||
1: '十',
|
||||
2: '百',
|
||||
3: '千',
|
||||
4: '万',
|
||||
8: '亿',
|
||||
})
|
||||
DIGITS = {str(i): tran for i, tran in enumerate("零一二三四五六七八九")}
|
||||
UNITS = OrderedDict(
|
||||
{
|
||||
1: "十",
|
||||
2: "百",
|
||||
3: "千",
|
||||
4: "万",
|
||||
8: "亿",
|
||||
}
|
||||
)
|
||||
|
||||
COM_QUANTIFIERS = '(处|台|架|枚|趟|幅|平|方|堵|间|床|株|批|项|例|列|篇|栋|注|亩|封|艘|把|目|套|段|人|所|朵|匹|张|座|回|场|尾|条|个|首|阙|阵|网|炮|顶|丘|棵|只|支|袭|辆|挑|担|颗|壳|窠|曲|墙|群|腔|砣|座|客|贯|扎|捆|刀|令|打|手|罗|坡|山|岭|江|溪|钟|队|单|双|对|出|口|头|脚|板|跳|枝|件|贴|针|线|管|名|位|身|堂|课|本|页|家|户|层|丝|毫|厘|分|钱|两|斤|担|铢|石|钧|锱|忽|(千|毫|微)克|毫|厘|(公)分|分|寸|尺|丈|里|寻|常|铺|程|(千|分|厘|毫|微)米|米|撮|勺|合|升|斗|石|盘|碗|碟|叠|桶|笼|盆|盒|杯|钟|斛|锅|簋|篮|盘|桶|罐|瓶|壶|卮|盏|箩|箱|煲|啖|袋|钵|年|月|日|季|刻|时|周|天|秒|分|小时|旬|纪|岁|世|更|夜|春|夏|秋|冬|代|伏|辈|丸|泡|粒|颗|幢|堆|条|根|支|道|面|片|张|颗|块|元|(亿|千万|百万|万|千|百)|(亿|千万|百万|万|千|百|美|)元|(亿|千万|百万|万|千|百|十|)吨|(亿|千万|百万|万|千|百|)块|角|毛|分)'
|
||||
COM_QUANTIFIERS = "(处|台|架|枚|趟|幅|平|方|堵|间|床|株|批|项|例|列|篇|栋|注|亩|封|艘|把|目|套|段|人|所|朵|匹|张|座|回|场|尾|条|个|首|阙|阵|网|炮|顶|丘|棵|只|支|袭|辆|挑|担|颗|壳|窠|曲|墙|群|腔|砣|座|客|贯|扎|捆|刀|令|打|手|罗|坡|山|岭|江|溪|钟|队|单|双|对|出|口|头|脚|板|跳|枝|件|贴|针|线|管|名|位|身|堂|课|本|页|家|户|层|丝|毫|厘|分|钱|两|斤|担|铢|石|钧|锱|忽|(千|毫|微)克|毫|厘|(公)分|分|寸|尺|丈|里|寻|常|铺|程|(千|分|厘|毫|微)米|米|撮|勺|合|升|斗|石|盘|碗|碟|叠|桶|笼|盆|盒|杯|钟|斛|锅|簋|篮|盘|桶|罐|瓶|壶|卮|盏|箩|箱|煲|啖|袋|钵|年|月|日|季|刻|时|周|天|秒|分|小时|旬|纪|岁|世|更|夜|春|夏|秋|冬|代|伏|辈|丸|泡|粒|颗|幢|堆|条|根|支|道|面|片|张|颗|块|元|(亿|千万|百万|万|千|百)|(亿|千万|百万|万|千|百|美|)元|(亿|千万|百万|万|千|百|十|)吨|(亿|千万|百万|万|千|百|)块|角|毛|分)"
|
||||
|
||||
# 分数表达式
|
||||
RE_FRAC = re.compile(r'(-?)(\d+)/(\d+)')
|
||||
RE_FRAC = re.compile(r"(-?)(\d+)/(\d+)")
|
||||
|
||||
|
||||
def replace_frac(match) -> str:
|
||||
@@ -52,7 +55,7 @@ def replace_frac(match) -> str:
|
||||
|
||||
|
||||
# 百分数表达式
|
||||
RE_PERCENTAGE = re.compile(r'(-?)(\d+(\.\d+)?)%')
|
||||
RE_PERCENTAGE = re.compile(r"(-?)(\d+(\.\d+)?)%")
|
||||
|
||||
|
||||
def replace_percentage(match) -> str:
|
||||
@@ -72,7 +75,7 @@ def replace_percentage(match) -> str:
|
||||
|
||||
# 整数表达式
|
||||
# 带负号的整数 -10
|
||||
RE_INTEGER = re.compile(r'(-)' r'(\d+)')
|
||||
RE_INTEGER = re.compile(r"(-)" r"(\d+)")
|
||||
|
||||
|
||||
def replace_negative_num(match) -> str:
|
||||
@@ -92,7 +95,7 @@ def replace_negative_num(match) -> str:
|
||||
|
||||
# 编号-无符号整形
|
||||
# 00078
|
||||
RE_DEFAULT_NUM = re.compile(r'\d{3}\d*')
|
||||
RE_DEFAULT_NUM = re.compile(r"\d{3}\d*")
|
||||
|
||||
|
||||
def replace_default_num(match):
|
||||
@@ -110,15 +113,11 @@ def replace_default_num(match):
|
||||
# RE_ASMD = re.compile(
|
||||
# r'((-?)((\d+)(\.\d+)?)|(\.(\d+)))([\+\-\×÷=])((-?)((\d+)(\.\d+)?)|(\.(\d+)))')
|
||||
RE_ASMD = re.compile(
|
||||
r'((-?)((\d+)(\.\d+)?[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|(\.\d+[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|([A-Za-z][⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*))([\+\-\×÷=])((-?)((\d+)(\.\d+)?[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|(\.\d+[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|([A-Za-z][⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*))')
|
||||
r"((-?)((\d+)(\.\d+)?[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|(\.\d+[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|([A-Za-z][⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*))([\+\-\×÷=])((-?)((\d+)(\.\d+)?[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|(\.\d+[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*)|([A-Za-z][⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]*))"
|
||||
)
|
||||
|
||||
asmd_map = {"+": "加", "-": "减", "×": "乘", "÷": "除", "=": "等于"}
|
||||
|
||||
asmd_map = {
|
||||
'+': '加',
|
||||
'-': '减',
|
||||
'×': '乘',
|
||||
'÷': '除',
|
||||
'=': '等于'
|
||||
}
|
||||
|
||||
def replace_asmd(match) -> str:
|
||||
"""
|
||||
@@ -132,24 +131,25 @@ def replace_asmd(match) -> str:
|
||||
|
||||
|
||||
# 次方专项
|
||||
RE_POWER = re.compile(r'[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]+')
|
||||
RE_POWER = re.compile(r"[⁰¹²³⁴⁵⁶⁷⁸⁹ˣʸⁿ]+")
|
||||
|
||||
power_map = {
|
||||
'⁰': '0',
|
||||
'¹': '1',
|
||||
'²': '2',
|
||||
'³': '3',
|
||||
'⁴': '4',
|
||||
'⁵': '5',
|
||||
'⁶': '6',
|
||||
'⁷': '7',
|
||||
'⁸': '8',
|
||||
'⁹': '9',
|
||||
'ˣ': 'x',
|
||||
'ʸ': 'y',
|
||||
'ⁿ': 'n'
|
||||
"⁰": "0",
|
||||
"¹": "1",
|
||||
"²": "2",
|
||||
"³": "3",
|
||||
"⁴": "4",
|
||||
"⁵": "5",
|
||||
"⁶": "6",
|
||||
"⁷": "7",
|
||||
"⁸": "8",
|
||||
"⁹": "9",
|
||||
"ˣ": "x",
|
||||
"ʸ": "y",
|
||||
"ⁿ": "n",
|
||||
}
|
||||
|
||||
|
||||
def replace_power(match) -> str:
|
||||
"""
|
||||
Args:
|
||||
@@ -166,10 +166,10 @@ def replace_power(match) -> str:
|
||||
|
||||
# 数字表达式
|
||||
# 纯小数
|
||||
RE_DECIMAL_NUM = re.compile(r'(-?)((\d+)(\.\d+))' r'|(\.(\d+))')
|
||||
RE_DECIMAL_NUM = re.compile(r"(-?)((\d+)(\.\d+))" r"|(\.(\d+))")
|
||||
# 正整数 + 量词
|
||||
RE_POSITIVE_QUANTIFIERS = re.compile(r"(\d+)([多余几\+])?" + COM_QUANTIFIERS)
|
||||
RE_NUMBER = re.compile(r'(-?)((\d+)(\.\d+)?)' r'|(\.(\d+))')
|
||||
RE_NUMBER = re.compile(r"(-?)((\d+)(\.\d+)?)" r"|(\.(\d+))")
|
||||
|
||||
|
||||
def replace_positive_quantifier(match) -> str:
|
||||
@@ -220,7 +220,9 @@ RE_RANGE = re.compile(
|
||||
[-~] # 匹配范围分隔符
|
||||
((-?)((\d+)(\.\d+)?)) # 匹配范围结束的负数或正数（整数或小数）
|
||||
(?![\d\+\-\×÷=]) # 使用正向前瞻以确保数字范围之后没有其他数字和操作符
|
||||
""", re.VERBOSE)
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def replace_range(match) -> str:
|
||||
@@ -239,7 +241,9 @@ def replace_range(match) -> str:
|
||||
|
||||
# ~至表达式
|
||||
RE_TO_RANGE = re.compile(
|
||||
r'((-?)((\d+)(\.\d+)?)|(\.(\d+)))(%|°C|℃|度|摄氏度|cm2|cm²|cm3|cm³|cm|db|ds|kg|km|m2|m²|m³|m3|ml|m|mm|s)[~]((-?)((\d+)(\.\d+)?)|(\.(\d+)))(%|°C|℃|度|摄氏度|cm2|cm²|cm3|cm³|cm|db|ds|kg|km|m2|m²|m³|m3|ml|m|mm|s)')
|
||||
r"((-?)((\d+)(\.\d+)?)|(\.(\d+)))(%|°C|℃|度|摄氏度|cm2|cm²|cm3|cm³|cm|db|ds|kg|km|m2|m²|m³|m3|ml|m|mm|s)[~]((-?)((\d+)(\.\d+)?)|(\.(\d+)))(%|°C|℃|度|摄氏度|cm2|cm²|cm3|cm³|cm|db|ds|kg|km|m2|m²|m³|m3|ml|m|mm|s)"
|
||||
)
|
||||
|
||||
|
||||
def replace_to_range(match) -> str:
|
||||
"""
|
||||
@@ -248,71 +252,66 @@ def replace_to_range(match) -> str:
|
||||
Returns:
|
||||
str
|
||||
"""
|
||||
result = match.group(0).replace('~', '至')
|
||||
result = match.group(0).replace("~", "至")
|
||||
return result
|
||||
|
||||
|
||||
def _get_value(value_string: str, use_zero: bool=True) -> List[str]:
|
||||
stripped = value_string.lstrip('0')
|
||||
def _get_value(value_string: str, use_zero: bool = True) -> List[str]:
|
||||
stripped = value_string.lstrip("0")
|
||||
if len(stripped) == 0:
|
||||
return []
|
||||
elif len(stripped) == 1:
|
||||
if use_zero and len(stripped) < len(value_string):
|
||||
return [DIGITS['0'], DIGITS[stripped]]
|
||||
return [DIGITS["0"], DIGITS[stripped]]
|
||||
else:
|
||||
return [DIGITS[stripped]]
|
||||
else:
|
||||
largest_unit = next(
|
||||
power for power in reversed(UNITS.keys()) if power < len(stripped))
|
||||
largest_unit = next(power for power in reversed(UNITS.keys()) if power < len(stripped))
|
||||
first_part = value_string[:-largest_unit]
|
||||
second_part = value_string[-largest_unit:]
|
||||
return _get_value(first_part) + [UNITS[largest_unit]] + _get_value(
|
||||
second_part)
|
||||
return _get_value(first_part) + [UNITS[largest_unit]] + _get_value(second_part)
|
||||
|
||||
|
||||
def verbalize_cardinal(value_string: str) -> str:
|
||||
if not value_string:
|
||||
return ''
|
||||
return ""
|
||||
|
||||
# 000 -> '零' , 0 -> '零'
|
||||
value_string = value_string.lstrip('0')
|
||||
value_string = value_string.lstrip("0")
|
||||
if len(value_string) == 0:
|
||||
return DIGITS['0']
|
||||
return DIGITS["0"]
|
||||
|
||||
result_symbols = _get_value(value_string)
|
||||
# verbalized number starting with '一十*' is abbreviated as `十*`
|
||||
if len(result_symbols) >= 2 and result_symbols[0] == DIGITS[
|
||||
'1'] and result_symbols[1] == UNITS[1]:
|
||||
if len(result_symbols) >= 2 and result_symbols[0] == DIGITS["1"] and result_symbols[1] == UNITS[1]:
|
||||
result_symbols = result_symbols[1:]
|
||||
return ''.join(result_symbols)
|
||||
return "".join(result_symbols)
|
||||
|
||||
|
||||
def verbalize_digit(value_string: str, alt_one=False) -> str:
|
||||
result_symbols = [DIGITS[digit] for digit in value_string]
|
||||
result = ''.join(result_symbols)
|
||||
result = "".join(result_symbols)
|
||||
if alt_one:
|
||||
result = result.replace("一", "幺")
|
||||
return result
|
||||
|
||||
|
||||
def num2str(value_string: str) -> str:
|
||||
integer_decimal = value_string.split('.')
|
||||
integer_decimal = value_string.split(".")
|
||||
if len(integer_decimal) == 1:
|
||||
integer = integer_decimal[0]
|
||||
decimal = ''
|
||||
decimal = ""
|
||||
elif len(integer_decimal) == 2:
|
||||
integer, decimal = integer_decimal
|
||||
else:
|
||||
raise ValueError(
|
||||
f"The value string: '${value_string}' has more than one point in it."
|
||||
)
|
||||
raise ValueError(f"The value string: '${value_string}' has more than one point in it.")
|
||||
|
||||
result = verbalize_cardinal(integer)
|
||||
|
||||
decimal = decimal.rstrip('0')
|
||||
decimal = decimal.rstrip("0")
|
||||
if decimal:
|
||||
# '.22' is verbalized as '零点二二'
|
||||
# '3.20' is verbalized as '三点二
|
||||
result = result if result else "零"
|
||||
result += '点' + verbalize_digit(decimal)
|
||||
result += "点" + verbalize_digit(decimal)
|
||||
return result
|
||||
|
||||
@@ -21,10 +21,8 @@ from .num import verbalize_digit
|
||||
# 移动：139、138、137、136、135、134、159、158、157、150、151、152、188、187、182、183、184、178、198
|
||||
# 联通：130、131、132、156、155、186、185、176
|
||||
# 电信：133、153、189、180、181、177
|
||||
RE_MOBILE_PHONE = re.compile(
|
||||
r"(?<!\d)((\+?86 ?)?1([38]\d|5[0-35-9]|7[678]|9[89])\d{8})(?!\d)")
|
||||
RE_TELEPHONE = re.compile(
|
||||
r"(?<!\d)((0(10|2[1-3]|[3-9]\d{2})-?)?[1-9]\d{6,7})(?!\d)")
|
||||
RE_MOBILE_PHONE = re.compile(r"(?<!\d)((\+?86 ?)?1([38]\d|5[0-35-9]|7[678]|9[89])\d{8})(?!\d)")
|
||||
RE_TELEPHONE = re.compile(r"(?<!\d)((0(10|2[1-3]|[3-9]\d{2})-?)?[1-9]\d{6,7})(?!\d)")
|
||||
|
||||
# 全国统一的号码400开头
|
||||
RE_NATIONAL_UNIFORM_NUMBER = re.compile(r"(400)(-)?\d{3}(-)?\d{4}")
|
||||
@@ -32,14 +30,12 @@ RE_NATIONAL_UNIFORM_NUMBER = re.compile(r"(400)(-)?\d{3}(-)?\d{4}")
|
||||
|
||||
def phone2str(phone_string: str, mobile=True) -> str:
|
||||
if mobile:
|
||||
sp_parts = phone_string.strip('+').split()
|
||||
result = '，'.join(
|
||||
[verbalize_digit(part, alt_one=True) for part in sp_parts])
|
||||
sp_parts = phone_string.strip("+").split()
|
||||
result = "，".join([verbalize_digit(part, alt_one=True) for part in sp_parts])
|
||||
return result
|
||||
else:
|
||||
sil_parts = phone_string.split('-')
|
||||
result = '，'.join(
|
||||
[verbalize_digit(part, alt_one=True) for part in sil_parts])
|
||||
sil_parts = phone_string.split("-")
|
||||
result = "，".join([verbalize_digit(part, alt_one=True) for part in sil_parts])
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from .num import num2str
|
||||
|
||||
# 温度表达式，温度会影响负号的读法
|
||||
# -3°C 零下三度
|
||||
RE_TEMPERATURE = re.compile(r'(-?)(\d+(\.\d+)?)(°C|℃|度|摄氏度)')
|
||||
RE_TEMPERATURE = re.compile(r"(-?)(\d+(\.\d+)?)(°C|℃|度|摄氏度)")
|
||||
measure_dict = {
|
||||
"cm2": "平方厘米",
|
||||
"cm²": "平方厘米",
|
||||
@@ -35,7 +35,7 @@ measure_dict = {
|
||||
"ml": "毫升",
|
||||
"m": "米",
|
||||
"mm": "毫米",
|
||||
"s": "秒"
|
||||
"s": "秒",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -56,9 +56,9 @@ from .quantifier import replace_measure
|
||||
from .quantifier import replace_temperature
|
||||
|
||||
|
||||
class TextNormalizer():
|
||||
class TextNormalizer:
|
||||
def __init__(self):
|
||||
self.SENTENCE_SPLITOR = re.compile(r'([：、，；。？！,;?!][”’]?)')
|
||||
self.SENTENCE_SPLITOR = re.compile(r"([：、，；。？！,;?!][”’]?)")
|
||||
|
||||
def _split(self, text: str, lang="zh") -> List[str]:
|
||||
"""Split long text into sentences with sentence-splitting punctuations.
|
||||
@@ -71,66 +71,64 @@ class TextNormalizer():
|
||||
if lang == "zh":
|
||||
text = text.replace(" ", "")
|
||||
# 过滤掉特殊字符
|
||||
text = re.sub(r'[——《》【】<>{}()（）#&@“”^_|\\]', '', text)
|
||||
text = self.SENTENCE_SPLITOR.sub(r'\1\n', text)
|
||||
text = re.sub(r"[——《》【】<>{}()（）#&@“”^_|\\]", "", text)
|
||||
text = self.SENTENCE_SPLITOR.sub(r"\1\n", text)
|
||||
text = text.strip()
|
||||
sentences = [sentence.strip() for sentence in re.split(r'\n+', text)]
|
||||
sentences = [sentence.strip() for sentence in re.split(r"\n+", text)]
|
||||
return sentences
|
||||
|
||||
def _post_replace(self, sentence: str) -> str:
|
||||
sentence = sentence.replace('/', '每')
|
||||
sentence = sentence.replace("/", "每")
|
||||
# sentence = sentence.replace('~', '至')
|
||||
# sentence = sentence.replace('～', '至')
|
||||
sentence = sentence.replace('①', '一')
|
||||
sentence = sentence.replace('②', '二')
|
||||
sentence = sentence.replace('③', '三')
|
||||
sentence = sentence.replace('④', '四')
|
||||
sentence = sentence.replace('⑤', '五')
|
||||
sentence = sentence.replace('⑥', '六')
|
||||
sentence = sentence.replace('⑦', '七')
|
||||
sentence = sentence.replace('⑧', '八')
|
||||
sentence = sentence.replace('⑨', '九')
|
||||
sentence = sentence.replace('⑩', '十')
|
||||
sentence = sentence.replace('α', '阿尔法')
|
||||
sentence = sentence.replace('β', '贝塔')
|
||||
sentence = sentence.replace('γ', '伽玛').replace('Γ', '伽玛')
|
||||
sentence = sentence.replace('δ', '德尔塔').replace('Δ', '德尔塔')
|
||||
sentence = sentence.replace('ε', '艾普西龙')
|
||||
sentence = sentence.replace('ζ', '捷塔')
|
||||
sentence = sentence.replace('η', '依塔')
|
||||
sentence = sentence.replace('θ', '西塔').replace('Θ', '西塔')
|
||||
sentence = sentence.replace('ι', '艾欧塔')
|
||||
sentence = sentence.replace('κ', '喀帕')
|
||||
sentence = sentence.replace('λ', '拉姆达').replace('Λ', '拉姆达')
|
||||
sentence = sentence.replace('μ', '缪')
|
||||
sentence = sentence.replace('ν', '拗')
|
||||
sentence = sentence.replace('ξ', '克西').replace('Ξ', '克西')
|
||||
sentence = sentence.replace('ο', '欧米克伦')
|
||||
sentence = sentence.replace('π', '派').replace('Π', '派')
|
||||
sentence = sentence.replace('ρ', '肉')
|
||||
sentence = sentence.replace('ς', '西格玛').replace('Σ', '西格玛').replace(
|
||||
'σ', '西格玛')
|
||||
sentence = sentence.replace('τ', '套')
|
||||
sentence = sentence.replace('υ', '宇普西龙')
|
||||
sentence = sentence.replace('φ', '服艾').replace('Φ', '服艾')
|
||||
sentence = sentence.replace('χ', '器')
|
||||
sentence = sentence.replace('ψ', '普赛').replace('Ψ', '普赛')
|
||||
sentence = sentence.replace('ω', '欧米伽').replace('Ω', '欧米伽')
|
||||
sentence = sentence.replace("①", "一")
|
||||
sentence = sentence.replace("②", "二")
|
||||
sentence = sentence.replace("③", "三")
|
||||
sentence = sentence.replace("④", "四")
|
||||
sentence = sentence.replace("⑤", "五")
|
||||
sentence = sentence.replace("⑥", "六")
|
||||
sentence = sentence.replace("⑦", "七")
|
||||
sentence = sentence.replace("⑧", "八")
|
||||
sentence = sentence.replace("⑨", "九")
|
||||
sentence = sentence.replace("⑩", "十")
|
||||
sentence = sentence.replace("α", "阿尔法")
|
||||
sentence = sentence.replace("β", "贝塔")
|
||||
sentence = sentence.replace("γ", "伽玛").replace("Γ", "伽玛")
|
||||
sentence = sentence.replace("δ", "德尔塔").replace("Δ", "德尔塔")
|
||||
sentence = sentence.replace("ε", "艾普西龙")
|
||||
sentence = sentence.replace("ζ", "捷塔")
|
||||
sentence = sentence.replace("η", "依塔")
|
||||
sentence = sentence.replace("θ", "西塔").replace("Θ", "西塔")
|
||||
sentence = sentence.replace("ι", "艾欧塔")
|
||||
sentence = sentence.replace("κ", "喀帕")
|
||||
sentence = sentence.replace("λ", "拉姆达").replace("Λ", "拉姆达")
|
||||
sentence = sentence.replace("μ", "缪")
|
||||
sentence = sentence.replace("ν", "拗")
|
||||
sentence = sentence.replace("ξ", "克西").replace("Ξ", "克西")
|
||||
sentence = sentence.replace("ο", "欧米克伦")
|
||||
sentence = sentence.replace("π", "派").replace("Π", "派")
|
||||
sentence = sentence.replace("ρ", "肉")
|
||||
sentence = sentence.replace("ς", "西格玛").replace("Σ", "西格玛").replace("σ", "西格玛")
|
||||
sentence = sentence.replace("τ", "套")
|
||||
sentence = sentence.replace("υ", "宇普西龙")
|
||||
sentence = sentence.replace("φ", "服艾").replace("Φ", "服艾")
|
||||
sentence = sentence.replace("χ", "器")
|
||||
sentence = sentence.replace("ψ", "普赛").replace("Ψ", "普赛")
|
||||
sentence = sentence.replace("ω", "欧米伽").replace("Ω", "欧米伽")
|
||||
# 兜底数学运算，顺便兼容懒人用语
|
||||
sentence = sentence.replace('+', '加')
|
||||
sentence = sentence.replace('-', '减')
|
||||
sentence = sentence.replace('×', '乘')
|
||||
sentence = sentence.replace('÷', '除')
|
||||
sentence = sentence.replace('=', '等')
|
||||
sentence = sentence.replace("+", "加")
|
||||
sentence = sentence.replace("-", "减")
|
||||
sentence = sentence.replace("×", "乘")
|
||||
sentence = sentence.replace("÷", "除")
|
||||
sentence = sentence.replace("=", "等")
|
||||
# re filter special characters, have one more character "-" than line 68
|
||||
sentence = re.sub(r'[-——《》【】<=>{}()（）#&@“”^_|\\]', '', sentence)
|
||||
sentence = re.sub(r"[-——《》【】<=>{}()（）#&@“”^_|\\]", "", sentence)
|
||||
return sentence
|
||||
|
||||
def normalize_sentence(self, sentence: str) -> str:
|
||||
# basic character conversions
|
||||
sentence = tranditional_to_simplified(sentence)
|
||||
sentence = sentence.translate(F2H_ASCII_LETTERS).translate(
|
||||
F2H_DIGITS).translate(F2H_SPACE)
|
||||
sentence = sentence.translate(F2H_ASCII_LETTERS).translate(F2H_DIGITS).translate(F2H_SPACE)
|
||||
|
||||
# number related NSW verbalization
|
||||
sentence = RE_DATE.sub(replace_date, sentence)
|
||||
@@ -161,8 +159,7 @@ class TextNormalizer():
|
||||
|
||||
sentence = RE_INTEGER.sub(replace_negative_num, sentence)
|
||||
sentence = RE_DECIMAL_NUM.sub(replace_number, sentence)
|
||||
sentence = RE_POSITIVE_QUANTIFIERS.sub(replace_positive_quantifier,
|
||||
sentence)
|
||||
sentence = RE_POSITIVE_QUANTIFIERS.sub(replace_positive_quantifier, sentence)
|
||||
sentence = RE_DEFAULT_NUM.sub(replace_default_num, sentence)
|
||||
sentence = RE_NUMBER.sub(replace_number, sentence)
|
||||
sentence = self._post_replace(sentence)
|
||||
|
||||
Reference in New Issue
Block a user