diff --git a/itn/main.py b/itn/main.py index 430fb5a..eb8131b 100644 --- a/itn/main.py +++ b/itn/main.py @@ -14,19 +14,9 @@ import argparse -# TODO(xcsong): multi-language support from itn.chinese.inverse_normalizer import InverseNormalizer as ZhInverseNormalizer from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer - - -def str2bool(s, default=False): - s = s.lower() - if s == "true": - return True - elif s == "false": - return False - else: - return default +from tn.utils import str2bool def main(): @@ -62,7 +52,7 @@ def main(): print(normalizer.tag(args.text)) print(normalizer.normalize(args.text)) elif args.file: - with open(args.file) as fin: + with open(args.file, encoding="utf-8") as fin: for line in fin: print(normalizer.tag(line.strip())) print(normalizer.normalize(line.strip())) diff --git a/tn/chinese/data/cardinal/phone.tsv b/tn/chinese/data/cardinal/phone.tsv new file mode 100644 index 0000000..3d20f7a --- /dev/null +++ b/tn/chinese/data/cardinal/phone.tsv @@ -0,0 +1,29 @@ +110 幺幺零 +111 幺幺幺 +112 幺幺二 +114 幺幺四 +119 幺幺九 +120 幺二零 +122 幺二二 +10086 幺零零八六 +10010 幺零零幺零 +10000 幺零零零零 +12306 幺二三零六 +12315 幺二三幺五 +12345 幺二三四五 +12110 幺二幺幺零 +12320 幺二三二零 +12366 幺二三六六 +12333 幺二三三三 +12328 幺二三二八 +95110 九五幺幺零 +95120 九五幺二零 +95188 九五幺八八 +95500 九五五零零 +95511 九五五幺幺 +95533 九五五三三 +95555 九五五五五 +95566 九五五六六 +95588 九五五八八 +95598 九五五九八 +95599 九五五九九 diff --git a/tn/chinese/rules/cardinal.py b/tn/chinese/rules/cardinal.py index f747884..820783a 100644 --- a/tn/chinese/rules/cardinal.py +++ b/tn/chinese/rules/cardinal.py @@ -91,9 +91,10 @@ def build_tagger(self): cardinal |= digits.plus + (delete("-") + digits.plus) ** 2 # xxx-xxxxxxxx cardinal |= digits**3 + delete("-") + digits**8 - # three or five or eleven phone numbers + # well-known short phone numbers (110, 12306, etc.) and 11-digit mobile phone_digits = digits @ self.build_rule(cross("一", "幺")) - phone = phone_digits**3 | phone_digits**5 | phone_digits**11 + phone = string_file(get_abs_path("chinese/data/cardinal/phone.tsv")) + phone |= phone_digits**11 phone |= accep("尾号") + (accep("是") | accep("为")).ques + phone_digits**4 cardinal |= add_weight(phone, -1.0) diff --git a/tn/main.py b/tn/main.py index 874c0c1..26f7e45 100644 --- a/tn/main.py +++ b/tn/main.py @@ -14,7 +14,7 @@ import argparse -from itn.main import str2bool +from tn.utils import str2bool # TODO(pzd17 & sxc19): multi-language support from tn.chinese.normalizer import Normalizer as ZhNormalizer @@ -63,7 +63,7 @@ def main(): print(normalizer.tag(args.text)) print(normalizer.normalize(args.text)) elif args.file: - with open(args.file) as fin: + with open(args.file, encoding="utf-8") as fin: for line in fin: print(normalizer.tag(line.strip())) print(normalizer.normalize(line.strip())) diff --git a/tn/utils.py b/tn/utils.py index 8a66012..7dc5dd1 100644 --- a/tn/utils.py +++ b/tn/utils.py @@ -82,3 +82,13 @@ def get_formats(input_f, input_case="cased", is_default=True): multiple_formats = pynini.string_map(multiple_formats) return multiple_formats + + +def str2bool(s, default=False): + s = s.lower() + if s == "true": + return True + elif s == "false": + return False + else: + return default