refactor: fix encoding and move str2bool to shared utils

itn/main.py

@@ -14,19 +14,9 @@
 
 import argparse
 
-# TODO(xcsong): multi-language support
 from itn.chinese.inverse_normalizer import InverseNormalizer as ZhInverseNormalizer
 from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer
-
-
-def str2bool(s, default=False):
-    s = s.lower()
-    if s == "true":
-        return True
-    elif s == "false":
-        return False
-    else:
-        return default
+from tn.utils import str2bool
 
 
 def main():
@@ -62,7 +52,7 @@ def main():
         print(normalizer.tag(args.text))
         print(normalizer.normalize(args.text))
     elif args.file:
-        with open(args.file) as fin:
+        with open(args.file, encoding="utf-8") as fin:
             for line in fin:
                 print(normalizer.tag(line.strip()))
                 print(normalizer.normalize(line.strip()))

tn/chinese/data/cardinal/phone.tsv

@@ -0,0 +1,29 @@
+110	幺幺零
+111	幺幺幺
+112	幺幺二
+114	幺幺四
+119	幺幺九
+120	幺二零
+122	幺二二
+10086	幺零零八六
+10010	幺零零幺零
+10000	幺零零零零
+12306	幺二三零六
+12315	幺二三幺五
+12345	幺二三四五
+12110	幺二幺幺零
+12320	幺二三二零
+12366	幺二三六六
+12333	幺二三三三
+12328	幺二三二八
+95110	九五幺幺零
+95120	九五幺二零
+95188	九五幺八八
+95500	九五五零零
+95511	九五五幺幺
+95533	九五五三三
+95555	九五五五五
+95566	九五五六六
+95588	九五五八八
+95598	九五五九八
+95599	九五五九九

tn/chinese/rules/cardinal.py

@@ -91,9 +91,10 @@ def build_tagger(self):
         cardinal |= digits.plus + (delete("-") + digits.plus) ** 2
         # xxx-xxxxxxxx
         cardinal |= digits**3 + delete("-") + digits**8
-        # three or five or eleven phone numbers
+        # well-known short phone numbers (110, 12306, etc.) and 11-digit mobile
         phone_digits = digits @ self.build_rule(cross("一", "幺"))
-        phone = phone_digits**3 | phone_digits**5 | phone_digits**11
+        phone = string_file(get_abs_path("chinese/data/cardinal/phone.tsv"))
+        phone |= phone_digits**11
         phone |= accep("尾号") + (accep("是") | accep("为")).ques + phone_digits**4
         cardinal |= add_weight(phone, -1.0)
 

tn/main.py

@@ -14,7 +14,7 @@
 
 import argparse
 
-from itn.main import str2bool
+from tn.utils import str2bool
 
 # TODO(pzd17 & sxc19): multi-language support
 from tn.chinese.normalizer import Normalizer as ZhNormalizer
@@ -63,7 +63,7 @@ def main():
         print(normalizer.tag(args.text))
         print(normalizer.normalize(args.text))
     elif args.file:
-        with open(args.file) as fin:
+        with open(args.file, encoding="utf-8") as fin:
             for line in fin:
                 print(normalizer.tag(line.strip()))
                 print(normalizer.normalize(line.strip()))

tn/utils.py

@@ -82,3 +82,13 @@ def get_formats(input_f, input_case="cased", is_default=True):
 
     multiple_formats = pynini.string_map(multiple_formats)
     return multiple_formats
+
+
+def str2bool(s, default=False):
+    s = s.lower()
+    if s == "true":
+        return True
+    elif s == "false":
+        return False
+    else:
+        return default