feat: add 亿级 support and strip commas for Chinese TN

tn/chinese/rules/cardinal.py

@@ -36,35 +36,42 @@ def build_tagger(self):
         dot = string_file(get_abs_path("chinese/data/number/dot.tsv"))
 
         rmzero = delete("0") | delete("０")
-        rmpunct = delete(",").ques
         digits = zero | digit
         self.digits = digits
 
-        # 11 => 十一
+        # 11 => 十一, 10 => 十
         ten = teen + insert("十") + (digit | rmzero)
-        # 11 => 一十一
+        # 21 => 二十一, 30 => 三十
         tens = digit + insert("十") + (digit | rmzero)
-        # 111, 101, 100
+        # 111 => 一百一十一, 101 => 一百零一, 100 => 一百
         hundred = digit + insert("百") + (tens | (zero + digit) | rmzero**2)
-        # 1111, 1011, 1001, 1000
-        thousand = digit + insert("千") + rmpunct + (hundred | (zero + tens) | (rmzero + zero + digit) | rmzero**3)
-        # 10001111, 1001111, 101111, 11111, 10111, 10011, 10001, 10000
+        # 1111 => 一千一百一十一, 1001 => 一千零一, 1000 => 一千
+        thousand = digit + insert("千") + (hundred | (zero + tens) | (rmzero + zero + digit) | rmzero**3)
+
+        # exactly 4 input digits, not all zero
+        four_nonzero = thousand | (zero + hundred) | (rmzero + zero + tens) | (rmzero**2 + zero + digit)
+        # exactly 4 input digits, may be all zero
+        four_any = four_nonzero | rmzero**4
+
+        # 万级: 1,0000 ~ 9999,9999 (5~8位)
         ten_thousand = (
             (thousand | hundred | ten | digit)
             + insert("万")
-            + (
-                thousand
-                | (zero + rmpunct + hundred)
-                | (rmzero + rmpunct + zero + tens)
-                | (rmzero + rmpunct + rmzero + zero + digit)
-                | rmzero**4
-            )
+            + four_any
+        )
+
+        # 亿级: 1,0000,0000 ~ 9999,9999,9999 (9~13位)
+        hundred_million = (
+            (thousand | hundred | ten | digit)
+            + insert("亿")
+            + (four_nonzero + insert("万") + four_any | rmzero**4 + four_nonzero | rmzero**8)
         )
 
-        # 1.11, 1.01
-        number = digits | ten | hundred | thousand | ten_thousand
+        number = digits | ten | hundred | thousand | ten_thousand | hundred_million
         number = sign.ques + number + (dot + digits.plus).ques
-        number @= self.build_rule(cross("二百", "两百") | cross("二千", "两千") | cross("二万", "两万"), "[BOS]").optimize()
+        number @= self.build_rule(
+            cross("二百", "两百") | cross("二千", "两千") | cross("二万", "两万") | cross("二亿", "两亿"), "[BOS]"
+        ).optimize()
         percent = insert("百分之") + number + delete("%")
         self.number = accep("约").ques + accep("人均").ques + (number | percent)
 

tn/chinese/rules/measure.py

@@ -36,6 +36,8 @@ def build_tagger(self):
         to = cross("-", "到") | cross("~", "到") | accep("到")
 
         number = self.cardinal.number
+        strip_comma = self.build_rule(delete(","), self.DIGIT, self.DIGIT)
+        number = strip_comma @ number
         number @= self.build_rule(cross("二", "两"), "[BOS]", "[EOS]")
         # 1-11个，1个-11个
         prefix = number + (rmspace + units).ques + to