feat: add English ITN extended rules (time, whitelist)

itn/english/rules/date.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini import closure, cross, string_file
+from pynini.lib.pynutil import add_weight, delete, insert
+
+from itn.english.rules.cardinal import Cardinal
+from itn.english.rules.ordinal import Ordinal
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Date(Processor):
+
+    def __init__(self, cardinal=None, ordinal=None):
+        super().__init__(name="date", ordertype="itn")
+        self.cardinal = cardinal or Cardinal()
+        self.ordinal = ordinal or Ordinal(cardinal=self.cardinal)
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        ds = delete(" ")
+        month_names = string_file(get_abs_path("../itn/english/data/months.tsv"))
+        month = insert('month: "') + month_names + insert('"')
+
+        # Day: accept ordinal words ("fifth", "twenty first") or cardinal
+        # words ("thirty") -- both resolve to a number via the cardinal graph.
+        # Restrict to 1-31 range via composition with DIGIT{1,2}.
+        day_graph = self.ordinal.graph | self.cardinal.graph
+        day_graph = pynini.compose(day_graph, self.DIGIT + closure(self.DIGIT, 0, 1))
+        day = insert('day: "') + day_graph + insert('"')
+
+        # Year graph: handles common spoken year forms
+        digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
+        teen = string_file(get_abs_path("../itn/english/data/numbers/teen.tsv"))
+        ties = string_file(get_abs_path("../itn/english/data/numbers/ties.tsv"))
+
+        # Two-digit: 10-99
+        two_digit = teen | (ties + (ds + digit | insert("0")))
+
+        # "oh five" / "o five" => 05
+        oh_digit = (cross("oh", "0") | cross("o", "0")) + ds + digit
+
+        # Year as two groups of two digits: "twenty twelve" => 2012
+        year_two_parts = (teen | two_digit) + ds + (two_digit | oh_digit | teen)
+
+        # Year as "X thousand Y": "two thousand twelve" => 2012
+        # Need zero-padded variants so "two thousand three" => 2003
+        hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00"))
+        up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit
+        year_thousands = (
+            digit
+            + ds
+            + delete("thousand")
+            + (ds + up_to_999_padded | insert("000"))
+        )
+
+        # Year as hundreds: "nineteen oh five" => 1905
+        year_hundreds = (teen | two_digit) + ds + oh_digit
+
+        year_graph = year_two_parts | year_thousands | year_hundreds
+
+        # Delete optional "and" within year
+        delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
+        year_graph = (delete_and @ year_graph).optimize()
+
+        year = insert('year: "') + year_graph + insert('"')
+
+        # Marker to preserve field order through TokenParser
+        po = insert(' preserve_order: "true"')
+
+        # Format: month day year => "july twenty fifth two thousand twelve"
+        graph_mdy = month + ds + insert(" ") + day + ds + insert(" ") + year + po
+        # Format: month day (no year) => "january first"
+        graph_md = month + ds + insert(" ") + day + po
+        # Format: month year (no day) => "july two thousand twelve"
+        graph_my = month + ds + insert(" ") + add_weight(year, -0.1) + po
+        # Format: "the day of month year" => "the twenty fifth of july twenty twelve"
+        graph_dmy = (
+            delete("the")
+            + ds
+            + day
+            + ds
+            + delete("of")
+            + ds
+            + insert(" ")
+            + month
+            + ds
+            + insert(" ")
+            + year
+            + po
+        )
+        # Format: "the day of month" (no year) => "the fifteenth of january"
+        graph_dm = (
+            delete("the")
+            + ds
+            + day
+            + ds
+            + delete("of")
+            + ds
+            + insert(" ")
+            + month
+            + po
+        )
+        # Year only => "twenty twelve", "two thousand three"
+        graph_y = add_weight(year, 0.01) + po
+
+        final_graph = graph_mdy | graph_md | graph_my | graph_dmy | graph_dm | graph_y
+        self.tagger = self.add_tokens(final_graph)
+
+    def build_verbalizer(self):
+        month = (
+            delete("month:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        day = (
+            delete("day:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        year = (
+            delete("year:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        delete_po = (
+            delete("preserve_order:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + delete("true")
+            + delete('"')
+        )
+
+        optional_day = closure(self.DELETE_SPACE + insert(" ") + day, 0, 1)
+        optional_year = closure(self.DELETE_SPACE + insert(" ") + year, 0, 1)
+
+        # month (day) (year)
+        graph_mdy = month + optional_day + optional_year
+        # day month (year)
+        graph_dmy = day + self.DELETE_SPACE + insert(" ") + month + optional_year
+        # year only
+        graph_y = year
+
+        graph = (graph_mdy | graph_dmy | graph_y) + self.DELETE_SPACE + delete_po
+        self.verbalizer = self.delete_tokens(graph)

itn/english/rules/electronic.py

@@ -0,0 +1,117 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini import closure, cross, invert, string_file
+from pynini.lib.pynutil import add_weight, delete, insert
+
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Electronic(Processor):
+
+    def __init__(self):
+        super().__init__(name="electronic", ordertype="itn")
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        ds = delete(" ")
+
+        # Single characters: digits and letters
+        digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
+        zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
+        alpha_or_digit = self.ALPHA | digit | zero
+
+        # Symbols from TSV (symbol\tname): invert to get name -> symbol
+        symbols = invert(
+            string_file(get_abs_path("../itn/english/data/electronic/symbols.tsv"))
+        )
+
+        # A "token" is either a single char (letter/digit/symbol) or a
+        # multi-letter word kept verbatim (e.g. "gmail", "nvidia").
+        # Multi-letter words have lower priority so spelled-out letters are preferred.
+        word = add_weight(closure(self.ALPHA, 2), 0.01)
+        token = alpha_or_digit | symbols | word
+
+        # A component is one or more tokens separated by spaces
+        component = token + closure(ds + token)
+
+        username = insert('username: "') + component + insert('"')
+
+        # Domain: component(s) separated by "dot" => "."
+        dot = cross("dot", ".")
+        domain_content = component + closure(ds + dot + ds + component)
+        domain = insert('domain: "') + domain_content + insert('"')
+
+        # Email: username at domain
+        graph_email = (
+            username
+            + ds
+            + delete("at")
+            + ds
+            + insert(" ")
+            + domain
+        )
+
+        # URL protocol: "h t t p colon slash slash" or "h t t p s colon slash slash"
+        http = cross("h t t p", "http")
+        https = cross("h t t p s", "https")
+        colon_slash_slash = cross(" colon slash slash ", "://")
+        protocol_start = (http | https) + colon_slash_slash
+
+        # www prefix
+        www = cross("w w w", "www")
+
+        # URL: [protocol] [www.] domain
+        url_content = (
+            closure(protocol_start, 0, 1)
+            + closure(www + ds + dot + ds, 0, 1)
+            + domain_content
+        )
+        graph_url = insert('protocol: "') + url_content + insert('"')
+
+        final_graph = graph_email | graph_url
+        self.tagger = self.add_tokens(final_graph)
+
+    def build_verbalizer(self):
+        username = (
+            delete("username:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        domain = (
+            delete("domain:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        protocol = (
+            delete("protocol:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+
+        # Email: username@domain
+        graph_email = username + self.DELETE_SPACE + insert("@") + domain
+        # URL: just output the protocol content directly
+        graph_url = protocol
+
+        graph = graph_email | graph_url
+        self.verbalizer = self.delete_tokens(graph)

itn/english/rules/measure.py

@@ -0,0 +1,99 @@
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini import closure, cross, invert, string_file
+from pynini.lib.pynutil import delete, insert
+
+from itn.english.rules.cardinal import Cardinal
+from itn.english.rules.decimal import Decimal
+from tn.processor import Processor
+from tn.utils import get_abs_path
+
+
+class Measure(Processor):
+
+    def __init__(self, cardinal=None, decimal=None):
+        super().__init__(name="measure", ordertype="itn")
+        self.cardinal = cardinal or Cardinal()
+        self.decimal = decimal or Decimal(cardinal=self.cardinal)
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        ds = delete(" ")
+
+        # Load measurements: symbol\tname, invert to get name -> symbol
+        units_graph = invert(
+            string_file(get_abs_path("../itn/english/data/measurements.tsv"))
+        )
+
+        # Handle plurals: strip trailing "s" to match singular form
+        # e.g. "meters" -> "meter" -> "m", "kilograms" -> "kilogram" -> "kg"
+        depluralize = pynini.cdrewrite(
+            cross("s", ""), "", "[EOS]", self.VSIGMA
+        )
+        # Handle irregular plurals: "feet" -> "foot"
+        irregular = pynini.string_map([("feet", "foot")])
+        unit_singular = units_graph
+        unit_plural = (depluralize | irregular) @ units_graph
+
+        unit = unit_singular | unit_plural
+
+        # Handle "per" units: "per hour" -> "/h"
+        per_unit = insert("/") + delete("per") + ds + unit_singular
+        full_unit = unit + closure(ds + per_unit, 0, 1) | per_unit
+
+        # Cardinal value
+        cardinal_value = self.cardinal.graph
+
+        # Decimal value (reuse decimal's internal graph for the number)
+        decimal_digit = string_file(get_abs_path("../itn/english/data/numbers/digit.tsv"))
+        decimal_zero = string_file(get_abs_path("../itn/english/data/numbers/zero.tsv"))
+        frac_digit = decimal_digit | decimal_zero | cross("o", "0")
+        frac_graph = closure(frac_digit + ds) + frac_digit
+        decimal_value = cardinal_value + ds + delete("point") + ds + insert(".") + frac_graph
+
+        # Optional minus/negative prefix
+        minus = delete("minus") | delete("negative")
+        optional_sign = closure(insert("-") + minus + ds, 0, 1)
+
+        # "point X" with no integer part
+        point_only = delete("point") + ds + insert(".") + frac_graph
+
+        number = optional_sign + (decimal_value | cardinal_value | point_only)
+
+        value = insert('value: "') + number + insert('"')
+        units = insert('units: "') + full_unit + insert('"')
+
+        final_graph = value + ds + insert(" ") + units
+        self.tagger = self.add_tokens(final_graph)
+
+    def build_verbalizer(self):
+        value = (
+            delete("value:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        units = (
+            delete("units:")
+            + self.DELETE_SPACE
+            + delete('"')
+            + self.NOT_QUOTE.plus
+            + delete('"')
+        )
+        graph = value + self.DELETE_SPACE + insert(" ") + units
+        self.verbalizer = self.delete_tokens(graph)