Enhanced baseline and additional corpus improvements

.gitignore

@@ -1,6 +1,8 @@
 /src/data/extracted
 /src/data/**.bz2
-/src/data/**.jsonl
+/src/data/syriac**.jsonl
+/src/data/sedra**.jsonl
+/src/data/corpus**.jsonl
 /src/data/**.txt
 /logs/**
 /checkpoints/**

README.md

@@ -22,16 +22,16 @@ model-index:
       metrics:
         - name: Training Loss
           type: loss
-          value: 3.3484
+          value: 1.5250
         - name: Evaluation Loss
           type: loss
-          value: 5.3017
+          value: 3.8090
         - name: CER
           type: cer
-          value: 0.6500
+          value: 0.4829
         - name: Exact Match
           type: accuracy
-          value: 0.0220
+          value: 0.0725
 ---
 # AramT5 - T5 Fine-Tuned on Syriac-to-Latin Transliteration ♰
 

config.json

@@ -25,5 +25,5 @@
   "torch_dtype": "float32",
   "transformers_version": "4.51.2",
   "use_cache": true,
-  "vocab_size": 40100
+  "vocab_size": 16100
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc4646b05b0de4d7f5e8b3f6c0cac6a1d63dce0891ff2b8e6e1f6ccd55e39baf
-size 258368552
+oid sha256:f8b0e1794702889a150c716d691249f91bb4726bcffe0190cb1f33a2f478254f
+size 209216552

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:347fef5a38453d31d013802c803c9d91f9333fefacffc922784af95f94202f4b
-size 988532
+oid sha256:02090247a3d32c0cb86d464007c80682b2dcf3e0619b42e826dbce248d884dad
+size 511338

src/data/build_corpus_vocabulary.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""
+Build corpus vocabulary from pointed Syriac text.
+
+Extracts vocalised words from corpus files (TEI, Bible, etc.) and creates
+a lookup table for transliteration. This supplements SEDRA data with
+additional vocabulary found in the corpus.
+
+Usage:
+    python build_corpus_vocabulary.py [--output corpus_vocabulary.jsonl]
+
+The output file is loaded by sedra_lookup.py as a fallback vocabulary source.
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+
+def strip_vowels(word: str) -> str:
+    """Remove vowel diacritics to get unpointed form."""
+    result = []
+    for char in word:
+        code = ord(char)
+        # Skip vowels (0730-073F)
+        if 0x0730 <= code <= 0x073F:
+            continue
+        # Skip other diacritics (0740-074A)
+        if 0x0740 <= code <= 0x074A:
+            continue
+        result.append(char)
+    return "".join(result)
+
+
+def count_standard_vowels(word: str) -> int:
+    """Count standard vowel diacritics (U+0730-073F)."""
+    return sum(1 for c in word if 0x0730 <= ord(c) <= 0x073F)
+
+
+def has_standard_vowels(word: str) -> bool:
+    """Check if word has any standard vowels."""
+    return any(0x0730 <= ord(c) <= 0x073F for c in word)
+
+
+def build_vocabulary(corpus_files: list[Path], output_path: Path) -> int:
+    """
+    Build vocabulary from corpus files.
+    
+    Args:
+        corpus_files: List of corpus file paths to process
+        output_path: Output JSONL file path
+        
+    Returns:
+        Number of vocabulary entries created
+    """
+    from generate_syr_lat_pairs import (
+        has_vowel_diacritics,
+        extract_syriac_words,
+        transliterate_word_pointed,
+        SyriacDialect,
+    )
+
+    # Collect all pointed words from all corpus files
+    all_pointed_words = []
+    for corpus_file in corpus_files:
+        if not corpus_file.exists():
+            print(f"  Skipping {corpus_file.name} (not found)")
+            continue
+            
+        print(f"  Reading {corpus_file.name}...")
+        with open(corpus_file, "r", encoding="utf-8") as f:
+            content = f.read()
+
+        words = extract_syriac_words(content)
+        pointed = [w for w in words if has_vowel_diacritics(w)]
+        all_pointed_words.extend(pointed)
+        print(f"    Found {len(pointed)} pointed words")
+
+    print(f"\nTotal pointed words: {len(all_pointed_words)}")
+
+    # Build vocabulary, preferring forms with more standard vowels
+    vocab: dict[str, dict] = {}
+    for pw in all_pointed_words:
+        unpointed = strip_vowels(pw)
+        if not unpointed or len(unpointed) < 2:
+            continue
+
+        # Skip if no standard vowels (only has dots/diacritics)
+        if not has_standard_vowels(pw):
+            continue
+
+        vowel_count = count_standard_vowels(pw)
+
+        # Add or replace if this form has more vowels
+        if unpointed not in vocab or vowel_count > vocab[unpointed]["vowel_count"]:
+            west = transliterate_word_pointed(pw, SyriacDialect.WEST)
+            east = transliterate_word_pointed(pw, SyriacDialect.EAST)
+            vocab[unpointed] = {
+                "pointed": pw,
+                "west": west,
+                "east": east,
+                "vowel_count": vowel_count,
+            }
+
+    print(f"Built vocabulary with {len(vocab)} unique unpointed forms")
+
+    # Save to file (without vowel_count)
+    with open(output_path, "w", encoding="utf-8") as f:
+        for unpointed, data in vocab.items():
+            f.write(
+                json.dumps(
+                    {
+                        "unpointed": unpointed,
+                        "pointed": data["pointed"],
+                        "west": data["west"],
+                        "east": data["east"],
+                    },
+                    ensure_ascii=False,
+                )
+                + "\n"
+            )
+
+    print(f"Saved to {output_path}")
+    return len(vocab)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build corpus vocabulary from pointed Syriac text"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=Path(__file__).parent / "corpus_vocabulary.jsonl",
+        help="Output JSONL file path",
+    )
+    parser.add_argument(
+        "--corpus",
+        "-c",
+        type=Path,
+        nargs="*",
+        help="Corpus files to process (default: TEI and Bible corpora)",
+    )
+    args = parser.parse_args()
+
+    data_dir = Path(__file__).parent
+
+    # Default corpus files
+    if args.corpus:
+        corpus_files = args.corpus
+    else:
+        corpus_files = [
+            data_dir / "syriac_corpus_tei.txt",
+            data_dir / "syriac_bible_corpus.txt",
+        ]
+
+    print("Building corpus vocabulary...")
+    print(f"Corpus files: {[f.name for f in corpus_files]}")
+    print()
+
+    count = build_vocabulary(corpus_files, args.output)
+    print(f"\nDone! Created {count} vocabulary entries.")
+
+
+if __name__ == "__main__":
+    main()

src/data/fetch_sedra_vocalised.py

@@ -335,4 +335,4 @@ if __name__ == "__main__":
         try:
             asyncio.run(main(force=force))
         except KeyboardInterrupt:
-            print("\nInterrupted. Progress saved - run again to resume.")
+            print("\nInterrupted. Progress saved - run again to resume.")

src/data/generate_syr_lat_pairs.py

@@ -417,6 +417,148 @@ def transliterate_word_unpointed(word: str, dialect: SyriacDialect) -> str:
     return "".join(result)
 
 
+# Syriac proclitics (prefixes) that should be separated with hyphens
+# ܒ beth "in/at/by", ܕ dalath "of/that", ܘ waw "and", ܠ lamadh "to/for"
+SYRIAC_PROCLITICS = {
+    "\u0712": "b",  # ܒ beth
+    "\u0715": "d",  # ܕ dalath
+    "\u0718": "w",  # ܘ waw
+    "\u0720": "l",  # ܠ lamadh
+}
+
+# Syriac vowel diacritics range (U+0730 - U+073F)
+SYRIAC_VOWEL_RANGE = set(chr(c) for c in range(0x0730, 0x0740))
+
+
+def count_syriac_consonants(word: str) -> int:
+    """Count the number of Syriac consonants in a word."""
+    return sum(1 for c in word if c in SYRIAC_CONSONANTS)
+
+
+def try_strip_proclitic(
+    word: str, dialect: SyriacDialect, use_sedra: bool = True, _depth: int = 0
+) -> Optional[Tuple[str, str]]:
+    """
+    Try to identify and strip proclitic prefix(es) from a Syriac word.
+
+    Recursively handles multiple proclitics (e.g., ܘܠܡܠܟܐ → w-l-malko).
+
+    Only strips if:
+    1. The full word is NOT a known root word (in manual/SEDRA, not corpus)
+    2. The remainder has at least 3 consonants (triliteral root)
+
+    For unpointed words, uses SEDRA lookup for remainder if available,
+    otherwise falls back to heuristic transliteration.
+
+    This avoids incorrectly splitting root words like ܒܝܬ (bayt), ܒܪܐ (bro), ܕܡܐ (dmo).
+
+    Returns:
+        Tuple of (prefix_latin, remainder_latin) if successful, None otherwise.
+    """
+    # Prevent infinite recursion (max 3 proclitics: w-d-l- etc.)
+    if _depth > 3:
+        return None
+
+    if len(word) < 2:
+        return None
+
+    first_char = word[0]
+    if first_char not in SYRIAC_PROCLITICS:
+        return None
+
+    # Check: if the FULL word is a known root word (manual/SEDRA), don't split
+    # (e.g., ܒܝܬܐ "bayto/house" should not become b-yto)
+    if use_sedra:
+        sedra = get_sedra_lookup()
+        if sedra is not None:
+            dialect_str = "west" if dialect == SyriacDialect.WEST else "east"
+            if sedra.is_root_word(word, dialect_str):
+                return None  # Full word is a known root, don't split
+
+    remainder = word[1:]
+    prefix_latin = SYRIAC_PROCLITICS[first_char]
+
+    # Skip any vowel diacritics that might follow the prefix
+    # (they belong to the prefix, not the remainder)
+    while remainder and remainder[0] in SYRIAC_VOWEL_RANGE:
+        remainder = remainder[1:]
+
+    if not remainder:
+        return None
+
+    # Require at least 3 consonants in remainder (triliteral root minimum)
+    # This prevents splitting root words like ܒܝܬ (b-y-t), ܒܪܐ (b-r-ʾ), ܕܡܐ (d-m-ʾ)
+    if count_syriac_consonants(remainder) < 3:
+        return None
+
+    # Check if remainder has vowel diacritics → use pointed transliteration
+    if has_vowel_diacritics(remainder):
+        remainder_latin = transliterate_word_pointed(remainder, dialect)
+        return (prefix_latin, remainder_latin)
+
+    # Recursively check if remainder also starts with a proclitic
+    nested_result = try_strip_proclitic(remainder, dialect, use_sedra, _depth + 1)
+    if nested_result is not None:
+        nested_prefix, nested_remainder = nested_result
+        return (prefix_latin, f"{nested_prefix}-{nested_remainder}")
+
+    # Check if remainder is in SEDRA
+    if use_sedra:
+        sedra = get_sedra_lookup()
+        if sedra is not None:
+            dialect_str = "west" if dialect == SyriacDialect.WEST else "east"
+            vocalised = sedra.get_vocalised(remainder, dialect_str)
+            if vocalised is not None:
+                return (prefix_latin, vocalised.latin)
+
+    # Fallback: remainder not in SEDRA, use heuristic transliteration
+    # This handles inflected forms like ܡܕܪܟܘܬܗܘܢ (mdarakutahun)
+    remainder_latin = transliterate_word_unpointed(remainder, dialect)
+    return (prefix_latin, remainder_latin)
+
+
+def try_strip_proclitic_pointed(word: str, dialect: SyriacDialect) -> Optional[Tuple[str, str]]:
+    """
+    Try to strip a proclitic prefix from a POINTED Syriac word.
+
+    For pointed words, we identify proclitics by checking if the first consonant
+    has no vowel directly after it (true proclitics are vowelless or have shwa).
+    If the first consonant has a full vowel, it's part of the root, not a proclitic.
+
+    Returns:
+        Tuple of (prefix_latin, remainder_latin) if successful, None otherwise.
+    """
+    if len(word) < 3:  # Need at least: proclitic + consonant + vowel
+        return None
+
+    first_char = word[0]
+    if first_char not in SYRIAC_PROCLITICS:
+        return None
+
+    # Check if the proclitic has a vowel immediately after it
+    # If yes, it's probably part of the root, not a proclitic
+    if len(word) > 1 and word[1] in SYRIAC_VOWEL_RANGE:
+        return None  # Has vowel → part of root, don't split
+
+    prefix_latin = SYRIAC_PROCLITICS[first_char]
+
+    # Remainder starts at position 1 (right after the proclitic consonant)
+    remainder = word[1:]
+
+    # Remainder must start with a consonant
+    if remainder[0] not in SYRIAC_CONSONANTS:
+        return None
+
+    # Remainder must have at least 3 consonants (triliteral root)
+    consonant_count = sum(1 for c in remainder if c in SYRIAC_CONSONANTS)
+    if consonant_count < 3:
+        return None
+
+    # Transliterate the remainder
+    remainder_latin = transliterate_word_pointed(remainder, dialect)
+    return (prefix_latin, remainder_latin)
+
+
 def transliterate_word_with_sedra(
     word: str, dialect: SyriacDialect, use_sedra: bool = True, skip_heuristics: bool = False
 ) -> Optional[str]:
@@ -424,10 +566,12 @@ def transliterate_word_with_sedra(
     Transliterate a single Syriac word, using SEDRA lookup when available.
 
     Strategy:
-    1. If word has vowel diacritics → use pointed transliteration
-    2. If SEDRA has the word → use vocalised form from SEDRA
-    3. If skip_heuristics=True and no SEDRA entry → return None
-    4. Otherwise → fall back to heuristic transliteration
+    1. Try stripping proclitics from pointed words (ܒ, ܕ, ܘ, ܠ)
+    2. If word has vowel diacritics → use pointed transliteration
+    3. Try stripping proclitics from unpointed words
+    4. If SEDRA has the word → use vocalised form from SEDRA
+    5. If skip_heuristics=True and no SEDRA entry → return None
+    6. Otherwise → fall back to heuristic transliteration
 
     Args:
         word: Syriac word to transliterate
@@ -438,10 +582,22 @@ def transliterate_word_with_sedra(
     Returns:
         Latin transliteration, or None if skip_heuristics=True and no SEDRA entry
     """
-    # If word already has vowel diacritics, use pointed transliteration
+    # For pointed words: try proclitic stripping first, then transliterate
     if has_vowel_diacritics(word):
+        # Try stripping proclitic from pointed word
+        proclitic_result = try_strip_proclitic_pointed(word, dialect)
+        if proclitic_result is not None:
+            prefix, remainder = proclitic_result
+            return f"{prefix}-{remainder}"
+        # No proclitic found, transliterate the whole word
         return transliterate_word_pointed(word, dialect)
 
+    # For unpointed words: try proclitic stripping with SEDRA lookup
+    proclitic_result = try_strip_proclitic(word, dialect, use_sedra)
+    if proclitic_result is not None:
+        prefix, remainder = proclitic_result
+        return f"{prefix}-{remainder}"
+
     # Try SEDRA lookup for unpointed words
     if use_sedra:
         sedra = get_sedra_lookup()
@@ -893,4 +1049,4 @@ if __name__ == "__main__":
     print()
 
     # Run main processing
-    main()
+    main()

src/data/manual_vocabulary.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d31b677a114e26ce16e07a2544c72e3f6f4948ee9ed0e2624d59dc23f2d1d1db
+size 11573

src/data/sedra_lookup.py

@@ -74,7 +74,40 @@ class SedraLookup:
         from generate_syr_lat_pairs import (SyriacDialect,
                                             transliterate_word_pointed)
 
+        # --- Load manual vocabulary first (highest priority) ---
+        manual_vocab_path = Path(__file__).parent / "manual_vocabulary.jsonl"
+        manual_count = 0
+        if manual_vocab_path.exists():
+            with open(manual_vocab_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    record = json.loads(line)
+                    unpointed = record.get("unpointed", "").strip()
+                    pointed = record.get("pointed", "").strip()
+                    west_latin = record.get("west", "")
+                    east_latin = record.get("east", "")
+
+                    if not unpointed:
+                        continue
+
+                    if west_latin:
+                        self._west_lookup[unpointed] = VocalisedForm(
+                            unpointed=unpointed,
+                            pointed=pointed,
+                            latin=west_latin,
+                            category="manual",
+                        )
+                        manual_count += 1
+
+                    if east_latin:
+                        self._east_lookup[unpointed] = VocalisedForm(
+                            unpointed=unpointed,
+                            pointed=pointed,
+                            latin=east_latin,
+                            category="manual",
+                        )
+
         count = 0
+        extracted_count = 0
         with open(self.data_path, "r", encoding="utf-8") as f:
             for line in f:
                 record = json.loads(line)
@@ -86,8 +119,8 @@ class SedraLookup:
                 if not unpointed:
                     continue
 
-                # Store West Syriac form
-                if western:
+                # Store West Syriac form (don't overwrite manual entries)
+                if western and unpointed not in self._west_lookup:
                     west_latin = transliterate_word_pointed(western, SyriacDialect.WEST)
                     self._west_lookup[unpointed] = VocalisedForm(
                         unpointed=unpointed,
@@ -97,7 +130,7 @@ class SedraLookup:
                     )
 
                 # Store East Syriac form (or infer from West)
-                if eastern:
+                if eastern and unpointed not in self._east_lookup:
                     east_latin = transliterate_word_pointed(eastern, SyriacDialect.EAST)
                     self._east_lookup[unpointed] = VocalisedForm(
                         unpointed=unpointed,
@@ -119,9 +152,87 @@ class SedraLookup:
 
                 count += 1
 
+                # --- Extract individual words from compound entries ---
+                # If entry contains spaces (is a phrase), extract each word
+                unpointed_words = unpointed.split()
+                western_words = western.split() if western else []
+                eastern_words = eastern.split() if eastern else []
+
+                if len(unpointed_words) > 1:
+                    # Extract West words
+                    if len(unpointed_words) == len(western_words):
+                        for uw, ww in zip(unpointed_words, western_words):
+                            # Only add if not already present (don't overwrite direct entries)
+                            if uw not in self._west_lookup:
+                                wlat = transliterate_word_pointed(ww, SyriacDialect.WEST)
+                                self._west_lookup[uw] = VocalisedForm(
+                                    unpointed=uw,
+                                    pointed=ww,
+                                    latin=wlat,
+                                    category=category,
+                                )
+                                extracted_count += 1
+
+                    # Extract East words (or infer from West)
+                    if len(unpointed_words) == len(eastern_words):
+                        for uw, ew in zip(unpointed_words, eastern_words):
+                            if uw not in self._east_lookup:
+                                elat = transliterate_word_pointed(ew, SyriacDialect.EAST)
+                                self._east_lookup[uw] = VocalisedForm(
+                                    unpointed=uw,
+                                    pointed=ew,
+                                    latin=elat,
+                                    category=category,
+                                )
+                    elif len(unpointed_words) == len(western_words):
+                        # Infer East from West for extracted words
+                        for uw, ww in zip(unpointed_words, western_words):
+                            if uw not in self._east_lookup:
+                                elat = transliterate_word_pointed(ww, SyriacDialect.EAST)
+                                self._east_lookup[uw] = VocalisedForm(
+                                    unpointed=uw,
+                                    pointed=ww,
+                                    latin=elat,
+                                    category=category,
+                                )
+
+        # --- Load corpus vocabulary as fallback ---
+        corpus_vocab_path = Path(__file__).parent / "corpus_vocabulary.jsonl"
+        corpus_count = 0
+        if corpus_vocab_path.exists():
+            with open(corpus_vocab_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    record = json.loads(line)
+                    unpointed = record.get("unpointed", "").strip()
+                    pointed = record.get("pointed", "").strip()
+                    west_latin = record.get("west", "")
+                    east_latin = record.get("east", "")
+
+                    if not unpointed or not pointed:
+                        continue
+
+                    # Only add if not already present (SEDRA takes priority)
+                    if unpointed not in self._west_lookup and west_latin:
+                        self._west_lookup[unpointed] = VocalisedForm(
+                            unpointed=unpointed,
+                            pointed=pointed,
+                            latin=west_latin,
+                            category="corpus",
+                        )
+                        corpus_count += 1
+
+                    if unpointed not in self._east_lookup and east_latin:
+                        self._east_lookup[unpointed] = VocalisedForm(
+                            unpointed=unpointed,
+                            pointed=pointed,
+                            latin=east_latin,
+                            category="corpus",
+                        )
+
         self._loaded = True
         print(
-            f"Loaded {count} SEDRA entries ({len(self._west_lookup)} West, {len(self._east_lookup)} East)"
+            f"Loaded {manual_count} manual + {count} SEDRA + {extracted_count} extracted + {corpus_count} corpus "
+            f"({len(self._west_lookup)} West, {len(self._east_lookup)} East)"
         )
         return True
 
@@ -202,6 +313,38 @@ class SedraLookup:
         clean = self._strip_non_vowel_diacritics(unpointed)
         return clean in self._west_lookup or unpointed in self._west_lookup
 
+    def is_root_word(self, unpointed: str, dialect: str | Dialect = "west") -> bool:
+        """
+        Check if a word is a known root word (from manual or SEDRA, not corpus).
+
+        This is used for proclitic detection - we only want to split a word
+        if the full word is NOT itself a known root. Words from corpus
+        may include prefixed forms, so we don't count those.
+
+        Args:
+            unpointed: Unpointed Syriac word
+            dialect: "west" or "east"
+
+        Returns:
+            True if the word is in manual or SEDRA vocabulary (not corpus)
+        """
+        if not self._loaded:
+            self.load()
+
+        if isinstance(dialect, str):
+            dialect = Dialect(dialect.lower())
+
+        clean = self._strip_non_vowel_diacritics(unpointed)
+        lookup = self._west_lookup if dialect == Dialect.WEST else self._east_lookup
+
+        # Check both clean and original form
+        form = lookup.get(clean) or lookup.get(unpointed)
+        if form is None:
+            return False
+
+        # Only consider it a root if NOT from corpus
+        return form.category != "corpus"
+
     @property
     def west_count(self) -> int:
         return len(self._west_lookup)

src/test_t5.py

@@ -98,4 +98,4 @@ while True:
     if text:
         result = transliterate(text, dialect)
         dialect_name = "East" if dialect == "east" else "West"
-        print(f"  [{dialect_name}] {rtl(text)} → {result}")
+        print(f"  [{dialect_name}] {rtl(text)} → {result}")

src/tokeniser/added_tokens.json

@@ -1,102 +1,102 @@
 {
-  "<extra_id_0>": 40099,
-  "<extra_id_10>": 40089,
-  "<extra_id_11>": 40088,
-  "<extra_id_12>": 40087,
-  "<extra_id_13>": 40086,
-  "<extra_id_14>": 40085,
-  "<extra_id_15>": 40084,
-  "<extra_id_16>": 40083,
-  "<extra_id_17>": 40082,
-  "<extra_id_18>": 40081,
-  "<extra_id_19>": 40080,
-  "<extra_id_1>": 40098,
-  "<extra_id_20>": 40079,
-  "<extra_id_21>": 40078,
-  "<extra_id_22>": 40077,
-  "<extra_id_23>": 40076,
-  "<extra_id_24>": 40075,
-  "<extra_id_25>": 40074,
-  "<extra_id_26>": 40073,
-  "<extra_id_27>": 40072,
-  "<extra_id_28>": 40071,
-  "<extra_id_29>": 40070,
-  "<extra_id_2>": 40097,
-  "<extra_id_30>": 40069,
-  "<extra_id_31>": 40068,
-  "<extra_id_32>": 40067,
-  "<extra_id_33>": 40066,
-  "<extra_id_34>": 40065,
-  "<extra_id_35>": 40064,
-  "<extra_id_36>": 40063,
-  "<extra_id_37>": 40062,
-  "<extra_id_38>": 40061,
-  "<extra_id_39>": 40060,
-  "<extra_id_3>": 40096,
-  "<extra_id_40>": 40059,
-  "<extra_id_41>": 40058,
-  "<extra_id_42>": 40057,
-  "<extra_id_43>": 40056,
-  "<extra_id_44>": 40055,
-  "<extra_id_45>": 40054,
-  "<extra_id_46>": 40053,
-  "<extra_id_47>": 40052,
-  "<extra_id_48>": 40051,
-  "<extra_id_49>": 40050,
-  "<extra_id_4>": 40095,
-  "<extra_id_50>": 40049,
-  "<extra_id_51>": 40048,
-  "<extra_id_52>": 40047,
-  "<extra_id_53>": 40046,
-  "<extra_id_54>": 40045,
-  "<extra_id_55>": 40044,
-  "<extra_id_56>": 40043,
-  "<extra_id_57>": 40042,
-  "<extra_id_58>": 40041,
-  "<extra_id_59>": 40040,
-  "<extra_id_5>": 40094,
-  "<extra_id_60>": 40039,
-  "<extra_id_61>": 40038,
-  "<extra_id_62>": 40037,
-  "<extra_id_63>": 40036,
-  "<extra_id_64>": 40035,
-  "<extra_id_65>": 40034,
-  "<extra_id_66>": 40033,
-  "<extra_id_67>": 40032,
-  "<extra_id_68>": 40031,
-  "<extra_id_69>": 40030,
-  "<extra_id_6>": 40093,
-  "<extra_id_70>": 40029,
-  "<extra_id_71>": 40028,
-  "<extra_id_72>": 40027,
-  "<extra_id_73>": 40026,
-  "<extra_id_74>": 40025,
-  "<extra_id_75>": 40024,
-  "<extra_id_76>": 40023,
-  "<extra_id_77>": 40022,
-  "<extra_id_78>": 40021,
-  "<extra_id_79>": 40020,
-  "<extra_id_7>": 40092,
-  "<extra_id_80>": 40019,
-  "<extra_id_81>": 40018,
-  "<extra_id_82>": 40017,
-  "<extra_id_83>": 40016,
-  "<extra_id_84>": 40015,
-  "<extra_id_85>": 40014,
-  "<extra_id_86>": 40013,
-  "<extra_id_87>": 40012,
-  "<extra_id_88>": 40011,
-  "<extra_id_89>": 40010,
-  "<extra_id_8>": 40091,
-  "<extra_id_90>": 40009,
-  "<extra_id_91>": 40008,
-  "<extra_id_92>": 40007,
-  "<extra_id_93>": 40006,
-  "<extra_id_94>": 40005,
-  "<extra_id_95>": 40004,
-  "<extra_id_96>": 40003,
-  "<extra_id_97>": 40002,
-  "<extra_id_98>": 40001,
-  "<extra_id_99>": 40000,
-  "<extra_id_9>": 40090
+  "<extra_id_0>": 16099,
+  "<extra_id_10>": 16089,
+  "<extra_id_11>": 16088,
+  "<extra_id_12>": 16087,
+  "<extra_id_13>": 16086,
+  "<extra_id_14>": 16085,
+  "<extra_id_15>": 16084,
+  "<extra_id_16>": 16083,
+  "<extra_id_17>": 16082,
+  "<extra_id_18>": 16081,
+  "<extra_id_19>": 16080,
+  "<extra_id_1>": 16098,
+  "<extra_id_20>": 16079,
+  "<extra_id_21>": 16078,
+  "<extra_id_22>": 16077,
+  "<extra_id_23>": 16076,
+  "<extra_id_24>": 16075,
+  "<extra_id_25>": 16074,
+  "<extra_id_26>": 16073,
+  "<extra_id_27>": 16072,
+  "<extra_id_28>": 16071,
+  "<extra_id_29>": 16070,
+  "<extra_id_2>": 16097,
+  "<extra_id_30>": 16069,
+  "<extra_id_31>": 16068,
+  "<extra_id_32>": 16067,
+  "<extra_id_33>": 16066,
+  "<extra_id_34>": 16065,
+  "<extra_id_35>": 16064,
+  "<extra_id_36>": 16063,
+  "<extra_id_37>": 16062,
+  "<extra_id_38>": 16061,
+  "<extra_id_39>": 16060,
+  "<extra_id_3>": 16096,
+  "<extra_id_40>": 16059,
+  "<extra_id_41>": 16058,
+  "<extra_id_42>": 16057,
+  "<extra_id_43>": 16056,
+  "<extra_id_44>": 16055,
+  "<extra_id_45>": 16054,
+  "<extra_id_46>": 16053,
+  "<extra_id_47>": 16052,
+  "<extra_id_48>": 16051,
+  "<extra_id_49>": 16050,
+  "<extra_id_4>": 16095,
+  "<extra_id_50>": 16049,
+  "<extra_id_51>": 16048,
+  "<extra_id_52>": 16047,
+  "<extra_id_53>": 16046,
+  "<extra_id_54>": 16045,
+  "<extra_id_55>": 16044,
+  "<extra_id_56>": 16043,
+  "<extra_id_57>": 16042,
+  "<extra_id_58>": 16041,
+  "<extra_id_59>": 16040,
+  "<extra_id_5>": 16094,
+  "<extra_id_60>": 16039,
+  "<extra_id_61>": 16038,
+  "<extra_id_62>": 16037,
+  "<extra_id_63>": 16036,
+  "<extra_id_64>": 16035,
+  "<extra_id_65>": 16034,
+  "<extra_id_66>": 16033,
+  "<extra_id_67>": 16032,
+  "<extra_id_68>": 16031,
+  "<extra_id_69>": 16030,
+  "<extra_id_6>": 16093,
+  "<extra_id_70>": 16029,
+  "<extra_id_71>": 16028,
+  "<extra_id_72>": 16027,
+  "<extra_id_73>": 16026,
+  "<extra_id_74>": 16025,
+  "<extra_id_75>": 16024,
+  "<extra_id_76>": 16023,
+  "<extra_id_77>": 16022,
+  "<extra_id_78>": 16021,
+  "<extra_id_79>": 16020,
+  "<extra_id_7>": 16092,
+  "<extra_id_80>": 16019,
+  "<extra_id_81>": 16018,
+  "<extra_id_82>": 16017,
+  "<extra_id_83>": 16016,
+  "<extra_id_84>": 16015,
+  "<extra_id_85>": 16014,
+  "<extra_id_86>": 16013,
+  "<extra_id_87>": 16012,
+  "<extra_id_88>": 16011,
+  "<extra_id_89>": 16010,
+  "<extra_id_8>": 16091,
+  "<extra_id_90>": 16009,
+  "<extra_id_91>": 16008,
+  "<extra_id_92>": 16007,
+  "<extra_id_93>": 16006,
+  "<extra_id_94>": 16005,
+  "<extra_id_95>": 16004,
+  "<extra_id_96>": 16003,
+  "<extra_id_97>": 16002,
+  "<extra_id_98>": 16001,
+  "<extra_id_99>": 16000,
+  "<extra_id_9>": 16090
 }

src/tokeniser/aramt5_sp.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:347fef5a38453d31d013802c803c9d91f9333fefacffc922784af95f94202f4b
-size 988532
+oid sha256:02090247a3d32c0cb86d464007c80682b2dcf3e0619b42e826dbce248d884dad
+size 511338

src/tokeniser/spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:347fef5a38453d31d013802c803c9d91f9333fefacffc922784af95f94202f4b
-size 988532
+oid sha256:02090247a3d32c0cb86d464007c80682b2dcf3e0619b42e826dbce248d884dad
+size 511338

src/tokeniser/tokenizer_config.json

@@ -25,7 +25,7 @@
       "single_word": false,
       "special": true
     },
-    "40000": {
+    "16000": {
       "content": "<extra_id_99>",
       "lstrip": true,
       "normalized": false,
@@ -33,7 +33,7 @@
       "single_word": false,
       "special": true
     },
-    "40001": {
+    "16001": {
       "content": "<extra_id_98>",
       "lstrip": true,
       "normalized": false,
@@ -41,7 +41,7 @@
       "single_word": false,
       "special": true
     },
-    "40002": {
+    "16002": {
       "content": "<extra_id_97>",
       "lstrip": true,
       "normalized": false,
@@ -49,7 +49,7 @@
       "single_word": false,
       "special": true
     },
-    "40003": {
+    "16003": {
       "content": "<extra_id_96>",
       "lstrip": true,
       "normalized": false,
@@ -57,7 +57,7 @@
       "single_word": false,
       "special": true
     },
-    "40004": {
+    "16004": {
       "content": "<extra_id_95>",
       "lstrip": true,
       "normalized": false,
@@ -65,7 +65,7 @@
       "single_word": false,
       "special": true
     },
-    "40005": {
+    "16005": {
       "content": "<extra_id_94>",
       "lstrip": true,
       "normalized": false,
@@ -73,7 +73,7 @@
       "single_word": false,
       "special": true
     },
-    "40006": {
+    "16006": {
       "content": "<extra_id_93>",
       "lstrip": true,
       "normalized": false,
@@ -81,7 +81,7 @@
       "single_word": false,
       "special": true
     },
-    "40007": {
+    "16007": {
       "content": "<extra_id_92>",
       "lstrip": true,
       "normalized": false,
@@ -89,7 +89,7 @@
       "single_word": false,
       "special": true
     },
-    "40008": {
+    "16008": {
       "content": "<extra_id_91>",
       "lstrip": true,
       "normalized": false,
@@ -97,7 +97,7 @@
       "single_word": false,
       "special": true
     },
-    "40009": {
+    "16009": {
       "content": "<extra_id_90>",
       "lstrip": true,
       "normalized": false,
@@ -105,7 +105,7 @@
       "single_word": false,
       "special": true
     },
-    "40010": {
+    "16010": {
       "content": "<extra_id_89>",
       "lstrip": true,
       "normalized": false,
@@ -113,7 +113,7 @@
       "single_word": false,
       "special": true
     },
-    "40011": {
+    "16011": {
       "content": "<extra_id_88>",
       "lstrip": true,
       "normalized": false,
@@ -121,7 +121,7 @@
       "single_word": false,
       "special": true
     },
-    "40012": {
+    "16012": {
       "content": "<extra_id_87>",
       "lstrip": true,
       "normalized": false,
@@ -129,7 +129,7 @@
       "single_word": false,
       "special": true
     },
-    "40013": {
+    "16013": {
       "content": "<extra_id_86>",
       "lstrip": true,
       "normalized": false,
@@ -137,7 +137,7 @@
       "single_word": false,
       "special": true
     },
-    "40014": {
+    "16014": {
       "content": "<extra_id_85>",
       "lstrip": true,
       "normalized": false,
@@ -145,7 +145,7 @@
       "single_word": false,
       "special": true
     },
-    "40015": {
+    "16015": {
       "content": "<extra_id_84>",
       "lstrip": true,
       "normalized": false,
@@ -153,7 +153,7 @@
       "single_word": false,
       "special": true
     },
-    "40016": {
+    "16016": {
       "content": "<extra_id_83>",
       "lstrip": true,
       "normalized": false,
@@ -161,7 +161,7 @@
       "single_word": false,
       "special": true
     },
-    "40017": {
+    "16017": {
       "content": "<extra_id_82>",
       "lstrip": true,
       "normalized": false,
@@ -169,7 +169,7 @@
       "single_word": false,
       "special": true
     },
-    "40018": {
+    "16018": {
       "content": "<extra_id_81>",
       "lstrip": true,
       "normalized": false,
@@ -177,7 +177,7 @@
       "single_word": false,
       "special": true
     },
-    "40019": {
+    "16019": {
       "content": "<extra_id_80>",
       "lstrip": true,
       "normalized": false,
@@ -185,7 +185,7 @@
       "single_word": false,
       "special": true
     },
-    "40020": {
+    "16020": {
       "content": "<extra_id_79>",
       "lstrip": true,
       "normalized": false,
@@ -193,7 +193,7 @@
       "single_word": false,
       "special": true
     },
-    "40021": {
+    "16021": {
       "content": "<extra_id_78>",
       "lstrip": true,
       "normalized": false,
@@ -201,7 +201,7 @@
       "single_word": false,
       "special": true
     },
-    "40022": {
+    "16022": {
       "content": "<extra_id_77>",
       "lstrip": true,
       "normalized": false,
@@ -209,7 +209,7 @@
       "single_word": false,
       "special": true
     },
-    "40023": {
+    "16023": {
       "content": "<extra_id_76>",
       "lstrip": true,
       "normalized": false,
@@ -217,7 +217,7 @@
       "single_word": false,
       "special": true
     },
-    "40024": {
+    "16024": {
       "content": "<extra_id_75>",
       "lstrip": true,
       "normalized": false,
@@ -225,7 +225,7 @@
       "single_word": false,
       "special": true
     },
-    "40025": {
+    "16025": {
       "content": "<extra_id_74>",
       "lstrip": true,
       "normalized": false,
@@ -233,7 +233,7 @@
       "single_word": false,
       "special": true
     },
-    "40026": {
+    "16026": {
       "content": "<extra_id_73>",
       "lstrip": true,
       "normalized": false,
@@ -241,7 +241,7 @@
       "single_word": false,
       "special": true
     },
-    "40027": {
+    "16027": {
       "content": "<extra_id_72>",
       "lstrip": true,
       "normalized": false,
@@ -249,7 +249,7 @@
       "single_word": false,
       "special": true
     },
-    "40028": {
+    "16028": {
       "content": "<extra_id_71>",
       "lstrip": true,
       "normalized": false,
@@ -257,7 +257,7 @@
       "single_word": false,
       "special": true
     },
-    "40029": {
+    "16029": {
       "content": "<extra_id_70>",
       "lstrip": true,
       "normalized": false,
@@ -265,7 +265,7 @@
       "single_word": false,
       "special": true
     },
-    "40030": {
+    "16030": {
       "content": "<extra_id_69>",
       "lstrip": true,
       "normalized": false,
@@ -273,7 +273,7 @@
       "single_word": false,
       "special": true
     },
-    "40031": {
+    "16031": {
       "content": "<extra_id_68>",
       "lstrip": true,
       "normalized": false,
@@ -281,7 +281,7 @@
       "single_word": false,
       "special": true
     },
-    "40032": {
+    "16032": {
       "content": "<extra_id_67>",
       "lstrip": true,
       "normalized": false,
@@ -289,7 +289,7 @@
       "single_word": false,
       "special": true
     },
-    "40033": {
+    "16033": {
       "content": "<extra_id_66>",
       "lstrip": true,
       "normalized": false,
@@ -297,7 +297,7 @@
       "single_word": false,
       "special": true
     },
-    "40034": {
+    "16034": {
       "content": "<extra_id_65>",
       "lstrip": true,
       "normalized": false,
@@ -305,7 +305,7 @@
       "single_word": false,
       "special": true
     },
-    "40035": {
+    "16035": {
       "content": "<extra_id_64>",
       "lstrip": true,
       "normalized": false,
@@ -313,7 +313,7 @@
       "single_word": false,
       "special": true
     },
-    "40036": {
+    "16036": {
       "content": "<extra_id_63>",
       "lstrip": true,
       "normalized": false,
@@ -321,7 +321,7 @@
       "single_word": false,
       "special": true
     },
-    "40037": {
+    "16037": {
       "content": "<extra_id_62>",
       "lstrip": true,
       "normalized": false,
@@ -329,7 +329,7 @@
       "single_word": false,
       "special": true
     },
-    "40038": {
+    "16038": {
       "content": "<extra_id_61>",
       "lstrip": true,
       "normalized": false,
@@ -337,7 +337,7 @@
       "single_word": false,
       "special": true
     },
-    "40039": {
+    "16039": {
       "content": "<extra_id_60>",
       "lstrip": true,
       "normalized": false,
@@ -345,7 +345,7 @@
       "single_word": false,
       "special": true
     },
-    "40040": {
+    "16040": {
       "content": "<extra_id_59>",
       "lstrip": true,
       "normalized": false,
@@ -353,7 +353,7 @@
       "single_word": false,
       "special": true
     },
-    "40041": {
+    "16041": {
       "content": "<extra_id_58>",
       "lstrip": true,
       "normalized": false,
@@ -361,7 +361,7 @@
       "single_word": false,
       "special": true
     },
-    "40042": {
+    "16042": {
       "content": "<extra_id_57>",
       "lstrip": true,
       "normalized": false,
@@ -369,7 +369,7 @@
       "single_word": false,
       "special": true
     },
-    "40043": {
+    "16043": {
       "content": "<extra_id_56>",
       "lstrip": true,
       "normalized": false,
@@ -377,7 +377,7 @@
       "single_word": false,
       "special": true
     },
-    "40044": {
+    "16044": {
       "content": "<extra_id_55>",
       "lstrip": true,
       "normalized": false,
@@ -385,7 +385,7 @@
       "single_word": false,
       "special": true
     },
-    "40045": {
+    "16045": {
       "content": "<extra_id_54>",
       "lstrip": true,
       "normalized": false,
@@ -393,7 +393,7 @@
       "single_word": false,
       "special": true
     },
-    "40046": {
+    "16046": {
       "content": "<extra_id_53>",
       "lstrip": true,
       "normalized": false,
@@ -401,7 +401,7 @@
       "single_word": false,
       "special": true
     },
-    "40047": {
+    "16047": {
       "content": "<extra_id_52>",
       "lstrip": true,
       "normalized": false,
@@ -409,7 +409,7 @@
       "single_word": false,
       "special": true
     },
-    "40048": {
+    "16048": {
       "content": "<extra_id_51>",
       "lstrip": true,
       "normalized": false,
@@ -417,7 +417,7 @@
       "single_word": false,
       "special": true
     },
-    "40049": {
+    "16049": {
       "content": "<extra_id_50>",
       "lstrip": true,
       "normalized": false,
@@ -425,7 +425,7 @@
       "single_word": false,
       "special": true
     },
-    "40050": {
+    "16050": {
       "content": "<extra_id_49>",
       "lstrip": true,
       "normalized": false,
@@ -433,7 +433,7 @@
       "single_word": false,
       "special": true
     },
-    "40051": {
+    "16051": {
       "content": "<extra_id_48>",
       "lstrip": true,
       "normalized": false,
@@ -441,7 +441,7 @@
       "single_word": false,
       "special": true
     },
-    "40052": {
+    "16052": {
       "content": "<extra_id_47>",
       "lstrip": true,
       "normalized": false,
@@ -449,7 +449,7 @@
       "single_word": false,
       "special": true
     },
-    "40053": {
+    "16053": {
       "content": "<extra_id_46>",
       "lstrip": true,
       "normalized": false,
@@ -457,7 +457,7 @@
       "single_word": false,
       "special": true
     },
-    "40054": {
+    "16054": {
       "content": "<extra_id_45>",
       "lstrip": true,
       "normalized": false,
@@ -465,7 +465,7 @@
       "single_word": false,
       "special": true
     },
-    "40055": {
+    "16055": {
       "content": "<extra_id_44>",
       "lstrip": true,
       "normalized": false,
@@ -473,7 +473,7 @@
       "single_word": false,
       "special": true
     },
-    "40056": {
+    "16056": {
       "content": "<extra_id_43>",
       "lstrip": true,
       "normalized": false,
@@ -481,7 +481,7 @@
       "single_word": false,
       "special": true
     },
-    "40057": {
+    "16057": {
       "content": "<extra_id_42>",
       "lstrip": true,
       "normalized": false,
@@ -489,7 +489,7 @@
       "single_word": false,
       "special": true
     },
-    "40058": {
+    "16058": {
       "content": "<extra_id_41>",
       "lstrip": true,
       "normalized": false,
@@ -497,7 +497,7 @@
       "single_word": false,
       "special": true
     },
-    "40059": {
+    "16059": {
       "content": "<extra_id_40>",
       "lstrip": true,
       "normalized": false,
@@ -505,7 +505,7 @@
       "single_word": false,
       "special": true
     },
-    "40060": {
+    "16060": {
       "content": "<extra_id_39>",
       "lstrip": true,
       "normalized": false,
@@ -513,7 +513,7 @@
       "single_word": false,
       "special": true
     },
-    "40061": {
+    "16061": {
       "content": "<extra_id_38>",
       "lstrip": true,
       "normalized": false,
@@ -521,7 +521,7 @@
       "single_word": false,
       "special": true
     },
-    "40062": {
+    "16062": {
       "content": "<extra_id_37>",
       "lstrip": true,
       "normalized": false,
@@ -529,7 +529,7 @@
       "single_word": false,
       "special": true
     },
-    "40063": {
+    "16063": {
       "content": "<extra_id_36>",
       "lstrip": true,
       "normalized": false,
@@ -537,7 +537,7 @@
       "single_word": false,
       "special": true
     },
-    "40064": {
+    "16064": {
       "content": "<extra_id_35>",
       "lstrip": true,
       "normalized": false,
@@ -545,7 +545,7 @@
       "single_word": false,
       "special": true
     },
-    "40065": {
+    "16065": {
       "content": "<extra_id_34>",
       "lstrip": true,
       "normalized": false,
@@ -553,7 +553,7 @@
       "single_word": false,
       "special": true
     },
-    "40066": {
+    "16066": {
       "content": "<extra_id_33>",
       "lstrip": true,
       "normalized": false,
@@ -561,7 +561,7 @@
       "single_word": false,
       "special": true
     },
-    "40067": {
+    "16067": {
       "content": "<extra_id_32>",
       "lstrip": true,
       "normalized": false,
@@ -569,7 +569,7 @@
       "single_word": false,
       "special": true
     },
-    "40068": {
+    "16068": {
       "content": "<extra_id_31>",
       "lstrip": true,
       "normalized": false,
@@ -577,7 +577,7 @@
       "single_word": false,
       "special": true
     },
-    "40069": {
+    "16069": {
       "content": "<extra_id_30>",
       "lstrip": true,
       "normalized": false,
@@ -585,7 +585,7 @@
       "single_word": false,
       "special": true
     },
-    "40070": {
+    "16070": {
       "content": "<extra_id_29>",
       "lstrip": true,
       "normalized": false,
@@ -593,7 +593,7 @@
       "single_word": false,
       "special": true
     },
-    "40071": {
+    "16071": {
       "content": "<extra_id_28>",
       "lstrip": true,
       "normalized": false,
@@ -601,7 +601,7 @@
       "single_word": false,
       "special": true
     },
-    "40072": {
+    "16072": {
       "content": "<extra_id_27>",
       "lstrip": true,
       "normalized": false,
@@ -609,7 +609,7 @@
       "single_word": false,
       "special": true
     },
-    "40073": {
+    "16073": {
       "content": "<extra_id_26>",
       "lstrip": true,
       "normalized": false,
@@ -617,7 +617,7 @@
       "single_word": false,
       "special": true
     },
-    "40074": {
+    "16074": {
       "content": "<extra_id_25>",
       "lstrip": true,
       "normalized": false,
@@ -625,7 +625,7 @@
       "single_word": false,
       "special": true
     },
-    "40075": {
+    "16075": {
       "content": "<extra_id_24>",
       "lstrip": true,
       "normalized": false,
@@ -633,7 +633,7 @@
       "single_word": false,
       "special": true
     },
-    "40076": {
+    "16076": {
       "content": "<extra_id_23>",
       "lstrip": true,
       "normalized": false,
@@ -641,7 +641,7 @@
       "single_word": false,
       "special": true
     },
-    "40077": {
+    "16077": {
       "content": "<extra_id_22>",
       "lstrip": true,
       "normalized": false,
@@ -649,7 +649,7 @@
       "single_word": false,
       "special": true
     },
-    "40078": {
+    "16078": {
       "content": "<extra_id_21>",
       "lstrip": true,
       "normalized": false,
@@ -657,7 +657,7 @@
       "single_word": false,
       "special": true
     },
-    "40079": {
+    "16079": {
       "content": "<extra_id_20>",
       "lstrip": true,
       "normalized": false,
@@ -665,7 +665,7 @@
       "single_word": false,
       "special": true
     },
-    "40080": {
+    "16080": {
       "content": "<extra_id_19>",
       "lstrip": true,
       "normalized": false,
@@ -673,7 +673,7 @@
       "single_word": false,
       "special": true
     },
-    "40081": {
+    "16081": {
       "content": "<extra_id_18>",
       "lstrip": true,
       "normalized": false,
@@ -681,7 +681,7 @@
       "single_word": false,
       "special": true
     },
-    "40082": {
+    "16082": {
       "content": "<extra_id_17>",
       "lstrip": true,
       "normalized": false,
@@ -689,7 +689,7 @@
       "single_word": false,
       "special": true
     },
-    "40083": {
+    "16083": {
       "content": "<extra_id_16>",
       "lstrip": true,
       "normalized": false,
@@ -697,7 +697,7 @@
       "single_word": false,
       "special": true
     },
-    "40084": {
+    "16084": {
       "content": "<extra_id_15>",
       "lstrip": true,
       "normalized": false,
@@ -705,7 +705,7 @@
       "single_word": false,
       "special": true
     },
-    "40085": {
+    "16085": {
       "content": "<extra_id_14>",
       "lstrip": true,
       "normalized": false,
@@ -713,7 +713,7 @@
       "single_word": false,
       "special": true
     },
-    "40086": {
+    "16086": {
       "content": "<extra_id_13>",
       "lstrip": true,
       "normalized": false,
@@ -721,7 +721,7 @@
       "single_word": false,
       "special": true
     },
-    "40087": {
+    "16087": {
       "content": "<extra_id_12>",
       "lstrip": true,
       "normalized": false,
@@ -729,7 +729,7 @@
       "single_word": false,
       "special": true
     },
-    "40088": {
+    "16088": {
       "content": "<extra_id_11>",
       "lstrip": true,
       "normalized": false,
@@ -737,7 +737,7 @@
       "single_word": false,
       "special": true
     },
-    "40089": {
+    "16089": {
       "content": "<extra_id_10>",
       "lstrip": true,
       "normalized": false,
@@ -745,7 +745,7 @@
       "single_word": false,
       "special": true
     },
-    "40090": {
+    "16090": {
       "content": "<extra_id_9>",
       "lstrip": true,
       "normalized": false,
@@ -753,7 +753,7 @@
       "single_word": false,
       "special": true
     },
-    "40091": {
+    "16091": {
       "content": "<extra_id_8>",
       "lstrip": true,
       "normalized": false,
@@ -761,7 +761,7 @@
       "single_word": false,
       "special": true
     },
-    "40092": {
+    "16092": {
       "content": "<extra_id_7>",
       "lstrip": true,
       "normalized": false,
@@ -769,7 +769,7 @@
       "single_word": false,
       "special": true
     },
-    "40093": {
+    "16093": {
       "content": "<extra_id_6>",
       "lstrip": true,
       "normalized": false,
@@ -777,7 +777,7 @@
       "single_word": false,
       "special": true
     },
-    "40094": {
+    "16094": {
       "content": "<extra_id_5>",
       "lstrip": true,
       "normalized": false,
@@ -785,7 +785,7 @@
       "single_word": false,
       "special": true
     },
-    "40095": {
+    "16095": {
       "content": "<extra_id_4>",
       "lstrip": true,
       "normalized": false,
@@ -793,7 +793,7 @@
       "single_word": false,
       "special": true
     },
-    "40096": {
+    "16096": {
       "content": "<extra_id_3>",
       "lstrip": true,
       "normalized": false,
@@ -801,7 +801,7 @@
       "single_word": false,
       "special": true
     },
-    "40097": {
+    "16097": {
       "content": "<extra_id_2>",
       "lstrip": true,
       "normalized": false,
@@ -809,7 +809,7 @@
       "single_word": false,
       "special": true
     },
-    "40098": {
+    "16098": {
       "content": "<extra_id_1>",
       "lstrip": true,
       "normalized": false,
@@ -817,7 +817,7 @@
       "single_word": false,
       "special": true
     },
-    "40099": {
+    "16099": {
       "content": "<extra_id_0>",
       "lstrip": true,
       "normalized": false,

src/train_t5.py

@@ -692,4 +692,4 @@ def train(args):
 
 if __name__ == "__main__":
     args = parse_args()
-    train(args)
+    train(args)

src/train_tokeniser.py

@@ -31,9 +31,6 @@ with open("src/data/tokeniser_corpus.txt", "w", encoding="utf-8") as f_out:
 
         with open(corpus_file, "r", encoding="utf-8") as f_in:
             for i, line in enumerate(f_in):
-                if total_records >= 3000000:  # limit total records for tokeniser
-                    break
-
                 item = json.loads(line)
                 dialect = item["transliteration"].get("dialect", "west")
                 prefix = DIALECT_PREFIXES.get(dialect, DIALECT_PREFIXES["west"])
@@ -57,10 +54,10 @@ print("Training SentencePiece model...")
 spm.SentencePieceTrainer.Train(
     input="src/data/tokeniser_corpus.txt",
     model_prefix="src/tokeniser/aramt5_sp",
-    vocab_size=40000,  # adjust as needed
+    vocab_size=16000,  # adjust as needed
     model_type="unigram",  # worth testing with "bpe"
     character_coverage=1.0,  # to preserve rare characters
-    max_sentence_length=8384 * 3,  # to handle long sequences
+    max_sentence_length=8384 * 2,  # to handle long sequences
     pad_id=0,
     unk_id=1,
     bos_id=2,
@@ -80,4 +77,4 @@ tokenizer = T5Tokenizer(vocab_file="src/tokeniser/aramt5_sp.model", legacy=False
 
 tokenizer.save_pretrained("src/tokeniser/")
 print("Tokeniser saved to src/tokeniser/")
-print("Next step: Run train_t5.py to train the T5 model using this tokeniser.")
+print("Next step: Run train_t5.py to train the T5 model using this tokeniser.")

tokenizer_config.json

@@ -49,7 +49,7 @@
       "single_word": false,
       "special": false
     },
-    "40000": {
+    "16000": {
       "content": "<extra_id_99>",
       "lstrip": true,
       "normalized": false,
@@ -57,7 +57,7 @@
       "single_word": false,
       "special": true
     },
-    "40001": {
+    "16001": {
       "content": "<extra_id_98>",
       "lstrip": true,
       "normalized": false,
@@ -65,7 +65,7 @@
       "single_word": false,
       "special": true
     },
-    "40002": {
+    "16002": {
       "content": "<extra_id_97>",
       "lstrip": true,
       "normalized": false,
@@ -73,7 +73,7 @@
       "single_word": false,
       "special": true
     },
-    "40003": {
+    "16003": {
       "content": "<extra_id_96>",
       "lstrip": true,
       "normalized": false,
@@ -81,7 +81,7 @@
       "single_word": false,
       "special": true
     },
-    "40004": {
+    "16004": {
       "content": "<extra_id_95>",
       "lstrip": true,
       "normalized": false,
@@ -89,7 +89,7 @@
       "single_word": false,
       "special": true
     },
-    "40005": {
+    "16005": {
       "content": "<extra_id_94>",
       "lstrip": true,
       "normalized": false,
@@ -97,7 +97,7 @@
       "single_word": false,
       "special": true
     },
-    "40006": {
+    "16006": {
       "content": "<extra_id_93>",
       "lstrip": true,
       "normalized": false,
@@ -105,7 +105,7 @@
       "single_word": false,
       "special": true
     },
-    "40007": {
+    "16007": {
       "content": "<extra_id_92>",
       "lstrip": true,
       "normalized": false,
@@ -113,7 +113,7 @@
       "single_word": false,
       "special": true
     },
-    "40008": {
+    "16008": {
       "content": "<extra_id_91>",
       "lstrip": true,
       "normalized": false,
@@ -121,7 +121,7 @@
       "single_word": false,
       "special": true
     },
-    "40009": {
+    "16009": {
       "content": "<extra_id_90>",
       "lstrip": true,
       "normalized": false,
@@ -129,7 +129,7 @@
       "single_word": false,
       "special": true
     },
-    "40010": {
+    "16010": {
       "content": "<extra_id_89>",
       "lstrip": true,
       "normalized": false,
@@ -137,7 +137,7 @@
       "single_word": false,
       "special": true
     },
-    "40011": {
+    "16011": {
       "content": "<extra_id_88>",
       "lstrip": true,
       "normalized": false,
@@ -145,7 +145,7 @@
       "single_word": false,
       "special": true
     },
-    "40012": {
+    "16012": {
       "content": "<extra_id_87>",
       "lstrip": true,
       "normalized": false,
@@ -153,7 +153,7 @@
       "single_word": false,
       "special": true
     },
-    "40013": {
+    "16013": {
       "content": "<extra_id_86>",
       "lstrip": true,
       "normalized": false,
@@ -161,7 +161,7 @@
       "single_word": false,
       "special": true
     },
-    "40014": {
+    "16014": {
       "content": "<extra_id_85>",
       "lstrip": true,
       "normalized": false,
@@ -169,7 +169,7 @@
       "single_word": false,
       "special": true
     },
-    "40015": {
+    "16015": {
       "content": "<extra_id_84>",
       "lstrip": true,
       "normalized": false,
@@ -177,7 +177,7 @@
       "single_word": false,
       "special": true
     },
-    "40016": {
+    "16016": {
       "content": "<extra_id_83>",
       "lstrip": true,
       "normalized": false,
@@ -185,7 +185,7 @@
       "single_word": false,
       "special": true
     },
-    "40017": {
+    "16017": {
       "content": "<extra_id_82>",
       "lstrip": true,
       "normalized": false,
@@ -193,7 +193,7 @@
       "single_word": false,
       "special": true
     },
-    "40018": {
+    "16018": {
       "content": "<extra_id_81>",
       "lstrip": true,
       "normalized": false,
@@ -201,7 +201,7 @@
       "single_word": false,
       "special": true
     },
-    "40019": {
+    "16019": {
       "content": "<extra_id_80>",
       "lstrip": true,
       "normalized": false,
@@ -209,7 +209,7 @@
       "single_word": false,
       "special": true
     },
-    "40020": {
+    "16020": {
       "content": "<extra_id_79>",
       "lstrip": true,
       "normalized": false,
@@ -217,7 +217,7 @@
       "single_word": false,
       "special": true
     },
-    "40021": {
+    "16021": {
       "content": "<extra_id_78>",
       "lstrip": true,
       "normalized": false,
@@ -225,7 +225,7 @@
       "single_word": false,
       "special": true
     },
-    "40022": {
+    "16022": {
       "content": "<extra_id_77>",
       "lstrip": true,
       "normalized": false,
@@ -233,7 +233,7 @@
       "single_word": false,
       "special": true
     },
-    "40023": {
+    "16023": {
       "content": "<extra_id_76>",
       "lstrip": true,
       "normalized": false,
@@ -241,7 +241,7 @@
       "single_word": false,
       "special": true
     },
-    "40024": {
+    "16024": {
       "content": "<extra_id_75>",
       "lstrip": true,
       "normalized": false,
@@ -249,7 +249,7 @@
       "single_word": false,
       "special": true
     },
-    "40025": {
+    "16025": {
       "content": "<extra_id_74>",
       "lstrip": true,
       "normalized": false,
@@ -257,7 +257,7 @@
       "single_word": false,
       "special": true
     },
-    "40026": {
+    "16026": {
       "content": "<extra_id_73>",
       "lstrip": true,
       "normalized": false,
@@ -265,7 +265,7 @@
       "single_word": false,
       "special": true
     },
-    "40027": {
+    "16027": {
       "content": "<extra_id_72>",
       "lstrip": true,
       "normalized": false,
@@ -273,7 +273,7 @@
       "single_word": false,
       "special": true
     },
-    "40028": {
+    "16028": {
       "content": "<extra_id_71>",
       "lstrip": true,
       "normalized": false,
@@ -281,7 +281,7 @@
       "single_word": false,
       "special": true
     },
-    "40029": {
+    "16029": {
       "content": "<extra_id_70>",
       "lstrip": true,
       "normalized": false,
@@ -289,7 +289,7 @@
       "single_word": false,
       "special": true
     },
-    "40030": {
+    "16030": {
       "content": "<extra_id_69>",
       "lstrip": true,
       "normalized": false,
@@ -297,7 +297,7 @@
       "single_word": false,
       "special": true
     },
-    "40031": {
+    "16031": {
       "content": "<extra_id_68>",
       "lstrip": true,
       "normalized": false,
@@ -305,7 +305,7 @@
       "single_word": false,
       "special": true
     },
-    "40032": {
+    "16032": {
       "content": "<extra_id_67>",
       "lstrip": true,
       "normalized": false,
@@ -313,7 +313,7 @@
       "single_word": false,
       "special": true
     },
-    "40033": {
+    "16033": {
       "content": "<extra_id_66>",
       "lstrip": true,
       "normalized": false,
@@ -321,7 +321,7 @@
       "single_word": false,
       "special": true
     },
-    "40034": {
+    "16034": {
       "content": "<extra_id_65>",
       "lstrip": true,
       "normalized": false,
@@ -329,7 +329,7 @@
       "single_word": false,
       "special": true
     },
-    "40035": {
+    "16035": {
       "content": "<extra_id_64>",
       "lstrip": true,
       "normalized": false,
@@ -337,7 +337,7 @@
       "single_word": false,
       "special": true
     },
-    "40036": {
+    "16036": {
       "content": "<extra_id_63>",
       "lstrip": true,
       "normalized": false,
@@ -345,7 +345,7 @@
       "single_word": false,
       "special": true
     },
-    "40037": {
+    "16037": {
       "content": "<extra_id_62>",
       "lstrip": true,
       "normalized": false,
@@ -353,7 +353,7 @@
       "single_word": false,
       "special": true
     },
-    "40038": {
+    "16038": {
       "content": "<extra_id_61>",
       "lstrip": true,
       "normalized": false,
@@ -361,7 +361,7 @@
       "single_word": false,
       "special": true
     },
-    "40039": {
+    "16039": {
       "content": "<extra_id_60>",
       "lstrip": true,
       "normalized": false,
@@ -369,7 +369,7 @@
       "single_word": false,
       "special": true
     },
-    "40040": {
+    "16040": {
       "content": "<extra_id_59>",
       "lstrip": true,
       "normalized": false,
@@ -377,7 +377,7 @@
       "single_word": false,
       "special": true
     },
-    "40041": {
+    "16041": {
       "content": "<extra_id_58>",
       "lstrip": true,
       "normalized": false,
@@ -385,7 +385,7 @@
       "single_word": false,
       "special": true
     },
-    "40042": {
+    "16042": {
       "content": "<extra_id_57>",
       "lstrip": true,
       "normalized": false,
@@ -393,7 +393,7 @@
       "single_word": false,
       "special": true
     },
-    "40043": {
+    "16043": {
       "content": "<extra_id_56>",
       "lstrip": true,
       "normalized": false,
@@ -401,7 +401,7 @@
       "single_word": false,
       "special": true
     },
-    "40044": {
+    "16044": {
       "content": "<extra_id_55>",
       "lstrip": true,
       "normalized": false,
@@ -409,7 +409,7 @@
       "single_word": false,
       "special": true
     },
-    "40045": {
+    "16045": {
       "content": "<extra_id_54>",
       "lstrip": true,
       "normalized": false,
@@ -417,7 +417,7 @@
       "single_word": false,
       "special": true
     },
-    "40046": {
+    "16046": {
       "content": "<extra_id_53>",
       "lstrip": true,
       "normalized": false,
@@ -425,7 +425,7 @@
       "single_word": false,
       "special": true
     },
-    "40047": {
+    "16047": {
       "content": "<extra_id_52>",
       "lstrip": true,
       "normalized": false,
@@ -433,7 +433,7 @@
       "single_word": false,
       "special": true
     },
-    "40048": {
+    "16048": {
       "content": "<extra_id_51>",
       "lstrip": true,
       "normalized": false,
@@ -441,7 +441,7 @@
       "single_word": false,
       "special": true
     },
-    "40049": {
+    "16049": {
       "content": "<extra_id_50>",
       "lstrip": true,
       "normalized": false,
@@ -449,7 +449,7 @@
       "single_word": false,
       "special": true
     },
-    "40050": {
+    "16050": {
       "content": "<extra_id_49>",
       "lstrip": true,
       "normalized": false,
@@ -457,7 +457,7 @@
       "single_word": false,
       "special": true
     },
-    "40051": {
+    "16051": {
       "content": "<extra_id_48>",
       "lstrip": true,
       "normalized": false,
@@ -465,7 +465,7 @@
       "single_word": false,
       "special": true
     },
-    "40052": {
+    "16052": {
       "content": "<extra_id_47>",
       "lstrip": true,
       "normalized": false,
@@ -473,7 +473,7 @@
       "single_word": false,
       "special": true
     },
-    "40053": {
+    "16053": {
       "content": "<extra_id_46>",
       "lstrip": true,
       "normalized": false,
@@ -481,7 +481,7 @@
       "single_word": false,
       "special": true
     },
-    "40054": {
+    "16054": {
       "content": "<extra_id_45>",
       "lstrip": true,
       "normalized": false,
@@ -489,7 +489,7 @@
       "single_word": false,
       "special": true
     },
-    "40055": {
+    "16055": {
       "content": "<extra_id_44>",
       "lstrip": true,
       "normalized": false,
@@ -497,7 +497,7 @@
       "single_word": false,
       "special": true
     },
-    "40056": {
+    "16056": {
       "content": "<extra_id_43>",
       "lstrip": true,
       "normalized": false,
@@ -505,7 +505,7 @@
       "single_word": false,
       "special": true
     },
-    "40057": {
+    "16057": {
       "content": "<extra_id_42>",
       "lstrip": true,
       "normalized": false,
@@ -513,7 +513,7 @@
       "single_word": false,
       "special": true
     },
-    "40058": {
+    "16058": {
       "content": "<extra_id_41>",
       "lstrip": true,
       "normalized": false,
@@ -521,7 +521,7 @@
       "single_word": false,
       "special": true
     },
-    "40059": {
+    "16059": {
       "content": "<extra_id_40>",
       "lstrip": true,
       "normalized": false,
@@ -529,7 +529,7 @@
       "single_word": false,
       "special": true
     },
-    "40060": {
+    "16060": {
       "content": "<extra_id_39>",
       "lstrip": true,
       "normalized": false,
@@ -537,7 +537,7 @@
       "single_word": false,
       "special": true
     },
-    "40061": {
+    "16061": {
       "content": "<extra_id_38>",
       "lstrip": true,
       "normalized": false,
@@ -545,7 +545,7 @@
       "single_word": false,
       "special": true
     },
-    "40062": {
+    "16062": {
       "content": "<extra_id_37>",
       "lstrip": true,
       "normalized": false,
@@ -553,7 +553,7 @@
       "single_word": false,
       "special": true
     },
-    "40063": {
+    "16063": {
       "content": "<extra_id_36>",
       "lstrip": true,
       "normalized": false,
@@ -561,7 +561,7 @@
       "single_word": false,
       "special": true
     },
-    "40064": {
+    "16064": {
       "content": "<extra_id_35>",
       "lstrip": true,
       "normalized": false,
@@ -569,7 +569,7 @@
       "single_word": false,
       "special": true
     },
-    "40065": {
+    "16065": {
       "content": "<extra_id_34>",
       "lstrip": true,
       "normalized": false,
@@ -577,7 +577,7 @@
       "single_word": false,
       "special": true
     },
-    "40066": {
+    "16066": {
       "content": "<extra_id_33>",
       "lstrip": true,
       "normalized": false,
@@ -585,7 +585,7 @@
       "single_word": false,
       "special": true
     },
-    "40067": {
+    "16067": {
       "content": "<extra_id_32>",
       "lstrip": true,
       "normalized": false,
@@ -593,7 +593,7 @@
       "single_word": false,
       "special": true
     },
-    "40068": {
+    "16068": {
       "content": "<extra_id_31>",
       "lstrip": true,
       "normalized": false,
@@ -601,7 +601,7 @@
       "single_word": false,
       "special": true
     },
-    "40069": {
+    "16069": {
       "content": "<extra_id_30>",
       "lstrip": true,
       "normalized": false,
@@ -609,7 +609,7 @@
       "single_word": false,
       "special": true
     },
-    "40070": {
+    "16070": {
       "content": "<extra_id_29>",
       "lstrip": true,
       "normalized": false,
@@ -617,7 +617,7 @@
       "single_word": false,
       "special": true
     },
-    "40071": {
+    "16071": {
       "content": "<extra_id_28>",
       "lstrip": true,
       "normalized": false,
@@ -625,7 +625,7 @@
       "single_word": false,
       "special": true
     },
-    "40072": {
+    "16072": {
       "content": "<extra_id_27>",
       "lstrip": true,
       "normalized": false,
@@ -633,7 +633,7 @@
       "single_word": false,
       "special": true
     },
-    "40073": {
+    "16073": {
       "content": "<extra_id_26>",
       "lstrip": true,
       "normalized": false,
@@ -641,7 +641,7 @@
       "single_word": false,
       "special": true
     },
-    "40074": {
+    "16074": {
       "content": "<extra_id_25>",
       "lstrip": true,
       "normalized": false,
@@ -649,7 +649,7 @@
       "single_word": false,
       "special": true
     },
-    "40075": {
+    "16075": {
       "content": "<extra_id_24>",
       "lstrip": true,
       "normalized": false,
@@ -657,7 +657,7 @@
       "single_word": false,
       "special": true
     },
-    "40076": {
+    "16076": {
       "content": "<extra_id_23>",
       "lstrip": true,
       "normalized": false,
@@ -665,7 +665,7 @@
       "single_word": false,
       "special": true
     },
-    "40077": {
+    "16077": {
       "content": "<extra_id_22>",
       "lstrip": true,
       "normalized": false,
@@ -673,7 +673,7 @@
       "single_word": false,
       "special": true
     },
-    "40078": {
+    "16078": {
       "content": "<extra_id_21>",
       "lstrip": true,
       "normalized": false,
@@ -681,7 +681,7 @@
       "single_word": false,
       "special": true
     },
-    "40079": {
+    "16079": {
       "content": "<extra_id_20>",
       "lstrip": true,
       "normalized": false,
@@ -689,7 +689,7 @@
       "single_word": false,
       "special": true
     },
-    "40080": {
+    "16080": {
       "content": "<extra_id_19>",
       "lstrip": true,
       "normalized": false,
@@ -697,7 +697,7 @@
       "single_word": false,
       "special": true
     },
-    "40081": {
+    "16081": {
       "content": "<extra_id_18>",
       "lstrip": true,
       "normalized": false,
@@ -705,7 +705,7 @@
       "single_word": false,
       "special": true
     },
-    "40082": {
+    "16082": {
       "content": "<extra_id_17>",
       "lstrip": true,
       "normalized": false,
@@ -713,7 +713,7 @@
       "single_word": false,
       "special": true
     },
-    "40083": {
+    "16083": {
       "content": "<extra_id_16>",
       "lstrip": true,
       "normalized": false,
@@ -721,7 +721,7 @@
       "single_word": false,
       "special": true
     },
-    "40084": {
+    "16084": {
       "content": "<extra_id_15>",
       "lstrip": true,
       "normalized": false,
@@ -729,7 +729,7 @@
       "single_word": false,
       "special": true
     },
-    "40085": {
+    "16085": {
       "content": "<extra_id_14>",
       "lstrip": true,
       "normalized": false,
@@ -737,7 +737,7 @@
       "single_word": false,
       "special": true
     },
-    "40086": {
+    "16086": {
       "content": "<extra_id_13>",
       "lstrip": true,
       "normalized": false,
@@ -745,7 +745,7 @@
       "single_word": false,
       "special": true
     },
-    "40087": {
+    "16087": {
       "content": "<extra_id_12>",
       "lstrip": true,
       "normalized": false,
@@ -753,7 +753,7 @@
       "single_word": false,
       "special": true
     },
-    "40088": {
+    "16088": {
       "content": "<extra_id_11>",
       "lstrip": true,
       "normalized": false,
@@ -761,7 +761,7 @@
       "single_word": false,
       "special": true
     },
-    "40089": {
+    "16089": {
       "content": "<extra_id_10>",
       "lstrip": true,
       "normalized": false,
@@ -769,7 +769,7 @@
       "single_word": false,
       "special": true
     },
-    "40090": {
+    "16090": {
       "content": "<extra_id_9>",
       "lstrip": true,
       "normalized": false,
@@ -777,7 +777,7 @@
       "single_word": false,
       "special": true
     },
-    "40091": {
+    "16091": {
       "content": "<extra_id_8>",
       "lstrip": true,
       "normalized": false,
@@ -785,7 +785,7 @@
       "single_word": false,
       "special": true
     },
-    "40092": {
+    "16092": {
       "content": "<extra_id_7>",
       "lstrip": true,
       "normalized": false,
@@ -793,7 +793,7 @@
       "single_word": false,
       "special": true
     },
-    "40093": {
+    "16093": {
       "content": "<extra_id_6>",
       "lstrip": true,
       "normalized": false,
@@ -801,7 +801,7 @@
       "single_word": false,
       "special": true
     },
-    "40094": {
+    "16094": {
       "content": "<extra_id_5>",
       "lstrip": true,
       "normalized": false,
@@ -809,7 +809,7 @@
       "single_word": false,
       "special": true
     },
-    "40095": {
+    "16095": {
       "content": "<extra_id_4>",
       "lstrip": true,
       "normalized": false,
@@ -817,7 +817,7 @@
       "single_word": false,
       "special": true
     },
-    "40096": {
+    "16096": {
       "content": "<extra_id_3>",
       "lstrip": true,
       "normalized": false,
@@ -825,7 +825,7 @@
       "single_word": false,
       "special": true
     },
-    "40097": {
+    "16097": {
       "content": "<extra_id_2>",
       "lstrip": true,
       "normalized": false,
@@ -833,7 +833,7 @@
       "single_word": false,
       "special": true
     },
-    "40098": {
+    "16098": {
       "content": "<extra_id_1>",
       "lstrip": true,
       "normalized": false,
@@ -841,7 +841,7 @@
       "single_word": false,
       "special": true
     },
-    "40099": {
+    "16099": {
       "content": "<extra_id_0>",
       "lstrip": true,
       "normalized": false,