Upload folder using huggingface_hub

README.md

@@ -47,4 +47,8 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, tu, z, Ġ` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 3, 70, 81, 77, 87, 92, 4, 81, 284, 92, 223` |
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, tu, z, Ġ` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 13, 70, 81, 77, 87, 92, 31, 81, 284, 92, 223` |
+
+Command used to create this tokenizer:
+```bash
+['/home/gsa/tokenizers2/flexitok/tokenizer_training/train_tokenizers.py', 'algorithm=bpe', 'vocab_size=2000', 'langs=[ind_Latn]', 'data_dir=/scratch/gsa/data/multilingual-addition/', 'output_dir=/scratch/gsa/trained_tokenizers/multilingual_addition', 'pretokenizer=custom:addition', 'number_handling=ltr_3digit', 'add_numbers=false', 'handle_contractions=false', 'unicode_normalization=nfc', 'use_byte_level_regex=false', 'byte_fallback=false', 'strip_zero_width=false', 'cjk_char_split=false', 'add_cjk_chars=false', 'max_lines=-1', 'test_string=yirmi iki+dokuz=otuz bir\\ntwenty two+nine=thirty one', 'hf.publish_to_hf=true', 'hf_repo_prefix=flexitok/', 'hf.hf_repo_id=flexitok/maddition_ind_Latn_2000', 'hf.collections=[flexitok/multilingual_addition_tokenizers]']

special_tokens_map.json

@@ -1,8 +1,4 @@
 {
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>"

tokenizer.json

@@ -29,24 +29,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 3,
-      "content": "+",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 4,
-      "content": "=",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {
@@ -99,35 +81,35 @@
       "<s>": 0,
       "</s>": 1,
       "<pad>": 2,
-      "+": 3,
-      "=": 4,
-      "!": 5,
-      "\"": 6,
-      "#": 7,
-      "$": 8,
-      "%": 9,
-      "&": 10,
-      "'": 11,
-      "(": 12,
-      ")": 13,
-      "*": 14,
-      ",": 15,
-      "-": 16,
-      ".": 17,
-      "/": 18,
-      "0": 19,
-      "1": 20,
-      "2": 21,
-      "3": 22,
-      "4": 23,
-      "5": 24,
-      "6": 25,
-      "7": 26,
-      "8": 27,
-      "9": 28,
-      ":": 29,
-      ";": 30,
-      "<": 31,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
       ">": 32,
       "?": 33,
       "@": 34,

tokenizer_config.json

@@ -23,28 +23,8 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "3": {
-      "content": "+",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "=",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",

vocab.json

@@ -1,310 +1,310 @@
 {
-  "¿": 126,
-  "ĝ": 220,
-  "ga": 276,
-  "×": 150,
-  "ĉ": 200,
-  "H": 42,
-  "5": 24,
-  "/": 18,
-  "M": 47,
-  "Á": 128,
-  "´": 115,
-  "ą": 196,
-  "bu": 296,
-  "Å": 132,
-  "ã": 162,
-  "û": 186,
-  "sembilan": 293,
-  "Ļ": 250,
-  "Y": 59,
-  "Ę": 215,
-  "g": 73,
-  "ribu": 298,
-  "0": 19,
-  "Í": 140,
-  "ĺ": 249,
-  "Ê": 137,
-  "'": 11,
-  "im": 277,
-  "Ć": 197,
-  "s": 85,
-  "ĥ": 228,
-  "Ø": 151,
-  "N": 48,
-  "»": 122,
-  "¾": 125,
-  "S": 53,
-  "ē": 210,
   "Ó": 146,
+  "belas": 303,
+  "đ": 208,
+  "e": 71,
+  "Č": 203,
+  "Ö": 149,
+  "}": 95,
   "m": 79,
-  ".": 17,
+  "ē": 210,
+  "ï": 174,
+  "ě": 218,
   "L": 46,
-  "đ": 208,
-  "Ð": 143,
-  "Ĩ": 231,
-  "se": 295,
-  "!": 5,
-  "ķ": 246,
-  "À": 127,
-  "x": 90,
-  "1": 20,
-  "satu": 302,
-  "`": 66,
-  "Â": 129,
-  "È": 135,
-  "¥": 101,
-  "9": 28,
-  "Ñ": 144,
-  "õ": 180,
-  "ë": 170,
+  "|": 94,
+  ")": 11,
   "Ċ": 201,
-  "à": 159,
-  "juh": 279,
-  "ø": 183,
-  "=": 4,
-  "C": 37,
-  "w": 89,
-  "æ": 165,
-  "«": 107,
-  "Ç": 134,
-  "h": 74,
-  "puluh": 266,
-  "ì": 171,
-  "v": 88,
-  "ŀ": 255,
-  "ė": 214,
-  "ï": 174,
+  "7": 25,
+  "ģ": 226,
+  "Ü": 155,
+  "I": 43,
+  "<pad>": 2,
+  "9": 27,
+  "=": 31,
+  "³": 114,
+  "!": 3,
+  "bu": 296,
+  "W": 57,
+  "t": 86,
+  "*": 12,
+  "ö": 181,
+  "ò": 177,
+  "tu": 284,
+  "<s>": 0,
   "A": 35,
-  "iga": 278,
   "pu": 265,
-  "İ": 239,
-  "Þ": 157,
-  "ä": 163,
-  "Č": 203,
-  "é": 168,
-  "uh": 260,
-  "ß": 158,
-  "tiga": 285,
-  "í": 172,
-  "j": 76,
-  "dela": 274,
-  "ibu": 297,
-  "F": 40,
-  "ĩ": 232,
-  "Ù": 152,
-  "(": 12,
-  "&": 10,
-  "]": 63,
-  "į": 238,
-  "Ë": 138,
-  "*": 14,
-  "¬": 108,
+  "2": 20,
+  "ķ": 246,
+  "6": 24,
+  "Ê": 137,
+  "Ç": 134,
   "c": 69,
-  "±": 112,
-  "Į": 237,
-  ":": 29,
-  "d": 70,
-  "ratus": 263,
+  "iga": 278,
+  "Ľ": 252,
+  "¿": 126,
+  "ć": 198,
+  "á": 160,
+  "ã": 162,
+  "luh": 264,
+  "ibu": 297,
   "ā": 192,
-  "®": 109,
-  "W": 57,
+  "Ą": 195,
+  "Ě": 217,
   "ĸ": 247,
-  "ĕ": 212,
-  "Ē": 209,
-  "E": 39,
-  "lim": 280,
-  "©": 105,
+  "«": 107,
   "T": 54,
-  "Û": 154,
-  "o": 81,
-  "ò": 177,
-  "¸": 119,
-  "nol": 307,
-  "|": 94,
-  "tujuh": 294,
-  "ü": 187,
-  "bilan": 288,
-  "P": 50,
-  "<pad>": 2,
-  "t": 86,
-  "Ĝ": 219,
-  "³": 114,
-  "em": 268,
-  "^": 64,
-  "Ą": 195,
-  "ý": 188,
-  "seribu": 299,
-  "ċ": 202,
-  "Ò": 145,
-  "Õ": 148,
-  "â": 161,
-  "la": 267,
-  ",": 15,
-  "¯": 110,
-  "ę": 216,
-  "Ľ": 252,
-  "Ĕ": 211,
-  "ù": 184,
-  "-": 16,
+  "Ġ": 223,
+  "ą": 196,
+  "×": 150,
+  "~": 96,
+  "å": 164,
+  "Ĥ": 227,
+  "{": 93,
+  "ğ": 222,
   "Ä": 131,
-  "tu": 284,
-  "Ĵ": 243,
-  "§": 103,
-  "Ĺ": 248,
-  "Ď": 205,
-  "ñ": 176,
-  "ª": 106,
-  "º": 121,
-  "Ĉ": 199,
-  "_": 65,
-  "4": 23,
   "\\": 62,
-  "u": 87,
-  "[": 61,
-  "½": 124,
-  "¦": 102,
-  "%": 9,
-  "Q": 51,
-  "luh": 264,
-  "Ħ": 229,
-  "sebelas": 305,
-  "seratus": 300,
-  "î": 173,
-  "J": 44,
-  "Ģ": 225,
-  "ĵ": 244,
-  "Î": 141,
-  "X": 58,
-  "ô": 179,
-  "ij": 242,
-  "ela": 269,
-  "ī": 234,
-  "¶": 117,
-  "ă": 194,
+  "Ã": 130,
+  "U": 55,
+  "Ē": 209,
+  "¹": 120,
+  "</s>": 1,
   "ÿ": 190,
-  "Ń": 258,
-  "¡": 97,
-  "Z": 60,
-  "e": 71,
-  "ć": 198,
-  "Ô": 147,
-  "Ü": 155,
-  "?": 33,
+  "IJ": 241,
+  "Ú": 153,
+  "dua": 289,
+  "bi": 272,
+  "Ī": 233,
+  "¥": 101,
+  "¤": 100,
+  "ß": 158,
+  "Ŀ": 254,
+  "S": 53,
+  "Q": 51,
+  "s": 85,
+  "¾": 125,
+  "\"": 4,
+  "Ò": 145,
+  "ú": 185,
+  "`": 66,
+  "Ý": 156,
+  "1": 19,
+  "ä": 163,
+  "uh": 260,
+  "ĥ": 228,
+  "ª": 106,
+  "ê": 169,
+  "Å": 132,
+  "ĝ": 220,
+  "ë": 170,
+  "F": 40,
+  "Ĵ": 243,
+  "İ": 239,
+  "ŀ": 255,
+  "ľ": 253,
   "q": 83,
-  "</s>": 1,
-  "D": 38,
-  "V": 56,
-  "Ġ": 223,
   "ratu": 262,
-  "B": 36,
-  "Ł": 256,
-  "no": 306,
-  "y": 91,
-  "8": 27,
-  "¨": 104,
-  ";": 30,
-  "è": 167,
-  "du": 273,
-  "lan": 286,
-  "ĭ": 236,
-  "k": 77,
-  "bi": 272,
-  "b": 68,
-  "sem": 283,
-  "lima": 292,
-  "bela": 301,
-  "3": 22,
-  "Ì": 139,
-  "~": 96,
-  "þ": 189,
-  "ð": 175,
-  "pat": 281,
-  "Ï": 142,
-  "n": 80,
-  "z": 92,
-  "U": 55,
   "K": 45,
-  "á": 160,
-  "ģ": 226,
-  "<": 31,
-  "¹": 120,
-  "Ú": 153,
-  "{": 93,
+  "Y": 59,
   "@": 34,
-  "ê": 169,
-  "empat": 287,
-  "ď": 206,
-  "I": 43,
-  "ħ": 230,
-  "°": 111,
-  "Ý": 156,
-  "G": 41,
-  ">": 32,
-  "č": 204,
-  "2": 21,
+  "ratus": 263,
+  "Ļ": 250,
+  "´": 115,
+  "µ": 116,
+  "am": 270,
+  "g": 73,
+  "x": 90,
+  "se": 295,
+  "Ù": 152,
+  "D": 38,
+  "N": 48,
+  "V": 56,
+  "Ĭ": 235,
+  "n": 80,
+  "Ì": 139,
+  "o": 81,
+  "[": 61,
+  "w": 89,
+  "dela": 274,
+  "Ę": 215,
+  "Ć": 197,
+  "M": 47,
+  "Þ": 157,
+  "lan": 286,
+  "À": 127,
+  "tujuh": 294,
+  "juh": 279,
+  "Ā": 191,
+  "æ": 165,
+  "¦": 102,
+  "P": 50,
+  ";": 29,
   "O": 49,
-  "ġ": 224,
-  "å": 164,
+  "sembilan": 293,
   "r": 84,
-  "Ě": 217,
-  "IJ": 241,
-  "p": 82,
+  "Î": 141,
+  "ĩ": 232,
   "ı": 240,
-  "pan": 282,
-  "¢": 98,
-  "Ķ": 245,
-  "Ğ": 221,
-  "#": 7,
-  "l": 78,
-  "}": 95,
-  "£": 99,
-  ")": 13,
-  "<s>": 0,
-  "a": 67,
+  "ĺ": 249,
   "ł": 257,
-  "Ö": 149,
-  "an": 271,
-  "\"": 6,
-  "belas": 303,
-  "6": 25,
-  "sepuluh": 304,
-  "µ": 116,
+  "ĭ": 236,
+  "satu": 302,
+  "¢": 98,
+  "Ď": 205,
+  "Ë": 138,
+  "3": 21,
   "·": 118,
-  "Ã": 130,
-  "ç": 166,
-  "am": 270,
-  "ö": 181,
+  "ċ": 202,
+  "Æ": 133,
+  "y": 91,
+  "at": 259,
+  "ribu": 298,
+  "°": 111,
+  "Z": 60,
+  ",": 14,
+  "Ð": 143,
+  "]": 63,
+  ":": 28,
+  "a": 67,
+  "ù": 184,
+  "sem": 283,
+  "nol": 307,
   "delapan": 290,
+  "Í": 140,
   "enam": 291,
-  "R": 52,
+  "ĕ": 212,
+  "C": 37,
+  "tiga": 285,
+  "#": 5,
+  "Û": 154,
+  "Ń": 258,
+  "H": 42,
+  "¡": 97,
+  "4": 22,
+  "no": 306,
+  "ă": 194,
+  "ĉ": 200,
+  ">": 32,
+  "¸": 119,
+  "¨": 104,
+  "¯": 110,
+  "sepuluh": 304,
+  "¶": 117,
+  "»": 122,
+  "Ï": 142,
+  "i": 75,
+  "l": 78,
+  "ø": 183,
+  "Ô": 147,
+  "¼": 123,
+  "Ĝ": 219,
+  "Đ": 207,
+  "ď": 206,
+  "Á": 128,
+  "ý": 188,
+  "en": 275,
+  "Ă": 193,
+  "¬": 108,
+  "(": 10,
+  "Ĕ": 211,
+  "õ": 180,
+  "ę": 216,
+  "<": 30,
+  "Ñ": 144,
+  "^": 64,
+  "ü": 187,
   "²": 113,
-  "ğ": 222,
+  "seratus": 300,
+  "ij": 242,
+  "lima": 292,
+  "£": 99,
+  "ó": 178,
+  "è": 167,
   "÷": 182,
+  "G": 41,
+  "/": 17,
+  "R": 52,
+  "ħ": 230,
+  "0": 18,
+  "sebelas": 305,
+  "puluh": 266,
+  "z": 92,
+  "į": 238,
+  "ī": 234,
+  "empat": 287,
+  "ļ": 251,
+  "h": 74,
+  "seribu": 299,
   "Ė": 213,
-  "Ă": 193,
-  "ó": 178,
-  "en": 275,
-  "7": 26,
-  "ú": 185,
-  "$": 8,
-  "¼": 123,
-  "dua": 289,
-  "ľ": 253,
-  "Đ": 207,
-  "f": 72,
-  "ě": 218,
-  "i": 75,
-  "Ĭ": 235,
-  "¤": 100,
+  "þ": 189,
+  "%": 7,
+  "û": 186,
+  "â": 161,
+  "$": 6,
+  "B": 36,
+  "î": 173,
+  "u": 87,
+  "Į": 237,
+  "©": 105,
+  "é": 168,
   "É": 136,
-  "Ŀ": 254,
-  "+": 3,
-  "ļ": 251,
-  "Ĥ": 227,
-  "at": 259,
-  "Ī": 233,
-  "Ā": 191,
-  "Æ": 133,
-  "atu": 261
+  "Ħ": 229,
+  "Ĩ": 231,
+  "Ķ": 245,
+  "ga": 276,
+  "ô": 179,
+  "+": 13,
+  "ĵ": 244,
+  "à": 159,
+  "f": 72,
+  "an": 271,
+  "la": 267,
+  "Ğ": 221,
+  "pan": 282,
+  "5": 23,
+  "Ø": 151,
+  ".": 16,
+  "Ĺ": 248,
+  "lim": 280,
+  "Õ": 148,
+  "º": 121,
+  "atu": 261,
+  "ð": 175,
+  "®": 109,
+  "Ģ": 225,
+  "d": 70,
+  "j": 76,
+  "&": 8,
+  "em": 268,
+  "pat": 281,
+  "?": 33,
+  "-": 15,
+  "§": 103,
+  "im": 277,
+  "½": 124,
+  "bela": 301,
+  "b": 68,
+  "p": 82,
+  "k": 77,
+  "č": 204,
+  "ela": 269,
+  "ì": 171,
+  "ġ": 224,
+  "±": 112,
+  "ė": 214,
+  "E": 39,
+  "8": 26,
+  "v": 88,
+  "J": 44,
+  "_": 65,
+  "'": 9,
+  "du": 273,
+  "bilan": 288,
+  "X": 58,
+  "í": 172,
+  "Ĉ": 199,
+  "ñ": 176,
+  "È": 135,
+  "ç": 166,
+  "Â": 129,
+  "Ł": 256
 }