File size: 15,036 Bytes
9846dad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | [
{
"name": "shared_bpe_8000",
"tokenizer_type": "shared",
"algorithm": "bpe",
"vocab_size": 8000,
"fertility_overall": 1.5744684296670828,
"fertility_ar": 1.4755656463463651,
"fertility_az": 1.7595859759130557,
"cpt_overall": 3.5313313486070697,
"cpt_ar": 3.812953050264278,
"cpt_az": 3.0042165735724513,
"fertility_disparity": 0.16141315823985886,
"cpt_disparity": 0.21210239571026782,
"oov_rate": 0.0,
"vocab_gini": 0.7419718947527658,
"shannon_entropy": 10.725417379440934,
"exact_match_ar": 0.9990480507706255,
"exact_match_az": 0.9992363821483116
},
{
"name": "concat_bpe_8000",
"tokenizer_type": "concatenated",
"algorithm": "bpe",
"vocab_size": 8000,
"fertility_overall": 1.625089409755206,
"fertility_ar": 1.677255929872786,
"fertility_az": 1.5274486975717831,
"cpt_overall": 3.4121822358457132,
"cpt_ar": 3.3745329363977343,
"cpt_az": 3.4826508889784487,
"fertility_disparity": 0.0893168595399542,
"cpt_disparity": 0.031044728865266227,
"oov_rate": 0.0,
"vocab_gini": 0.7155357357913302,
"shannon_entropy": 10.758976129080413,
"exact_match_ar": 0.9989120580235721,
"exact_match_az": 0.9964364500254539
},
{
"name": "shared_unigram_8000",
"tokenizer_type": "shared",
"algorithm": "unigram",
"vocab_size": 8000,
"fertility_overall": 1.5860234519252308,
"fertility_ar": 1.5098217608254827,
"fertility_az": 1.7286510868870875,
"cpt_overall": 3.513922137310213,
"cpt_ar": 3.7495598617218993,
"cpt_az": 3.072876133532698,
"fertility_disparity": 0.12658964421540223,
"cpt_disparity": 0.18047017600579116,
"oov_rate": 0.0,
"vocab_gini": 0.8011276471980863,
"shannon_entropy": 10.278366552533573,
"exact_match_ar": 0.9990480507706255,
"exact_match_az": 0.9992363821483116
},
{
"name": "concat_unigram_8000",
"tokenizer_type": "concatenated",
"algorithm": "unigram",
"vocab_size": 8000,
"fertility_overall": 1.630557495524919,
"fertility_ar": 1.6828221737842166,
"fertility_az": 1.5327330595500248,
"cpt_overall": 3.4256065103693167,
"cpt_ar": 3.3890978047599254,
"cpt_az": 3.4939403000980764,
"fertility_disparity": 0.08918893307465849,
"cpt_disparity": 0.030006950987459077,
"oov_rate": 0.0,
"vocab_gini": 0.7843741193591066,
"shannon_entropy": 10.258501048163726,
"exact_match_ar": 0.9989120580235721,
"exact_match_az": 0.9964364500254539
},
{
"name": "shared_wordpiece_8000",
"tokenizer_type": "shared",
"algorithm": "wordpiece",
"vocab_size": 8000,
"fertility_overall": 1.5720141231575526,
"fertility_ar": 1.4715587661386824,
"fertility_az": 1.7600376405371791,
"cpt_overall": 3.5349251084819273,
"cpt_ar": 3.819823570325556,
"cpt_az": 3.001677181426908,
"fertility_disparity": 0.16390494598197936,
"cpt_disparity": 0.21418433962616734,
"oov_rate": 0.0,
"vocab_gini": 0.7254354187229622,
"shannon_entropy": 10.734803471006908,
"exact_match_ar": 0.0,
"exact_match_az": 0.0
},
{
"name": "concat_wordpiece_8000",
"tokenizer_type": "concatenated",
"algorithm": "wordpiece",
"vocab_size": 8000,
"fertility_overall": 1.6305196797375283,
"fertility_ar": 1.679880351462672,
"fertility_az": 1.5381307082241507,
"cpt_overall": 3.397665609111008,
"cpt_ar": 3.366673383613362,
"cpt_az": 3.4556741357085015,
"fertility_disparity": 0.08438079718897826,
"cpt_disparity": 0.02575496085567456,
"oov_rate": 0.0,
"vocab_gini": 0.6940886180639433,
"shannon_entropy": 10.748321790799139,
"exact_match_ar": 0.0,
"exact_match_az": 0.0
},
{
"name": "shared_bbpe_8000",
"tokenizer_type": "shared",
"algorithm": "bbpe",
"vocab_size": 8000,
"fertility_overall": 1.7155041180862605,
"fertility_ar": 1.5432842580927282,
"fertility_az": 2.037850131276259,
"cpt_overall": 3.2675384671739276,
"cpt_ar": 3.6389791874263038,
"cpt_az": 2.5723083391604016,
"fertility_disparity": 0.24269001218151182,
"cpt_disparity": 0.2931236463103581,
"oov_rate": 0.0,
"vocab_gini": 0.7852085662214892,
"shannon_entropy": 10.17036760209476,
"exact_match_ar": 0.9994106980961015,
"exact_match_az": 0.9994909214322077
},
{
"name": "concat_bbpe_8000",
"tokenizer_type": "concatenated",
"algorithm": "bbpe",
"vocab_size": 8000,
"fertility_overall": 1.7695328842122233,
"fertility_ar": 1.733122664722859,
"fertility_az": 1.8376823362685084,
"cpt_overall": 3.130408968636395,
"cpt_ar": 3.268439481034545,
"cpt_az": 2.872055574482044,
"fertility_disparity": 0.056897576627940066,
"cpt_disparity": 0.12127619582756817,
"oov_rate": 0.0,
"vocab_gini": 0.7652784949485903,
"shannon_entropy": 10.232825855115562,
"exact_match_ar": 0.999909338168631,
"exact_match_az": 0.9997454607161038
},
{
"name": "shared_bpe_16000",
"tokenizer_type": "shared",
"algorithm": "bpe",
"vocab_size": 16000,
"fertility_overall": 1.406466326610023,
"fertility_ar": 1.3322457895485136,
"fertility_az": 1.545385811386613,
"cpt_overall": 3.9381818445330614,
"cpt_ar": 4.211736441809671,
"cpt_az": 3.426166367193676,
"fertility_disparity": 0.13792026577936378,
"cpt_disparity": 0.18651928615895444,
"oov_rate": 0.0,
"vocab_gini": 0.7705612275154623,
"shannon_entropy": 11.22051844286477,
"exact_match_ar": 0.9990480507706255,
"exact_match_az": 0.9992363821483116
},
{
"name": "concat_bpe_16000",
"tokenizer_type": "concatenated",
"algorithm": "bpe",
"vocab_size": 16000,
"fertility_overall": 1.4427306790554728,
"fertility_ar": 1.4863282848922221,
"fertility_az": 1.3611285082800872,
"cpt_overall": 3.843167233342444,
"cpt_ar": 3.806560946253488,
"cpt_az": 3.9116836675170874,
"fertility_disparity": 0.08423426902705648,
"cpt_disparity": 0.02687403435419539,
"oov_rate": 0.0,
"vocab_gini": 0.7502784093339115,
"shannon_entropy": 11.289933167496804,
"exact_match_ar": 0.9989120580235721,
"exact_match_az": 0.9964364500254539
},
{
"name": "shared_unigram_16000",
"tokenizer_type": "shared",
"algorithm": "unigram",
"vocab_size": 16000,
"fertility_overall": 1.43915848471923,
"fertility_ar": 1.383798509647741,
"fertility_az": 1.542776425502961,
"cpt_overall": 3.856083364492832,
"cpt_ar": 4.074326867754011,
"cpt_az": 3.4475943366681565,
"fertility_disparity": 0.10304663282846806,
"cpt_disparity": 0.1538248038089647,
"oov_rate": 0.0,
"vocab_gini": 0.8202410337846489,
"shannon_entropy": 10.661726018051654,
"exact_match_ar": 0.9990480507706255,
"exact_match_az": 0.9992363821483116
},
{
"name": "concat_unigram_16000",
"tokenizer_type": "concatenated",
"algorithm": "unigram",
"vocab_size": 16000,
"fertility_overall": 1.4690109341605053,
"fertility_ar": 1.5133852390561142,
"fertility_az": 1.3859550062802124,
"cpt_overall": 3.7990700186667543,
"cpt_ar": 3.7636569223952008,
"cpt_az": 3.865353143030446,
"fertility_disparity": 0.08420211158883709,
"cpt_disparity": 0.026309684231209756,
"oov_rate": 0.0,
"vocab_gini": 0.8131049651034313,
"shannon_entropy": 10.680674921480614,
"exact_match_ar": 0.9989120580235721,
"exact_match_az": 0.9964364500254539
},
{
"name": "shared_wordpiece_16000",
"tokenizer_type": "shared",
"algorithm": "wordpiece",
"vocab_size": 16000,
"fertility_overall": 1.4017223613822598,
"fertility_ar": 1.3278822099406427,
"fertility_az": 1.5399298737530456,
"cpt_overall": 3.949283003677344,
"cpt_ar": 4.223103615213092,
"cpt_az": 3.43676962420351,
"fertility_disparity": 0.1376995585491242,
"cpt_disparity": 0.18619812883040157,
"oov_rate": 0.0,
"vocab_gini": 0.7570477677642601,
"shannon_entropy": 11.225647699841687,
"exact_match_ar": 0.0,
"exact_match_az": 0.0
},
{
"name": "concat_wordpiece_16000",
"tokenizer_type": "concatenated",
"algorithm": "wordpiece",
"vocab_size": 16000,
"fertility_overall": 1.441782460775622,
"fertility_ar": 1.4835885687430344,
"fertility_az": 1.3635334584201908,
"cpt_overall": 3.8430675831500807,
"cpt_ar": 3.81082194270718,
"cpt_az": 3.9034221417934196,
"fertility_disparity": 0.08092210526032824,
"cpt_disparity": 0.0237228246708911,
"oov_rate": 0.0,
"vocab_gini": 0.7327455145352297,
"shannon_entropy": 11.280193007623335,
"exact_match_ar": 0.0,
"exact_match_az": 0.0
},
{
"name": "shared_bbpe_16000",
"tokenizer_type": "shared",
"algorithm": "bbpe",
"vocab_size": 16000,
"fertility_overall": 1.563251758755881,
"fertility_ar": 1.407745203379972,
"fertility_az": 1.8543152757754426,
"cpt_overall": 3.5796704536935944,
"cpt_ar": 3.9802348152214964,
"cpt_az": 2.8299292509695566,
"fertility_disparity": 0.24082747860054313,
"cpt_disparity": 0.2890044476403401,
"oov_rate": 0.0,
"vocab_gini": 0.8152607254871892,
"shannon_entropy": 10.485685853996797,
"exact_match_ar": 0.9994106980961015,
"exact_match_az": 0.9994909214322077
},
{
"name": "concat_bbpe_16000",
"tokenizer_type": "concatenated",
"algorithm": "bbpe",
"vocab_size": 16000,
"fertility_overall": 1.6044565681716603,
"fertility_ar": 1.5508741409303313,
"fertility_az": 1.7047474509939684,
"cpt_overall": 3.4578261987966146,
"cpt_ar": 3.653251881231358,
"cpt_az": 3.0920455646111016,
"fertility_disparity": 0.09026164548532975,
"cpt_disparity": 0.15361829265139457,
"oov_rate": 0.0,
"vocab_gini": 0.8039903620272112,
"shannon_entropy": 10.550324007405273,
"exact_match_ar": 0.999909338168631,
"exact_match_az": 0.9997454607161038
},
{
"name": "shared_bpe_32000",
"tokenizer_type": "shared",
"algorithm": "bpe",
"vocab_size": 32000,
"fertility_overall": 1.2796960189731112,
"fertility_ar": 1.2326026481342502,
"fertility_az": 1.3678412557544855,
"cpt_overall": 4.304399960106754,
"cpt_ar": 4.537455476649384,
"cpt_az": 3.8681871063030555,
"fertility_disparity": 0.09887010429849855,
"cpt_disparity": 0.1474986088107117,
"oov_rate": 0.0,
"vocab_gini": 0.7839455718558345,
"shannon_entropy": 11.593993601001019,
"exact_match_ar": 0.9990480507706255,
"exact_match_az": 0.9992363821483116
},
{
"name": "concat_bpe_32000",
"tokenizer_type": "concatenated",
"algorithm": "bpe",
"vocab_size": 32000,
"fertility_overall": 1.2695706222898995,
"fertility_ar": 1.3077466165859795,
"fertility_az": 1.1981161479838143,
"cpt_overall": 4.4229147884371125,
"cpt_ar": 4.388664556275568,
"cpt_az": 4.487021365858093,
"fertility_disparity": 0.08383158267185423,
"cpt_disparity": 0.021920290001497422,
"oov_rate": 0.0,
"vocab_gini": 0.7084770707714895,
"shannon_entropy": 12.560099600380696,
"exact_match_ar": 0.0,
"exact_match_az": 0.0
},
{
"name": "shared_unigram_32000",
"tokenizer_type": "shared",
"algorithm": "unigram",
"vocab_size": 32000,
"fertility_overall": 1.3340159750699838,
"fertility_ar": 1.3027084505800648,
"fertility_az": 1.3926146506382522,
"cpt_overall": 4.136639454094978,
"cpt_ar": 4.308287618942264,
"cpt_az": 3.8153634896854167,
"fertility_disparity": 0.06455928064305677,
"cpt_disparity": 0.11441300415729108,
"oov_rate": 0.0,
"vocab_gini": 0.8250969225299254,
"shannon_entropy": 10.965056609467814,
"exact_match_ar": 0.9990480507706255,
"exact_match_az": 0.9992363821483116
},
{
"name": "concat_unigram_32000",
"tokenizer_type": "concatenated",
"algorithm": "unigram",
"vocab_size": 32000,
"fertility_overall": 1.3498732692189985,
"fertility_ar": 1.392269196943319,
"fertility_az": 1.2705202940282205,
"cpt_overall": 4.117905456086874,
"cpt_ar": 4.071549633983907,
"cpt_az": 4.204670213900507,
"fertility_disparity": 0.08744638119007023,
"cpt_disparity": 0.03166017146279571,
"oov_rate": 0.0,
"vocab_gini": 0.8281405210808994,
"shannon_entropy": 11.002763195930038,
"exact_match_ar": 0.9989120580235721,
"exact_match_az": 0.9964364500254539
},
{
"name": "shared_wordpiece_32000",
"tokenizer_type": "shared",
"algorithm": "wordpiece",
"vocab_size": 32000,
"fertility_overall": 1.2742505256521945,
"fertility_ar": 1.2274699882356,
"fertility_az": 1.3618102283002589,
"cpt_overall": 4.319881028657028,
"cpt_ar": 4.553239284995183,
"cpt_az": 3.883101533084343,
"fertility_disparity": 0.0986482824646834,
"cpt_disparity": 0.1471782416793298,
"oov_rate": 0.0,
"vocab_gini": 0.7747440551637088,
"shannon_entropy": 11.589595542112136,
"exact_match_ar": 0.0,
"exact_match_az": 0.0
},
{
"name": "concat_wordpiece_32000",
"tokenizer_type": "concatenated",
"algorithm": "wordpiece",
"vocab_size": 32000,
"fertility_overall": 1.303008786024938,
"fertility_ar": 1.3416327230816638,
"fertility_az": 1.2307158917884393,
"cpt_overall": 4.240397119793184,
"cpt_ar": 4.201360994624003,
"cpt_az": 4.313461511548837,
"fertility_disparity": 0.08267302174804893,
"cpt_disparity": 0.025988528383688383,
"oov_rate": 0.0,
"vocab_gini": 0.7687350341879174,
"shannon_entropy": 11.65382526739417,
"exact_match_ar": 0.001042611060743427,
"exact_match_az": 0.00033938571186153065
},
{
"name": "shared_bbpe_32000",
"tokenizer_type": "shared",
"algorithm": "bbpe",
"vocab_size": 32000,
"fertility_overall": 1.4537480484685892,
"fertility_ar": 1.317581832517785,
"fertility_az": 1.7086120162163183,
"cpt_overall": 3.834237272964042,
"cpt_ar": 4.240919303648946,
"cpt_az": 3.0730455542376744,
"fertility_disparity": 0.22885838328847793,
"cpt_disparity": 0.2753822145133525,
"oov_rate": 0.0,
"vocab_gini": 0.8293407455122993,
"shannon_entropy": 10.685544129991571,
"exact_match_ar": 0.9994106980961015,
"exact_match_az": 0.9994909214322077
},
{
"name": "concat_bbpe_32000",
"tokenizer_type": "concatenated",
"algorithm": "bbpe",
"vocab_size": 32000,
"fertility_overall": 1.306778190060823,
"fertility_ar": 1.3463628334026025,
"fertility_az": 1.2326871301490916,
"cpt_overall": 4.231787559188005,
"cpt_ar": 4.190301018162676,
"cpt_az": 4.3094384241989285,
"fertility_disparity": 0.08443170030639024,
"cpt_disparity": 0.02764569169088408,
"oov_rate": 0.0,
"vocab_gini": 0.7818011980546499,
"shannon_entropy": 11.6730644846771,
"exact_match_ar": 0.9989120580235721,
"exact_match_az": 0.9964364500254539
}
] |