Hugging Face's logo

fakezeta
/
Hermes-2-Pro-Llama-3-8B-ov-int8

Text Generation
Transformers
OpenVINO
English
llama
Llama-3
instruct
finetune
chatml
DPO
RLHF
gpt4
synthetic data
distillation
function calling
json mode
axolotl
conversational
Model card Files
xet
 Community
fakezeta commited on
Commit
fddd869
·
verified ·
1 Parent(s): 69c51f2

Upload 2 files

Browse files
Files changed (2) hide show
  1. tokenizer.json +25 -25
  2. tokenizer_config.json +68 -68
tokenizer.json CHANGED
@@ -46,7 +46,7 @@
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
- "special": true
50
  },
51
  {
52
  "id": 128005,
@@ -55,7 +55,7 @@
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
- "special": true
59
  },
60
  {
61
  "id": 128006,
@@ -82,7 +82,7 @@
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
- "special": true
86
  },
87
  {
88
  "id": 128009,
@@ -100,7 +100,7 @@
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
- "special": true
104
  },
105
  {
106
  "id": 128011,
@@ -109,7 +109,7 @@
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
- "special": true
113
  },
114
  {
115
  "id": 128012,
@@ -118,7 +118,7 @@
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
- "special": true
122
  },
123
  {
124
  "id": 128013,
@@ -2426,7 +2426,7 @@
2426
  },
2427
  {
2428
  "id": 128269,
2429
- "content": "<|reserved_special_token_263|>",
2430
  "single_word": false,
2431
  "lstrip": false,
2432
  "rstrip": false,
@@ -2435,7 +2435,7 @@
2435
  },
2436
  {
2437
  "id": 128270,
2438
- "content": "<|reserved_special_token_264|>",
2439
  "single_word": false,
2440
  "lstrip": false,
2441
  "rstrip": false,
@@ -2444,7 +2444,7 @@
2444
  },
2445
  {
2446
  "id": 128271,
2447
- "content": "<|reserved_special_token_265|>",
2448
  "single_word": false,
2449
  "lstrip": false,
2450
  "rstrip": false,
@@ -2453,7 +2453,7 @@
2453
  },
2454
  {
2455
  "id": 128272,
2456
- "content": "<|reserved_special_token_266|>",
2457
  "single_word": false,
2458
  "lstrip": false,
2459
  "rstrip": false,
@@ -2462,7 +2462,7 @@
2462
  },
2463
  {
2464
  "id": 128273,
2465
- "content": "<|reserved_special_token_267|>",
2466
  "single_word": false,
2467
  "lstrip": false,
2468
  "rstrip": false,
@@ -2471,7 +2471,7 @@
2471
  },
2472
  {
2473
  "id": 128274,
2474
- "content": "<|reserved_special_token_268|>",
2475
  "single_word": false,
2476
  "lstrip": false,
2477
  "rstrip": false,
@@ -2480,7 +2480,7 @@
2480
  },
2481
  {
2482
  "id": 128275,
2483
- "content": "<|reserved_special_token_269|>",
2484
  "single_word": false,
2485
  "lstrip": false,
2486
  "rstrip": false,
@@ -2489,7 +2489,7 @@
2489
  },
2490
  {
2491
  "id": 128276,
2492
- "content": "<|reserved_special_token_270|>",
2493
  "single_word": false,
2494
  "lstrip": false,
2495
  "rstrip": false,
@@ -2498,7 +2498,7 @@
2498
  },
2499
  {
2500
  "id": 128277,
2501
- "content": "<|reserved_special_token_271|>",
2502
  "single_word": false,
2503
  "lstrip": false,
2504
  "rstrip": false,
@@ -2507,7 +2507,7 @@
2507
  },
2508
  {
2509
  "id": 128278,
2510
- "content": "<|reserved_special_token_272|>",
2511
  "single_word": false,
2512
  "lstrip": false,
2513
  "rstrip": false,
@@ -2516,7 +2516,7 @@
2516
  },
2517
  {
2518
  "id": 128279,
2519
- "content": "<|reserved_special_token_273|>",
2520
  "single_word": false,
2521
  "lstrip": false,
2522
  "rstrip": false,
@@ -2525,7 +2525,7 @@
2525
  },
2526
  {
2527
  "id": 128280,
2528
- "content": "<|reserved_special_token_274|>",
2529
  "single_word": false,
2530
  "lstrip": false,
2531
  "rstrip": false,
@@ -2534,7 +2534,7 @@
2534
  },
2535
  {
2536
  "id": 128281,
2537
- "content": "<|reserved_special_token_275|>",
2538
  "single_word": false,
2539
  "lstrip": false,
2540
  "rstrip": false,
@@ -2543,7 +2543,7 @@
2543
  },
2544
  {
2545
  "id": 128282,
2546
- "content": "<|reserved_special_token_276|>",
2547
  "single_word": false,
2548
  "lstrip": false,
2549
  "rstrip": false,
@@ -2552,7 +2552,7 @@
2552
  },
2553
  {
2554
  "id": 128283,
2555
- "content": "<|reserved_special_token_277|>",
2556
  "single_word": false,
2557
  "lstrip": false,
2558
  "rstrip": false,
@@ -2561,7 +2561,7 @@
2561
  },
2562
  {
2563
  "id": 128284,
2564
- "content": "<|reserved_special_token_278|>",
2565
  "single_word": false,
2566
  "lstrip": false,
2567
  "rstrip": false,
@@ -2570,7 +2570,7 @@
2570
  },
2571
  {
2572
  "id": 128285,
2573
- "content": "<|reserved_special_token_279|>",
2574
  "single_word": false,
2575
  "lstrip": false,
2576
  "rstrip": false,
@@ -2579,7 +2579,7 @@
2579
  },
2580
  {
2581
  "id": 128286,
2582
- "content": "<|reserved_special_token_280|>",
2583
  "single_word": false,
2584
  "lstrip": false,
2585
  "rstrip": false,
@@ -2588,7 +2588,7 @@
2588
  },
2589
  {
2590
  "id": 128287,
2591
- "content": "<|reserved_special_token_281|>",
2592
  "single_word": false,
2593
  "lstrip": false,
2594
  "rstrip": false,
 
46
  "lstrip": false,
47
  "rstrip": false,
48
  "normalized": false,
49
+ "special": false
50
  },
51
  {
52
  "id": 128005,
 
55
  "lstrip": false,
56
  "rstrip": false,
57
  "normalized": false,
58
+ "special": false
59
  },
60
  {
61
  "id": 128006,
 
82
  "lstrip": false,
83
  "rstrip": false,
84
  "normalized": false,
85
+ "special": false
86
  },
87
  {
88
  "id": 128009,
 
100
  "lstrip": false,
101
  "rstrip": false,
102
  "normalized": false,
103
+ "special": false
104
  },
105
  {
106
  "id": 128011,
 
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
+ "special": false
113
  },
114
  {
115
  "id": 128012,
 
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
+ "special": false
122
  },
123
  {
124
  "id": 128013,
 
2426
  },
2427
  {
2428
  "id": 128269,
2429
+ "content": "<|reserved_special_token_262|>",
2430
  "single_word": false,
2431
  "lstrip": false,
2432
  "rstrip": false,
 
2435
  },
2436
  {
2437
  "id": 128270,
2438
+ "content": "<|reserved_special_token_263|>",
2439
  "single_word": false,
2440
  "lstrip": false,
2441
  "rstrip": false,
 
2444
  },
2445
  {
2446
  "id": 128271,
2447
+ "content": "<|reserved_special_token_264|>",
2448
  "single_word": false,
2449
  "lstrip": false,
2450
  "rstrip": false,
 
2453
  },
2454
  {
2455
  "id": 128272,
2456
+ "content": "<|reserved_special_token_265|>",
2457
  "single_word": false,
2458
  "lstrip": false,
2459
  "rstrip": false,
 
2462
  },
2463
  {
2464
  "id": 128273,
2465
+ "content": "<|reserved_special_token_266|>",
2466
  "single_word": false,
2467
  "lstrip": false,
2468
  "rstrip": false,
 
2471
  },
2472
  {
2473
  "id": 128274,
2474
+ "content": "<|reserved_special_token_267|>",
2475
  "single_word": false,
2476
  "lstrip": false,
2477
  "rstrip": false,
 
2480
  },
2481
  {
2482
  "id": 128275,
2483
+ "content": "<|reserved_special_token_268|>",
2484
  "single_word": false,
2485
  "lstrip": false,
2486
  "rstrip": false,
 
2489
  },
2490
  {
2491
  "id": 128276,
2492
+ "content": "<|reserved_special_token_269|>",
2493
  "single_word": false,
2494
  "lstrip": false,
2495
  "rstrip": false,
 
2498
  },
2499
  {
2500
  "id": 128277,
2501
+ "content": "<|reserved_special_token_270|>",
2502
  "single_word": false,
2503
  "lstrip": false,
2504
  "rstrip": false,
 
2507
  },
2508
  {
2509
  "id": 128278,
2510
+ "content": "<|reserved_special_token_271|>",
2511
  "single_word": false,
2512
  "lstrip": false,
2513
  "rstrip": false,
 
2516
  },
2517
  {
2518
  "id": 128279,
2519
+ "content": "<|reserved_special_token_272|>",
2520
  "single_word": false,
2521
  "lstrip": false,
2522
  "rstrip": false,
 
2525
  },
2526
  {
2527
  "id": 128280,
2528
+ "content": "<|reserved_special_token_273|>",
2529
  "single_word": false,
2530
  "lstrip": false,
2531
  "rstrip": false,
 
2534
  },
2535
  {
2536
  "id": 128281,
2537
+ "content": "<|reserved_special_token_274|>",
2538
  "single_word": false,
2539
  "lstrip": false,
2540
  "rstrip": false,
 
2543
  },
2544
  {
2545
  "id": 128282,
2546
+ "content": "<|reserved_special_token_275|>",
2547
  "single_word": false,
2548
  "lstrip": false,
2549
  "rstrip": false,
 
2552
  },
2553
  {
2554
  "id": 128283,
2555
+ "content": "<|reserved_special_token_276|>",
2556
  "single_word": false,
2557
  "lstrip": false,
2558
  "rstrip": false,
 
2561
  },
2562
  {
2563
  "id": 128284,
2564
+ "content": "<|reserved_special_token_277|>",
2565
  "single_word": false,
2566
  "lstrip": false,
2567
  "rstrip": false,
 
2570
  },
2571
  {
2572
  "id": 128285,
2573
+ "content": "<|reserved_special_token_278|>",
2574
  "single_word": false,
2575
  "lstrip": false,
2576
  "rstrip": false,
 
2579
  },
2580
  {
2581
  "id": 128286,
2582
+ "content": "<|reserved_special_token_279|>",
2583
  "single_word": false,
2584
  "lstrip": false,
2585
  "rstrip": false,
 
2588
  },
2589
  {
2590
  "id": 128287,
2591
+ "content": "<|reserved_special_token_280|>",
2592
  "single_word": false,
2593
  "lstrip": false,
2594
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -38,7 +38,7 @@
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
- "special": true
42
  },
43
  "128005": {
44
  "content": "<tool_response>",
@@ -46,7 +46,7 @@
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
- "special": true
50
  },
51
  "128006": {
52
  "content": "<|start_header_id|>",
@@ -70,7 +70,7 @@
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
- "special": true
74
  },
75
  "128009": {
76
  "content": "<|eot_id>",
@@ -86,7 +86,7 @@
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
- "special": true
90
  },
91
  "128011": {
92
  "content": "</tool_call>",
@@ -94,7 +94,7 @@
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
- "special": true
98
  },
99
  "128012": {
100
  "content": "</tool_response>",
@@ -102,7 +102,7 @@
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
- "special": true
106
  },
107
  "128013": {
108
  "content": "<|reserved_special_token_8|>",
@@ -2058,250 +2058,250 @@
2058
  },
2059
  "128257": {
2060
  "content": "<|reserved_special_token_251|>",
 
2061
  "lstrip": false,
2062
- "normalized": false,
2063
  "rstrip": false,
2064
- "single_word": false,
2065
  "special": true
2066
  },
2067
  "128258": {
2068
  "content": "<|reserved_special_token_252|>",
 
2069
  "lstrip": false,
2070
- "normalized": false,
2071
  "rstrip": false,
2072
- "single_word": false,
2073
  "special": true
2074
  },
2075
  "128259": {
2076
  "content": "<|reserved_special_token_253|>",
 
2077
  "lstrip": false,
2078
- "normalized": false,
2079
  "rstrip": false,
2080
- "single_word": false,
2081
  "special": true
2082
  },
2083
  "128260": {
2084
  "content": "<|reserved_special_token_254|>",
 
2085
  "lstrip": false,
2086
- "normalized": false,
2087
  "rstrip": false,
2088
- "single_word": false,
2089
  "special": true
2090
  },
2091
  "128261": {
2092
  "content": "<|reserved_special_token_255|>",
 
2093
  "lstrip": false,
2094
- "normalized": false,
2095
  "rstrip": false,
2096
- "single_word": false,
2097
  "special": true
2098
  },
2099
  "128262": {
2100
  "content": "<|reserved_special_token_256|>",
 
2101
  "lstrip": false,
2102
- "normalized": false,
2103
  "rstrip": false,
2104
- "single_word": false,
2105
  "special": true
2106
  },
2107
  "128263": {
2108
  "content": "<|reserved_special_token_257|>",
 
2109
  "lstrip": false,
2110
- "normalized": false,
2111
  "rstrip": false,
2112
- "single_word": false,
2113
  "special": true
2114
  },
2115
  "128264": {
2116
  "content": "<|reserved_special_token_258|>",
 
2117
  "lstrip": false,
2118
- "normalized": false,
2119
  "rstrip": false,
2120
- "single_word": false,
2121
  "special": true
2122
  },
2123
  "128265": {
2124
  "content": "<|reserved_special_token_259|>",
 
2125
  "lstrip": false,
2126
- "normalized": false,
2127
  "rstrip": false,
2128
- "single_word": false,
2129
  "special": true
2130
  },
2131
  "128266": {
2132
  "content": "<|reserved_special_token_260|>",
 
2133
  "lstrip": false,
2134
- "normalized": false,
2135
  "rstrip": false,
2136
- "single_word": false,
2137
  "special": true
2138
  },
2139
  "128267": {
2140
  "content": "<|reserved_special_token_261|>",
 
2141
  "lstrip": false,
2142
- "normalized": false,
2143
  "rstrip": false,
2144
- "single_word": false,
2145
  "special": true
2146
  },
2147
  "128268": {
2148
  "content": "<|reserved_special_token_262|>",
 
2149
  "lstrip": false,
2150
- "normalized": false,
2151
  "rstrip": false,
2152
- "single_word": false,
2153
  "special": true
2154
  },
2155
  "128269": {
2156
  "content": "<|reserved_special_token_263|>",
 
2157
  "lstrip": false,
2158
- "normalized": false,
2159
  "rstrip": false,
2160
- "single_word": false,
2161
  "special": true
2162
  },
2163
  "128270": {
2164
  "content": "<|reserved_special_token_264|>",
 
2165
  "lstrip": false,
2166
- "normalized": false,
2167
  "rstrip": false,
2168
- "single_word": false,
2169
  "special": true
2170
  },
2171
  "128271": {
2172
  "content": "<|reserved_special_token_265|>",
 
2173
  "lstrip": false,
2174
- "normalized": false,
2175
  "rstrip": false,
2176
- "single_word": false,
2177
  "special": true
2178
  },
2179
  "128272": {
2180
  "content": "<|reserved_special_token_266|>",
 
2181
  "lstrip": false,
2182
- "normalized": false,
2183
  "rstrip": false,
2184
- "single_word": false,
2185
  "special": true
2186
  },
2187
  "128273": {
2188
  "content": "<|reserved_special_token_267|>",
 
2189
  "lstrip": false,
2190
- "normalized": false,
2191
  "rstrip": false,
2192
- "single_word": false,
2193
  "special": true
2194
  },
2195
  "128274": {
2196
  "content": "<|reserved_special_token_268|>",
 
2197
  "lstrip": false,
2198
- "normalized": false,
2199
  "rstrip": false,
2200
- "single_word": false,
2201
  "special": true
2202
  },
2203
  "128275": {
2204
  "content": "<|reserved_special_token_269|>",
 
2205
  "lstrip": false,
2206
- "normalized": false,
2207
  "rstrip": false,
2208
- "single_word": false,
2209
  "special": true
2210
  },
2211
  "128276": {
2212
  "content": "<|reserved_special_token_270|>",
 
2213
  "lstrip": false,
2214
- "normalized": false,
2215
  "rstrip": false,
2216
- "single_word": false,
2217
  "special": true
2218
  },
2219
  "128277": {
2220
  "content": "<|reserved_special_token_271|>",
 
2221
  "lstrip": false,
2222
- "normalized": false,
2223
  "rstrip": false,
2224
- "single_word": false,
2225
  "special": true
2226
  },
2227
  "128278": {
2228
  "content": "<|reserved_special_token_272|>",
 
2229
  "lstrip": false,
2230
- "normalized": false,
2231
  "rstrip": false,
2232
- "single_word": false,
2233
  "special": true
2234
  },
2235
  "128279": {
2236
  "content": "<|reserved_special_token_273|>",
 
2237
  "lstrip": false,
2238
- "normalized": false,
2239
  "rstrip": false,
2240
- "single_word": false,
2241
  "special": true
2242
  },
2243
  "128280": {
2244
  "content": "<|reserved_special_token_274|>",
 
2245
  "lstrip": false,
2246
- "normalized": false,
2247
  "rstrip": false,
2248
- "single_word": false,
2249
  "special": true
2250
  },
2251
  "128281": {
2252
  "content": "<|reserved_special_token_275|>",
 
2253
  "lstrip": false,
2254
- "normalized": false,
2255
  "rstrip": false,
2256
- "single_word": false,
2257
  "special": true
2258
  },
2259
  "128282": {
2260
  "content": "<|reserved_special_token_276|>",
 
2261
  "lstrip": false,
2262
- "normalized": false,
2263
  "rstrip": false,
2264
- "single_word": false,
2265
  "special": true
2266
  },
2267
  "128283": {
2268
  "content": "<|reserved_special_token_277|>",
 
2269
  "lstrip": false,
2270
- "normalized": false,
2271
  "rstrip": false,
2272
- "single_word": false,
2273
  "special": true
2274
  },
2275
  "128284": {
2276
  "content": "<|reserved_special_token_278|>",
 
2277
  "lstrip": false,
2278
- "normalized": false,
2279
  "rstrip": false,
2280
- "single_word": false,
2281
  "special": true
2282
  },
2283
  "128285": {
2284
  "content": "<|reserved_special_token_279|>",
 
2285
  "lstrip": false,
2286
- "normalized": false,
2287
  "rstrip": false,
2288
- "single_word": false,
2289
  "special": true
2290
  },
2291
  "128286": {
2292
  "content": "<|reserved_special_token_280|>",
 
2293
  "lstrip": false,
2294
- "normalized": false,
2295
  "rstrip": false,
2296
- "single_word": false,
2297
  "special": true
2298
  },
2299
  "128287": {
2300
  "content": "<|reserved_special_token_281|>",
 
2301
  "lstrip": false,
2302
- "normalized": false,
2303
  "rstrip": false,
2304
- "single_word": false,
2305
  "special": true
2306
  }
2307
  },
 
38
  "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
+ "special": false
42
  },
43
  "128005": {
44
  "content": "<tool_response>",
 
46
  "normalized": false,
47
  "rstrip": false,
48
  "single_word": false,
49
+ "special": false
50
  },
51
  "128006": {
52
  "content": "<|start_header_id|>",
 
70
  "normalized": false,
71
  "rstrip": false,
72
  "single_word": false,
73
+ "special": false
74
  },
75
  "128009": {
76
  "content": "<|eot_id>",
 
86
  "normalized": false,
87
  "rstrip": false,
88
  "single_word": false,
89
+ "special": false
90
  },
91
  "128011": {
92
  "content": "</tool_call>",
 
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
+ "special": false
98
  },
99
  "128012": {
100
  "content": "</tool_response>",
 
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
+ "special": false
106
  },
107
  "128013": {
108
  "content": "<|reserved_special_token_8|>",
 
2058
  },
2059
  "128257": {
2060
  "content": "<|reserved_special_token_251|>",
2061
+ "single_word": false,
2062
  "lstrip": false,
 
2063
  "rstrip": false,
2064
+ "normalized": false,
2065
  "special": true
2066
  },
2067
  "128258": {
2068
  "content": "<|reserved_special_token_252|>",
2069
+ "single_word": false,
2070
  "lstrip": false,
 
2071
  "rstrip": false,
2072
+ "normalized": false,
2073
  "special": true
2074
  },
2075
  "128259": {
2076
  "content": "<|reserved_special_token_253|>",
2077
+ "single_word": false,
2078
  "lstrip": false,
 
2079
  "rstrip": false,
2080
+ "normalized": false,
2081
  "special": true
2082
  },
2083
  "128260": {
2084
  "content": "<|reserved_special_token_254|>",
2085
+ "single_word": false,
2086
  "lstrip": false,
 
2087
  "rstrip": false,
2088
+ "normalized": false,
2089
  "special": true
2090
  },
2091
  "128261": {
2092
  "content": "<|reserved_special_token_255|>",
2093
+ "single_word": false,
2094
  "lstrip": false,
 
2095
  "rstrip": false,
2096
+ "normalized": false,
2097
  "special": true
2098
  },
2099
  "128262": {
2100
  "content": "<|reserved_special_token_256|>",
2101
+ "single_word": false,
2102
  "lstrip": false,
 
2103
  "rstrip": false,
2104
+ "normalized": false,
2105
  "special": true
2106
  },
2107
  "128263": {
2108
  "content": "<|reserved_special_token_257|>",
2109
+ "single_word": false,
2110
  "lstrip": false,
 
2111
  "rstrip": false,
2112
+ "normalized": false,
2113
  "special": true
2114
  },
2115
  "128264": {
2116
  "content": "<|reserved_special_token_258|>",
2117
+ "single_word": false,
2118
  "lstrip": false,
 
2119
  "rstrip": false,
2120
+ "normalized": false,
2121
  "special": true
2122
  },
2123
  "128265": {
2124
  "content": "<|reserved_special_token_259|>",
2125
+ "single_word": false,
2126
  "lstrip": false,
 
2127
  "rstrip": false,
2128
+ "normalized": false,
2129
  "special": true
2130
  },
2131
  "128266": {
2132
  "content": "<|reserved_special_token_260|>",
2133
+ "single_word": false,
2134
  "lstrip": false,
 
2135
  "rstrip": false,
2136
+ "normalized": false,
2137
  "special": true
2138
  },
2139
  "128267": {
2140
  "content": "<|reserved_special_token_261|>",
2141
+ "single_word": false,
2142
  "lstrip": false,
 
2143
  "rstrip": false,
2144
+ "normalized": false,
2145
  "special": true
2146
  },
2147
  "128268": {
2148
  "content": "<|reserved_special_token_262|>",
2149
+ "single_word": false,
2150
  "lstrip": false,
 
2151
  "rstrip": false,
2152
+ "normalized": false,
2153
  "special": true
2154
  },
2155
  "128269": {
2156
  "content": "<|reserved_special_token_263|>",
2157
+ "single_word": false,
2158
  "lstrip": false,
 
2159
  "rstrip": false,
2160
+ "normalized": false,
2161
  "special": true
2162
  },
2163
  "128270": {
2164
  "content": "<|reserved_special_token_264|>",
2165
+ "single_word": false,
2166
  "lstrip": false,
 
2167
  "rstrip": false,
2168
+ "normalized": false,
2169
  "special": true
2170
  },
2171
  "128271": {
2172
  "content": "<|reserved_special_token_265|>",
2173
+ "single_word": false,
2174
  "lstrip": false,
 
2175
  "rstrip": false,
2176
+ "normalized": false,
2177
  "special": true
2178
  },
2179
  "128272": {
2180
  "content": "<|reserved_special_token_266|>",
2181
+ "single_word": false,
2182
  "lstrip": false,
 
2183
  "rstrip": false,
2184
+ "normalized": false,
2185
  "special": true
2186
  },
2187
  "128273": {
2188
  "content": "<|reserved_special_token_267|>",
2189
+ "single_word": false,
2190
  "lstrip": false,
 
2191
  "rstrip": false,
2192
+ "normalized": false,
2193
  "special": true
2194
  },
2195
  "128274": {
2196
  "content": "<|reserved_special_token_268|>",
2197
+ "single_word": false,
2198
  "lstrip": false,
 
2199
  "rstrip": false,
2200
+ "normalized": false,
2201
  "special": true
2202
  },
2203
  "128275": {
2204
  "content": "<|reserved_special_token_269|>",
2205
+ "single_word": false,
2206
  "lstrip": false,
 
2207
  "rstrip": false,
2208
+ "normalized": false,
2209
  "special": true
2210
  },
2211
  "128276": {
2212
  "content": "<|reserved_special_token_270|>",
2213
+ "single_word": false,
2214
  "lstrip": false,
 
2215
  "rstrip": false,
2216
+ "normalized": false,
2217
  "special": true
2218
  },
2219
  "128277": {
2220
  "content": "<|reserved_special_token_271|>",
2221
+ "single_word": false,
2222
  "lstrip": false,
 
2223
  "rstrip": false,
2224
+ "normalized": false,
2225
  "special": true
2226
  },
2227
  "128278": {
2228
  "content": "<|reserved_special_token_272|>",
2229
+ "single_word": false,
2230
  "lstrip": false,
 
2231
  "rstrip": false,
2232
+ "normalized": false,
2233
  "special": true
2234
  },
2235
  "128279": {
2236
  "content": "<|reserved_special_token_273|>",
2237
+ "single_word": false,
2238
  "lstrip": false,
 
2239
  "rstrip": false,
2240
+ "normalized": false,
2241
  "special": true
2242
  },
2243
  "128280": {
2244
  "content": "<|reserved_special_token_274|>",
2245
+ "single_word": false,
2246
  "lstrip": false,
 
2247
  "rstrip": false,
2248
+ "normalized": false,
2249
  "special": true
2250
  },
2251
  "128281": {
2252
  "content": "<|reserved_special_token_275|>",
2253
+ "single_word": false,
2254
  "lstrip": false,
 
2255
  "rstrip": false,
2256
+ "normalized": false,
2257
  "special": true
2258
  },
2259
  "128282": {
2260
  "content": "<|reserved_special_token_276|>",
2261
+ "single_word": false,
2262
  "lstrip": false,
 
2263
  "rstrip": false,
2264
+ "normalized": false,
2265
  "special": true
2266
  },
2267
  "128283": {
2268
  "content": "<|reserved_special_token_277|>",
2269
+ "single_word": false,
2270
  "lstrip": false,
 
2271
  "rstrip": false,
2272
+ "normalized": false,
2273
  "special": true
2274
  },
2275
  "128284": {
2276
  "content": "<|reserved_special_token_278|>",
2277
+ "single_word": false,
2278
  "lstrip": false,
 
2279
  "rstrip": false,
2280
+ "normalized": false,
2281
  "special": true
2282
  },
2283
  "128285": {
2284
  "content": "<|reserved_special_token_279|>",
2285
+ "single_word": false,
2286
  "lstrip": false,
 
2287
  "rstrip": false,
2288
+ "normalized": false,
2289
  "special": true
2290
  },
2291
  "128286": {
2292
  "content": "<|reserved_special_token_280|>",
2293
+ "single_word": false,
2294
  "lstrip": false,
 
2295
  "rstrip": false,
2296
+ "normalized": false,
2297
  "special": true
2298
  },
2299
  "128287": {
2300
  "content": "<|reserved_special_token_281|>",
2301
+ "single_word": false,
2302
  "lstrip": false,
 
2303
  "rstrip": false,
2304
+ "normalized": false,
2305
  "special": true
2306
  }
2307
  },