Mark </assistant> (token 24) as special in tokenizer.json
Browse files- tokenizer.json +1 -1
tokenizer.json
CHANGED
|
@@ -226,7 +226,7 @@
|
|
| 226 |
"lstrip": false,
|
| 227 |
"rstrip": false,
|
| 228 |
"normalized": false,
|
| 229 |
-
"special":
|
| 230 |
},
|
| 231 |
{
|
| 232 |
"id": 25,
|
|
|
|
| 226 |
"lstrip": false,
|
| 227 |
"rstrip": false,
|
| 228 |
"normalized": false,
|
| 229 |
+
"special": true
|
| 230 |
},
|
| 231 |
{
|
| 232 |
"id": 25,
|