Replace stray [THINK]/[/THINK] special tokens with reserved <SPECIAL_34>/<SPECIAL_35> slots
Browse filesThis is a non-reasoning model. Renaming the unused [THINK]/[/THINK] special tokens to reserved <SPECIAL_34>/<SPECIAL_35> slots prevents tooling (e.g. mistral-common) from treating it as reasoning-capable. Only tokenizer files are changed.
- special_tokens_map.json +2 -2
- tekken.json +2 -2
- tokenizer.json +2 -2
- tokenizer_config.json +4 -4
special_tokens_map.json
CHANGED
|
@@ -239,14 +239,14 @@
|
|
| 239 |
"single_word": false
|
| 240 |
},
|
| 241 |
{
|
| 242 |
-
"content": "
|
| 243 |
"lstrip": false,
|
| 244 |
"normalized": false,
|
| 245 |
"rstrip": false,
|
| 246 |
"single_word": false
|
| 247 |
},
|
| 248 |
{
|
| 249 |
-
"content": "
|
| 250 |
"lstrip": false,
|
| 251 |
"normalized": false,
|
| 252 |
"rstrip": false,
|
|
|
|
| 239 |
"single_word": false
|
| 240 |
},
|
| 241 |
{
|
| 242 |
+
"content": "<SPECIAL_34>",
|
| 243 |
"lstrip": false,
|
| 244 |
"normalized": false,
|
| 245 |
"rstrip": false,
|
| 246 |
"single_word": false
|
| 247 |
},
|
| 248 |
{
|
| 249 |
+
"content": "<SPECIAL_35>",
|
| 250 |
"lstrip": false,
|
| 251 |
"normalized": false,
|
| 252 |
"rstrip": false,
|
tekken.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843
|
| 3 |
+
size 16753784
|
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135
|
| 3 |
+
size 17078128
|
tokenizer_config.json
CHANGED
|
@@ -276,7 +276,7 @@
|
|
| 276 |
"special": true
|
| 277 |
},
|
| 278 |
"34": {
|
| 279 |
-
"content": "
|
| 280 |
"lstrip": false,
|
| 281 |
"normalized": false,
|
| 282 |
"rstrip": false,
|
|
@@ -284,7 +284,7 @@
|
|
| 284 |
"special": true
|
| 285 |
},
|
| 286 |
"35": {
|
| 287 |
-
"content": "
|
| 288 |
"lstrip": false,
|
| 289 |
"normalized": false,
|
| 290 |
"rstrip": false,
|
|
@@ -8039,8 +8039,8 @@
|
|
| 8039 |
"<SPECIAL_31>",
|
| 8040 |
"[ARGS]",
|
| 8041 |
"[CALL_ID]",
|
| 8042 |
-
"
|
| 8043 |
-
"
|
| 8044 |
"<SPECIAL_36>",
|
| 8045 |
"<SPECIAL_37>",
|
| 8046 |
"<SPECIAL_38>",
|
|
|
|
| 276 |
"special": true
|
| 277 |
},
|
| 278 |
"34": {
|
| 279 |
+
"content": "<SPECIAL_34>",
|
| 280 |
"lstrip": false,
|
| 281 |
"normalized": false,
|
| 282 |
"rstrip": false,
|
|
|
|
| 284 |
"special": true
|
| 285 |
},
|
| 286 |
"35": {
|
| 287 |
+
"content": "<SPECIAL_35>",
|
| 288 |
"lstrip": false,
|
| 289 |
"normalized": false,
|
| 290 |
"rstrip": false,
|
|
|
|
| 8039 |
"<SPECIAL_31>",
|
| 8040 |
"[ARGS]",
|
| 8041 |
"[CALL_ID]",
|
| 8042 |
+
"<SPECIAL_34>",
|
| 8043 |
+
"<SPECIAL_35>",
|
| 8044 |
"<SPECIAL_36>",
|
| 8045 |
"<SPECIAL_37>",
|
| 8046 |
"<SPECIAL_38>",
|