Use Japanese tokenizer with proper config
Browse files- tokenizer.json +1 -1
tokenizer.json
CHANGED
|
@@ -38401,7 +38401,7 @@
|
|
| 38401 |
"अ": 31350,
|
| 38402 |
"╔": 31351,
|
| 38403 |
"无": 31352,
|
| 38404 |
-
"
|
| 38405 |
"은": 31354,
|
| 38406 |
"ʷ": 31355,
|
| 38407 |
"那": 31356,
|
|
|
|
| 38401 |
"अ": 31350,
|
| 38402 |
"╔": 31351,
|
| 38403 |
"无": 31352,
|
| 38404 |
+
"
|
| 38405 |
"은": 31354,
|
| 38406 |
"ʷ": 31355,
|
| 38407 |
"那": 31356,
|