Commit ·
a15090a
1
Parent(s): 528cdc5
Add eos_token to tokenizer implicitly to help downstream vllm integrate (#43)
Browse files- add eos token implicitly to help downstream vllm integrate this model (b51b7dc0fbf7f266a97880fb4b626c56d28f4b96)
Co-authored-by: dongw <dongwng@users.noreply.huggingface.co>
- generation_config.json +1 -0
- tokenizer_config.json +1 -0
generation_config.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
{
|
| 2 |
"_from_model_config": true,
|
| 3 |
"decoder_start_token_id": 8192,
|
|
|
|
| 4 |
"output_attentions": false,
|
| 5 |
"output_hidden_states": false,
|
| 6 |
"pad_token_id": 2,
|
|
|
|
| 1 |
{
|
| 2 |
"_from_model_config": true,
|
| 3 |
"decoder_start_token_id": 8192,
|
| 4 |
+
"eos_token_id": 3,
|
| 5 |
"output_attentions": false,
|
| 6 |
"output_hidden_states": false,
|
| 7 |
"pad_token_id": 2,
|
tokenizer_config.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
{
|
| 2 |
"backend": "tokenizers",
|
| 3 |
"clean_up_tokenization_spaces": false,
|
|
|
|
| 4 |
"model_max_length": 1000000000000000019884624838656,
|
| 5 |
"pad_token": "<pad>",
|
| 6 |
"processor_class": "ParakeetProcessor",
|
|
|
|
| 1 |
{
|
| 2 |
"backend": "tokenizers",
|
| 3 |
"clean_up_tokenization_spaces": false,
|
| 4 |
+
"eos_token": "<|endoftext|>",
|
| 5 |
"model_max_length": 1000000000000000019884624838656,
|
| 6 |
"pad_token": "<pad>",
|
| 7 |
"processor_class": "ParakeetProcessor",
|