Complete standard tokenizer and generation metadata

config.json

@@ -15,6 +15,9 @@
   "tie_word_embeddings": true,
   "attention_bias": false,
   "torch_dtype": "bfloat16",
+  "eos_token_id": 1,
+  "pad_token_id": 1,
+  "unk_token_id": 0,
   "use_cache": false,
   "auto_map": {
     "AutoConfig": "configuration_vortex.VortexConfig",

special_tokens_map.json

@@ -1,4 +1,5 @@
 {
   "unk_token": "<unk>",
-  "eos_token": "</s>"
+  "eos_token": "</s>",
+  "pad_token": "</s>"
 }

tokenizer_config.json

@@ -7,6 +7,8 @@
   "vocab_size": 8192,
   "unk_token": "<unk>",
   "eos_token": "</s>",
+  "pad_token": "</s>",
+  "model_max_length": 4096,
   "add_bos_token": false,
   "add_eos_token": false
 }