airev-ae commited on
Commit
8f7e644
·
verified ·
1 Parent(s): 31b2e2b

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -1
tokenizer_config.json CHANGED
@@ -9,6 +9,7 @@
9
  "eos_token": "<|im_end|>",
10
  "errors": "replace",
11
  "image_token": "<|image_pad|>",
 
12
  "model_max_length": 262144,
13
  "model_specific_special_tokens": {
14
  "audio_bos_token": "<|audio_start|>",
@@ -28,4 +29,4 @@
28
  "video_token": "<|video_pad|>",
29
  "vision_bos_token": "<|vision_start|>",
30
  "vision_eos_token": "<|vision_end|>"
31
- }
 
9
  "eos_token": "<|im_end|>",
10
  "errors": "replace",
11
  "image_token": "<|image_pad|>",
12
+ "is_local": true,
13
  "model_max_length": 262144,
14
  "model_specific_special_tokens": {
15
  "audio_bos_token": "<|audio_start|>",
 
29
  "video_token": "<|video_pad|>",
30
  "vision_bos_token": "<|vision_start|>",
31
  "vision_eos_token": "<|vision_end|>"
32
+ }