samschapiro commited on
Commit
7c63560
·
verified ·
1 Parent(s): 9d9213b

Add chat_template to tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -2
tokenizer_config.json CHANGED
@@ -15,5 +15,6 @@
15
  "model_max_length": 8192,
16
  "pad_token": "<|endoftext|>",
17
  "tokenizer_class": "Phi3SmallTokenizer",
18
- "trust_remote_code": true
19
- }
 
 
15
  "model_max_length": 8192,
16
  "pad_token": "<|endoftext|>",
17
  "tokenizer_class": "Phi3SmallTokenizer",
18
+ "trust_remote_code": true,
19
+ "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}"
20
+ }