{ "add_bos_token": false, "add_eos_token": false, "added_tokens_decoder": { "0": { "content": "<|start▁of▁sentence|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<|end▁of▁sentence|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<|▁pad▁|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "4": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "5": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "<|start▁of▁text|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "7": { "content": "<|end▁of▁text|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "8": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "9": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "10": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "11": { "content": "<|fim▁begin|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "12": { "content": "<|fim▁hole|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "13": { "content": "<|fim▁end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130972": { "content": "<|System|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130973": { "content": "<|User|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130974": { "content": "<|Tool|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130975": { "content": "<|Developer|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130976": { "content": "<|Bot|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130977": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130978": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130979": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130980": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130981": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130982": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130983": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130984": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "130986": { "content": "<||unused2||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130987": { "content": "<||unused3||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130988": { "content": "<||unused4||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130989": { "content": "<||unused5||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130990": { "content": "<||unused6||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130991": { "content": "<||unused7||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130992": { "content": "<||unused8||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130993": { "content": "<||unused9||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130994": { "content": "<||unused10||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130995": { "content": "<||unused11||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130996": { "content": "<||unused12||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130997": { "content": "<||unused13||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130998": { "content": "<||unused14||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "130999": { "content": "<||unused15||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131000": { "content": "<||unused16||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131001": { "content": "<||unused17||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131002": { "content": "<||unused18||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131003": { "content": "<||unused19||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131004": { "content": "<||unused20||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131005": { "content": "<||unused21||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131006": { "content": "<||unused22||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131007": { "content": "<||unused23||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131008": { "content": "<||unused24||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131009": { "content": "<||unused25||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131010": { "content": "<||unused26||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131011": { "content": "<||unused27||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131012": { "content": "<||unused28||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131013": { "content": "<||unused29||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131014": { "content": "<||unused30||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131015": { "content": "<||unused31||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131016": { "content": "<||unused32||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131017": { "content": "<||unused33||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131018": { "content": "<||unused34||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131019": { "content": "<||unused35||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131020": { "content": "<||unused36||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131021": { "content": "<||unused37||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131022": { "content": "<||unused38||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131023": { "content": "<||unused39||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131024": { "content": "<||unused40||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131025": { "content": "<||unused41||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131026": { "content": "<||unused42||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131027": { "content": "<||unused43||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131028": { "content": "<||unused44||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131029": { "content": "<||unused45||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131030": { "content": "<||unused46||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131031": { "content": "<||unused47||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131032": { "content": "<||unused48||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131033": { "content": "<||unused49||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131034": { "content": "<||unused50||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131035": { "content": "<||unused51||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131036": { "content": "<||unused52||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131037": { "content": "<||unused53||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131038": { "content": "<||unused54||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131039": { "content": "<||unused55||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131040": { "content": "<||unused56||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131041": { "content": "<||unused57||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131042": { "content": "<||unused58||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131043": { "content": "<||unused59||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131044": { "content": "<||unused60||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131045": { "content": "<||unused61||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131046": { "content": "<||unused62||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131047": { "content": "<||unused63||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131048": { "content": "<||unused64||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131049": { "content": "<||unused65||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131050": { "content": "<||unused66||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131051": { "content": "<||unused67||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131052": { "content": "<||unused68||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131053": { "content": "<||unused69||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131054": { "content": "<||unused70||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131055": { "content": "<||unused71||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131056": { "content": "<||unused72||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131057": { "content": "<||unused73||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131058": { "content": "<||unused74||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131059": { "content": "<||unused75||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131060": { "content": "<||unused76||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131061": { "content": "<||unused77||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131062": { "content": "<||unused78||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131063": { "content": "<||unused79||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131064": { "content": "<||unused80||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131065": { "content": "<||unused81||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131066": { "content": "<||unused82||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131067": { "content": "<||unused83||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131068": { "content": "<||unused84||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131069": { "content": "<||unused85||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131070": { "content": "<||unused86||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "131071": { "content": "<||unused87||>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "bos_token": "<|start▁of▁sentence|>", "clean_up_tokenization_spaces": false, "eos_token": "<|end▁of▁sentence|>", "extra_special_tokens": {}, "legacy": true, "model_max_length": 131072, "pad_token": "<|▁pad▁|>", "sp_model_kwargs": {}, "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "" }