{ "added_tokens_decoder": { "151851": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151852": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151853": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151854": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151855": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151856": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151857": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151858": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151859": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "", "", "", "", "", "", "", "", "" ], "clean_up_tokenization_spaces": true, "model_max_length": 8000, "pad_token": "<|endoftext|>", "padding_side": "right", "tokenizer_class": "QWenTokenizer" }