DEP-model / added_tokens.json
SnowCharmQ's picture
Upload tokenizer
4429d73 verified
{
"</tool_call>": 151658,
"<diff_token_end>": 151684,
"<diff_token_start>": 151683,
"<his_token_end>": 151682,
"<his_token_start>": 151681,
"<tool_call>": 151657,
"<|box_end|>": 151649,
"<|box_start|>": 151648,
"<|endoftext|>": 151643,
"<|file_sep|>": 151664,
"<|fim_middle|>": 151660,
"<|fim_pad|>": 151662,
"<|fim_prefix|>": 151659,
"<|fim_suffix|>": 151661,
"<|im_end|>": 151645,
"<|im_start|>": 151644,
"<|image_pad|>": 151655,
"<|object_ref_end|>": 151647,
"<|object_ref_start|>": 151646,
"<|quad_end|>": 151651,
"<|quad_start|>": 151650,
"<|repo_name|>": 151663,
"<|video_pad|>": 151656,
"<|vision_end|>": 151653,
"<|vision_pad|>": 151654,
"<|vision_start|>": 151652,
"[DIFF_TOKEN_0]": 151673,
"[DIFF_TOKEN_1]": 151674,
"[DIFF_TOKEN_2]": 151675,
"[DIFF_TOKEN_3]": 151676,
"[DIFF_TOKEN_4]": 151677,
"[DIFF_TOKEN_5]": 151678,
"[DIFF_TOKEN_6]": 151679,
"[DIFF_TOKEN_7]": 151680,
"[HIS_TOKEN_0]": 151665,
"[HIS_TOKEN_1]": 151666,
"[HIS_TOKEN_2]": 151667,
"[HIS_TOKEN_3]": 151668,
"[HIS_TOKEN_4]": 151669,
"[HIS_TOKEN_5]": 151670,
"[HIS_TOKEN_6]": 151671,
"[HIS_TOKEN_7]": 151672
}