grimjim commited on
Commit
bf72a3f
·
verified ·
1 Parent(s): d8b3704

Update tokenizer_config.json

Browse files

Added Tekken v7 style chat template

Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -9008,6 +9008,7 @@
9008
  ],
9009
  "bos_token": "<s>",
9010
  "clean_up_tokenization_spaces": false,
 
9011
  "eos_token": "</s>",
9012
  "extra_special_tokens": {},
9013
  "legacy": true,
 
9008
  ],
9009
  "bos_token": "<s>",
9010
  "clean_up_tokenization_spaces": false,
9011
+ "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{{- '[SYSTEM_PROMPT]' + messages[0]['content'] + '[/SYSTEM_PROMPT]' }}{%- set loop_messages = messages[1:] %}{%- else %}{%- set loop_messages = messages %}{%- endif %}{%- for message in loop_messages %}{%- if message['role'] == 'user' %}{{- '[INST]' + message['content'] + '[/INST]' }}{%- elif message['role'] == 'system' %}{{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}{%- elif message['role'] == 'assistant' %}{{- message['content'] + eos_token }}{%- else %}{{- raise_exception('Only user, system and assistant roles are supported!') }}{%- endif %}{%- endfor %}",
9012
  "eos_token": "</s>",
9013
  "extra_special_tokens": {},
9014
  "legacy": true,