Update chat-template
Browse files- special_tokens_map.json +7 -1
- tokenizer_config.json +1 -0
special_tokens_map.json
CHANGED
|
@@ -12,5 +12,11 @@
|
|
| 12 |
">>SUFFIX<<",
|
| 13 |
">>MIDDLE<<"
|
| 14 |
],
|
| 15 |
-
"eos_token":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
}
|
|
|
|
| 12 |
">>SUFFIX<<",
|
| 13 |
">>MIDDLE<<"
|
| 14 |
],
|
| 15 |
+
"eos_token": {
|
| 16 |
+
"content": "<|endoftext|>",
|
| 17 |
+
"lstrip": false,
|
| 18 |
+
"normalized": false,
|
| 19 |
+
"rstrip": false,
|
| 20 |
+
"single_word": false
|
| 21 |
+
}
|
| 22 |
}
|
tokenizer_config.json
CHANGED
|
@@ -111,6 +111,7 @@
|
|
| 111 |
">>SUFFIX<<",
|
| 112 |
">>MIDDLE<<"
|
| 113 |
],
|
|
|
|
| 114 |
"clean_up_tokenization_spaces": true,
|
| 115 |
"eos_token": "<|endoftext|>",
|
| 116 |
"model_input_names": [
|
|
|
|
| 111 |
">>SUFFIX<<",
|
| 112 |
">>MIDDLE<<"
|
| 113 |
],
|
| 114 |
+
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{% if not loop.first %}{% endif %}{% if message['role'] == 'system' %}{{ message['content'] + ' ' }}{% elif message['role'] == 'user' %}{{ 'USER: ' + message['content'] + ' ' }}{% elif message['role'] == 'assistant' %}{{ 'ASSISTANT: ' + message['content'] + '</s>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'ASSISTANT:' }}{% endif %}",
|
| 115 |
"clean_up_tokenization_spaces": true,
|
| 116 |
"eos_token": "<|endoftext|>",
|
| 117 |
"model_input_names": [
|