Upload tokenizer

tokenizer_config.json

@@ -2050,7 +2050,7 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
-  "chat_template": "{{bos_token}}{% if not messages | selectattr('role', 'equalto', 'system') | list %}\n{{\"<|start_header_id|>\" + \"system\" + \"<|end_header_id|>\" + \"\n\n\" + \"You are a helpful AI assistant. Please answer the user's questions kindly. 당신은 유능한 AI 어시스턴트 입니다. 사용자의 질문에 대해 친절하게 답변해주세요.\"  + eos_token }}{% endif %}\n{% for message in messages %}\n{% if message['role'] == 'system' %}\n{{ '<|start_header_id|>' + 'system' + '<|end_header_id|>' + '\n\n' +  message['content']  + eos_token }}{% elif message['role'] == 'user' %}\n{{ '<|start_header_id|>' + 'user' + '<|end_header_id|>' + '\n\n' +  message['content']  + eos_token }}{% elif message['role'] == 'assistant' %}\n{{ '<|start_header_id|>' + 'assistant' + '<|end_header_id|>' + '\n\n' +  message['content']  + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>' + 'assistant' + '<|end_header_id|>' + '\n\n' }}{% endif %}",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}\n{% set loop_messages = messages %}\n{% set default_system_prompt = \"You are a helpful AI assistant. Please answer the user's questions kindly. 당신은 유능한 AI 어시스턴트 입니다. 사용자의 질문에 대해 친절하게 답변해주세요.\" %}\n{% if not loop_messages | selectattr('role', 'equalto', 'system') | list %}\n{% set system_message = {'role': 'system', 'content': default_system_prompt} %}\n{% set loop_messages = [system_message] + loop_messages %}\n{% endif %}\n{% for message in loop_messages %}\n{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' %}\n{% if loop.index0 == 0 %}\n    {% set content = bos_token + content %}\n{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|eot_id|>",
   "model_input_names": [