Text Generation
Transformers
Safetensors
xlstm
sft
trl
conversational
🇪🇺 Region: EU
mrs83 commited on
Commit
be15fc2
·
verified ·
1 Parent(s): 9b59638

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -2473,7 +2473,7 @@
2473
  "eos_token": "<|im_end|>",
2474
  "extra_special_tokens": {},
2475
  "model_max_length": 1000000000000000019884624838656,
2476
- "pad_token": null,
2477
  "tokenizer_class": "GPTNeoXTokenizer",
2478
  "unk_token": "<|endoftext|>"
2479
  }
 
2473
  "eos_token": "<|im_end|>",
2474
  "extra_special_tokens": {},
2475
  "model_max_length": 1000000000000000019884624838656,
2476
+ "pad_token": "<|im_end|>",
2477
  "tokenizer_class": "GPTNeoXTokenizer",
2478
  "unk_token": "<|endoftext|>"
2479
  }