Update tokenizer_config.json
Browse filesassign pad token the same as ens token which will get the correct pad_token_id : 2
- tokenizer_config.json +2 -10
tokenizer_config.json
CHANGED
|
@@ -4930,14 +4930,6 @@
|
|
| 4930 |
"rstrip": false,
|
| 4931 |
"single_word": false,
|
| 4932 |
"special": false
|
| 4933 |
-
},
|
| 4934 |
-
"100352": {
|
| 4935 |
-
"content": "<pad>",
|
| 4936 |
-
"lstrip": false,
|
| 4937 |
-
"normalized": false,
|
| 4938 |
-
"rstrip": false,
|
| 4939 |
-
"single_word": false,
|
| 4940 |
-
"special": true
|
| 4941 |
}
|
| 4942 |
},
|
| 4943 |
"bos_token": null,
|
|
@@ -4946,12 +4938,12 @@
|
|
| 4946 |
"eos_token": null,
|
| 4947 |
"extra_special_tokens": {
|
| 4948 |
"image_token": "<|img|>",
|
| 4949 |
-
"pad_token": "
|
| 4950 |
},
|
| 4951 |
"image_token": "<|img|>",
|
| 4952 |
"legacy": true,
|
| 4953 |
"model_max_length": 1000000000000000019884624838656,
|
| 4954 |
-
"pad_token": "
|
| 4955 |
"processor_class": "AriaProcessor",
|
| 4956 |
"sp_model_kwargs": {},
|
| 4957 |
"spaces_between_special_tokens": false,
|
|
|
|
| 4930 |
"rstrip": false,
|
| 4931 |
"single_word": false,
|
| 4932 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4933 |
}
|
| 4934 |
},
|
| 4935 |
"bos_token": null,
|
|
|
|
| 4938 |
"eos_token": null,
|
| 4939 |
"extra_special_tokens": {
|
| 4940 |
"image_token": "<|img|>",
|
| 4941 |
+
"pad_token": "<|endoftext|>"
|
| 4942 |
},
|
| 4943 |
"image_token": "<|img|>",
|
| 4944 |
"legacy": true,
|
| 4945 |
"model_max_length": 1000000000000000019884624838656,
|
| 4946 |
+
"pad_token": "<|endoftext|>",
|
| 4947 |
"processor_class": "AriaProcessor",
|
| 4948 |
"sp_model_kwargs": {},
|
| 4949 |
"spaces_between_special_tokens": false,
|