Remove <sep> from tokenizer special tokens (#4)

- Remove <sep> from tokenizer special tokens (4f97bab9b60801470ee9a59ff56e1485c12caada)


Co-authored-by: Gabe Goodhart <[email protected]>

added_tokens.json

@@ -1,5 +1,4 @@
 {
-  "<sep>": 49155,
   "<|end_of_role|>": 49153,
   "<|start_of_role|>": 49152,
   "<|tool_call|>": 49154

tokenizer.json

@@ -200,15 +200,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 49155,
-      "content": "<sep>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -244960,4 +244951,4 @@
       ]
     ]
   }
-}
+}

tokenizer_config.json

@@ -177,14 +177,6 @@
             "rstrip": false,
             "single_word": false,
             "special": true
-        },
-        "49155": {
-            "content": "<sep>",
-            "lstrip": false,
-            "normalized": false,
-            "rstrip": false,
-            "single_word": false,
-            "special": true
         }
     },
     "additional_special_tokens": [
@@ -204,4 +196,4 @@
     "tokenizer_class": "GPT2Tokenizer",
     "unk_token": "<|end_of_text|>",
     "vocab_size": 49152
-}
+}