Upload tokenizer.json

tokenizer.json

@@ -45,7 +45,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -71,6 +71,12 @@
           "String": " "
         },
         "content": "▁"
+      },
+      {
+          "type":"Replace",
+          "pattern":{
+              "Regex":"!\\[\\]([^\\]]*)"},
+              "content":"<image>"
       }
     ]
   },
@@ -93415,4 +93421,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}