add amber and crystal_coder

vocab/__init__.py

@@ -135,7 +135,8 @@ all_tokenizers = [
     ("gpt_4", "", "tiktoken"),
 
     # 未分类
-
+    ("amber", ""),
+    ("crystal_coder", ""),
     ("mistral_7b",),
     ("mixtral_8_7b",),
 

vocab/amber/__init__.py

@@ -0,0 +1,3 @@
+from transformers import LlamaTokenizer
+
+tokenizer = LlamaTokenizer.from_pretrained("LLM360/Amber", revision="ckpt_356")

vocab/crystal_coder/__init__.py

@@ -0,0 +1,7 @@
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained(
+    "LLM360/CrystalCoder",
+    revision="CrystalCoder_phase1_checkpoint_055500",
+    trust_remote_code=True
+)