Upload processor

preprocessor_config.json

@@ -1,4 +1,7 @@
 {
+  "auto_map": {
+    "AutoProcessor": "processing_longclip.LongCLIPProcessor"
+  },
   "crop_size": {
     "height": 224,
     "width": 224
@@ -19,7 +22,7 @@
     0.26130258,
     0.27577711
   ],
-  "processor_class": "CLIPProcessor",
+  "processor_class": "LongCLIPProcessor",
   "resample": 3,
   "rescale_factor": 0.00392156862745098,
   "size": {

processing_longclip.py

@@ -0,0 +1,20 @@
+from typing import Union
+
+from transformers import CLIPProcessor, CLIPTokenizer, CLIPTokenizerFast
+
+
+class LongCLIPProcessor(CLIPProcessor):
+    tokenizer: Union[CLIPTokenizer, CLIPTokenizerFast]
+
+    def __call__(
+        self, text=None, short_text=None, images=None, return_tensors=None, **kwargs
+    ):
+        encoding = super().__call__(text, images, return_tensors, **kwargs)
+        if short_text is not None:
+            short_text_encoding = self.tokenizer(
+                short_text, return_tensors=return_tensors, **kwargs
+            )
+            encoding["short_input_ids"] = short_text_encoding.input_ids
+            encoding["short_attention_mask"] = short_text_encoding.attention_mask
+
+        return encoding

processor_config.json

@@ -0,0 +1,6 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_longclip.LongCLIPProcessor"
+  },
+  "processor_class": "LongCLIPProcessor"
+}

tokenizer_config.json

@@ -18,6 +18,9 @@
       "special": true
     }
   },
+  "auto_map": {
+    "AutoProcessor": "processing_longclip.LongCLIPProcessor"
+  },
   "bos_token": "<|startoftext|>",
   "clean_up_tokenization_spaces": false,
   "do_lower_case": true,
@@ -26,7 +29,7 @@
   "extra_special_tokens": {},
   "model_max_length": 248,
   "pad_token": "<|endoftext|>",
-  "processor_class": "CLIPProcessor",
+  "processor_class": "LongCLIPProcessor",
   "tokenizer_class": "CLIPTokenizer",
   "unk_token": "<|endoftext|>"
 }