Upload 10 files

jade/config.json

@@ -1,7 +1,7 @@
 {
   "groq_model": "moonshotai/kimi-k2-instruct-0905",
   "audio_model": "whisper-large-v3",
-  "caption_model": "Salesforce/blip-image-captioning-large",
+  "caption_model": "microsoft/Florence-2-base-ft",
   "max_context": 12,
   "language": "pt",
   "local_mode": false

jade/handlers.py

@@ -1,4 +1,4 @@
-from transformers import BlipProcessor, BlipForConditionalGeneration
+from transformers import AutoProcessor, AutoModelForCausalLM
 from PIL import Image
 import torch
 
@@ -13,11 +13,11 @@ class AudioHandler:
         
 class ImageHandler:
     def __init__(self, model_name):
-        self.processor = BlipProcessor.from_pretrained(model_name, use_fast=True)
-        self.model = BlipForConditionalGeneration.from_pretrained(model_name)
-        self.model.eval()
+        self.processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
+        self.model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
         self.device = "cuda" if torch.cuda.is_available() else "cpu"
         self.model.to(self.device)
+        self.model.eval()
 
     def process_pil_image(self, pil_image: Image.Image):
         """Processa um objeto PIL.Image vindo diretamente do Gradio."""
@@ -26,8 +26,29 @@ class ImageHandler:
         return self._generate_caption(pil_image.convert("RGB"))
 
     def _generate_caption(self, img):
-        """Lógica de geração de legenda reutilizável."""
+        """Lógica de geração de legenda reutilizável usando Florence-2."""
+        # Prompt para descrição detalhada
+        prompt = "<MORE_DETAILED_CAPTION>"
+        
         with torch.no_grad():
-            inputs = self.processor(img, "a photo of", return_tensors="pt").to(self.device)
-            out = self.model.generate(**inputs, max_new_tokens=60)
-            return self.processor.decode(out[0], skip_special_tokens=True).strip()
+            inputs = self.processor(text=prompt, images=img, return_tensors="pt").to(self.device)
+            
+            generated_ids = self.model.generate(
+                input_ids=inputs["input_ids"],
+                pixel_values=inputs["pixel_values"],
+                max_new_tokens=1024,
+                do_sample=False,
+                num_beams=3,
+            )
+            
+            generated_text = self.processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
+            
+            # O Florence-2 requer pós-processamento para extrair a resposta limpa
+            parsed_answer = self.processor.post_process_generation(
+                generated_text, 
+                task=prompt, 
+                image_size=(img.width, img.height)
+            )
+            
+            # parsed_answer retorna um dict, ex: {'<MORE_DETAILED_CAPTION>': 'texto da legenda'}
+            return parsed_answer.get(prompt, "")

requirements.txt

@@ -12,4 +12,6 @@ fastapi
 uvicorn[standard]
 joblib
 scikit-learn
-numpy
+numpy
+einops
+timm