experiments with open_clip, templates, clustering, recursion

experimental/clip_app.py

@@ -11,6 +11,7 @@ from clip_retrieval.load_clip import load_clip, get_tokenizer
 # from clip_retrieval.clip_client import ClipClient, Modality
 
 @serve.deployment(num_replicas=6, ray_actor_options={"num_cpus": .2, "num_gpus": 0.1})
+# @serve.deployment(num_replicas=3, ray_actor_options={"num_cpus": .2, "num_gpus": 0.2})
 class CLIPTransform:
     def __init__(self):
         # os.environ["OMP_NUM_THREADS"] = "20"
@@ -18,7 +19,7 @@ class CLIPTransform:
         # Load model
         self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
         self._clip_model="ViT-L/14"
-        self._clip_model_id ="laion5B-L-14"
+        # self._clip_model="open_clip:ViT-H-14"
         self.model, self.preprocess = load_clip(self._clip_model, use_jit=True, device=self.device)
         self.tokenizer = get_tokenizer(self._clip_model)
 
@@ -104,7 +105,7 @@ class CLIPTransform:
         else:
             print ("Invalid request")
             raise Exception("Invalid request")
-        return embeddings.cpu().numpy().tolist()
+        return embeddings.float().cpu().numpy().tolist()
 
         request = await http_request.json()
         # print(type(request))

experimental/clip_app_client.py

@@ -28,10 +28,11 @@ class ClipAppClient:
     """
 
     def __init__(self, clip_model="ViT-L/14", device=None):
+    # def __init__(self, clip_model="open_clip:ViT-H-14", device=None):
         self.clip_model = clip_model
         self.device = device or ("cuda:0" if torch.cuda.is_available() else "cpu")
         print("using device", self.device)
-        self.model, self.preprocess = load_clip(clip_model, use_jit=True, device=self.device)
+        _, self.preprocess = load_clip(clip_model, use_jit=True, device=self.device)
         self.tokenizer = get_tokenizer(clip_model)
 
     def preprocess_image(self, image_url):

experimental/vision001.py

@@ -12,6 +12,8 @@ from clip_retrieval.clip_client import ClipClient, Modality
 clip_retrieval_service_url = "https://knn.laion.ai/knn-service"
 map_clip_to_clip_retreval = {
     "ViT-L/14": "laion5B-L-14",
+    "open_clip:ViT-H-14": "laion5B-H-14",
+    "open_clip:ViT-L-14": "laion5B-L-14",
 }
 
 

experimental/vision002.py

@@ -12,6 +12,8 @@ from clip_retrieval.clip_client import ClipClient, Modality
 clip_retrieval_service_url = "https://knn.laion.ai/knn-service"
 map_clip_to_clip_retreval = {
     "ViT-L/14": "laion5B-L-14",
+    "open_clip:ViT-H-14": "laion5B-H-14",
+    "open_clip:ViT-L-14": "laion5B-L-14",
 }
 
 
@@ -55,8 +57,8 @@ def clustering_templates(embeddings, n_clusters=5):
     return templates
 
 # test_image_path = os.path.join(os.getcwd(), "images", "plant-001.png")
-test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-001.jpeg")
-# test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
+# test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-001.jpeg")
+test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
 # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "plant-002.jpeg")
 # test_image_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "car-002.jpeg")
 
@@ -78,6 +80,7 @@ print (f"embeddings: {preprocessed_image_embeddings.shape}")
 
 
 template = preprocessed_image_embeddings
+template = template / template.norm()
 for step_num in range(3):
     print (f"\n\n---- Step {step_num} ----")
 
@@ -123,7 +126,10 @@ for step_num in range(3):
     # template = clusters[cluster_similarity[0][1]]
     template = preprocessed_image_embeddings * (len(clusters)-1)
     for i in range(1, len(clusters)):
-        template -= clusters[cluster_similarity[i][1]]
+        cluster = clusters[cluster_similarity[i][1]]
+        normalized_cluster = cluster / cluster.norm()
+        template -= normalized_cluster
+    template = template / template.norm()
     print("---")
     print(f"seaching based on template")
     results = clip_retrieval_client.query(embedding_input=template[0].tolist())