Update

app.py

@@ -1,4 +1,5 @@
 import torch
+import torch.nn.functional as F
 import gradio as gr
 from transformers import CLIPProcessor, CLIPModel
 import spaces
@@ -34,18 +35,28 @@ def calculate_score(image, text, model_name):
     inputs = processor(text=labels, images=[image], return_tensors="pt", padding=True)
     inputs = {k: v.to("cuda") for k, v in inputs.items()}
     
-    # Calculate scores
+    # Calculate embeddings
     with torch.no_grad():
         outputs = model(**inputs)
+        image_embeds = outputs.image_embeds
+        text_embeds = outputs.text_embeds
     
-    logits_per_image = outputs.logits_per_image.cpu().numpy()
+    # Normalize embeddings
+    image_embeds = F.normalize(image_embeds, p=2, dim=1)
+    text_embeds = F.normalize(text_embeds, p=2, dim=1)
     
-    results_dict = {label: float(score) for label, score in zip(labels, logits_per_image[0])}
+    # Calculate cosine similarity
+    cosine_similarities = torch.mm(text_embeds, image_embeds.t()).squeeze(1)
+    
+    # Convert to percentages
+    percentages = ((cosine_similarities + 1) / 2 * 100).cpu().numpy()
+    
+    results_dict = {label: float(score) for label, score in zip(labels, percentages)}
     return results_dict
 
 with gr.Blocks() as demo:
     gr.Markdown("# Multi-Model CLIP Score")
-    gr.Markdown("Calculate the [CLIP](https://openai.com/blog/clip/) score of a given image and text using different CLIP model variants")
+    gr.Markdown("Calculate the CLIP score (cosine similarity) between the given image and text descriptions using different CLIP model variants")
     
     with gr.Row():
         image_input = gr.Image(type="pil")