Update README.md

README.md

@@ -25,7 +25,6 @@ The model is based on the paper [Instruct-CLIP: Improving Instruction-Guided Ima
 ## Capabilities
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/SherryXTChen/Instruct-CLIP/refs/heads/main/assets/teaser_1.png" alt="Figure 1" width="43%">
   <img src="https://raw.githubusercontent.com/SherryXTChen/Instruct-CLIP/refs/heads/main/assets/teaser_2.png" alt="Figure 2" width="50%">
 </p>
 
@@ -34,31 +33,83 @@ The model is based on the paper [Instruct-CLIP: Improving Instruction-Guided Ima
 pip install -r requirements.txt
 ```
 
-## Inference
+## Edit Instruction Refinement Inference
 
 ```python
-import PIL
-import requests
+from PIL import Image
 import torch
-from diffusers import StableDiffusionInstructPix2PixPipeline, EulerAncestralDiscreteScheduler
+from torchvision import transforms
 
-model_id = "timbrooks/instruct-pix2pix"
-pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
-pipe.load_lora_weights("SherryXTChen/InstructCLIP-InstructPix2Pix")
-pipe.to("cuda")
-pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
+from model import InstructCLIP
+from utils import get_sd_components, normalize
 
-url = "https://raw.githubusercontent.com/SherryXTChen/Instruct-CLIP/refs/heads/main/assets/1_input.jpg"
-def download_image(url):
-    image = PIL.Image.open(requests.get(url, stream=True).raw)
-    image = PIL.ImageOps.exif_transpose(image)
-    image = image.convert("RGB")
-    return image
-image = download_image(url)
+parser = argparse.ArgumentParser(description="Simple example of estimating edit instruction from image pair")
+parser.add_argument(
+    "--pretrained_instructclip_name_or_path",
+    type=str,
+    default="SherryXTChen/Instruct-CLIP",
+    help=(
+        "instructclip pretrained checkpoints"
+    ),
+)
+parser.add_argument(
+    "--pretrained_model_name_or_path",
+    type=str,
+    default="runwayml/stable-diffusion-v1-5",
+    help=(
+        "sd pretrained checkpoints"
+    ),
+)
+parser.add_argument(
+    "--input_path",
+    type=str,
+    default="assets/1_input.jpg",
+    help=(
+        "Input image path"
+    )
+)
+parser.add_argument(
+    "--output_path",
+    type=str,
+    default="assets/1_output.jpg",
+    help=(
+        "Output image path"
+    )
+)
+args = parser.parse_args()
+device = "cuda"
+    
+# load model for edit instruction estimation
+model = InstructCLIP.from_pretrained("SherryXTChen/Instruct-CLIP")
+model = model.to(device).eval()
 
-prompt = "as a 3 d sculpture"
-images = pipe(prompt, image=image, num_inference_steps=20).images
-images[0].save("output.jpg")
+# load model to preprocess/encode image to latent space
+tokenizer, _, vae, _, _ = get_sd_components(args, device, torch.float32)
+
+# prepare image input
+transform = transforms.Compose([
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.5], std=[0.5]),
+])
+image_list = [args.input_path, args.output_path]
+image_list = [
+    transform(Image.open(f).resize((512, 512))).unsqueeze(0).to(device) 
+    for f in image_list
+]
+
+with torch.no_grad():
+    image_list = [vae.encode(x).latent_dist.sample() * vae.config.scaling_factor for x in image_list]
+    
+    # get image feature
+    zero_timesteps = torch.zeros_like(torch.tensor([0])).to(device) 
+    img_feat = model.get_image_features(
+        inp=image_list[0], out=image_list[1], inp_t=zero_timesteps, out_t=zero_timesteps)
+    img_feat = normalize(img_feat)
+    
+    # get edit instruction
+    pred_instruct_input_ids = model.text_decoder.infer(img_feat[:1])[0]
+    pred_instruct = tokenizer.decode(pred_instruct_input_ids, skip_special_tokens=True)
+    print(pred_instruct)  # as a 3 d sculpture
 ```
 
 ## Citation