Update cac support

app.py

@@ -282,7 +282,8 @@ with gr.Blocks(css=css) as demo:
               </div>
               <p>
                 Demo for CycleDiffusion with Stable Diffusion. <br>
-                <a href="https://huggingface.co/docs/diffusers/main/en/api/pipelines/cycle_diffusion">🧨 Pipeline doc</a> | <a href="https://arxiv.org/abs/2210.05559">📄 Paper link</a>
+                CycleDiffusion (<a href="https://github.com/ChenWu98/cycle-diffusion">Github</a> | <a href="https://arxiv.org/abs/2210.05559">📄 Paper link</a> | <a href="https://huggingface.co/docs/diffusers/main/en/api/pipelines/cycle_diffusion">🧨 Pipeline doc</a>) is an image-to-image translation method that supports stochastic samplers for diffusion models. <br>
+                It also supports Cross Attention Control (<a href="https://github.com/google/prompt-to-prompt">Github</a> | <a href="https://arxiv.org/abs/2208.01626">📄 Paper link</a>), which is a technique to transfer the attention map from the source prompt to the target prompt. <br>
               </p>
               <p>You can skip the queue in the colab: <a href="https://colab.research.google.com/gist/ChenWu98/0aa4fe7be80f6b45d3d055df9f14353a/copy-of-fine-tuned-diffusion-gradio.ipynb"><img data-canonical-src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab" src="https://colab.research.google.com/assets/colab-badge.svg"></a></p>
                Running on <b>{device_print}</b>{(" in a <b>Google Colab</b>." if is_colab else "")}

ptp_utils.py

@@ -14,162 +14,7 @@
 
 import numpy as np
 import torch
-from PIL import Image, ImageDraw, ImageFont
-import cv2
-from typing import Optional, Union, Tuple, List, Callable, Dict
-from IPython.display import display
-from tqdm.notebook import tqdm
-
-
-def text_under_image(image: np.ndarray, text: str, text_color: Tuple[int, int, int] = (0, 0, 0)):
-    h, w, c = image.shape
-    offset = int(h * .2)
-    img = np.ones((h + offset, w, c), dtype=np.uint8) * 255
-    font = cv2.FONT_HERSHEY_SIMPLEX
-    # font = ImageFont.truetype("/usr/share/fonts/truetype/noto/NotoMono-Regular.ttf", font_size)
-    img[:h] = image
-    textsize = cv2.getTextSize(text, font, 1, 2)[0]
-    text_x, text_y = (w - textsize[0]) // 2, h + offset - textsize[1] // 2
-    cv2.putText(img, text, (text_x, text_y ), font, 1, text_color, 2)
-    return img
-
-
-def view_images(images, num_rows=1, offset_ratio=0.02):
-    if type(images) is list:
-        num_empty = len(images) % num_rows
-    elif images.ndim == 4:
-        num_empty = images.shape[0] % num_rows
-    else:
-        images = [images]
-        num_empty = 0
-
-    empty_images = np.ones(images[0].shape, dtype=np.uint8) * 255
-    images = [image.astype(np.uint8) for image in images] + [empty_images] * num_empty
-    num_items = len(images)
-
-    h, w, c = images[0].shape
-    offset = int(h * offset_ratio)
-    num_cols = num_items // num_rows
-    image_ = np.ones((h * num_rows + offset * (num_rows - 1),
-                      w * num_cols + offset * (num_cols - 1), 3), dtype=np.uint8) * 255
-    for i in range(num_rows):
-        for j in range(num_cols):
-            image_[i * (h + offset): i * (h + offset) + h:, j * (w + offset): j * (w + offset) + w] = images[
-                i * num_cols + j]
-
-    pil_img = Image.fromarray(image_)
-    display(pil_img)
-
-
-    
-def diffusion_step(model, controller, latents, context, t, guidance_scale, low_resource=False):
-    if low_resource:
-        noise_pred_uncond = model.unet(latents, t, encoder_hidden_states=context[0])["sample"]
-        noise_prediction_text = model.unet(latents, t, encoder_hidden_states=context[1])["sample"]
-    else:
-        latents_input = torch.cat([latents] * 2)
-        noise_pred = model.unet(latents_input, t, encoder_hidden_states=context)["sample"]
-        noise_pred_uncond, noise_prediction_text = noise_pred.chunk(2)
-    noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond)
-    latents = model.scheduler.step(noise_pred, t, latents)["prev_sample"]
-    latents = controller.step_callback(latents)
-    return latents
-
-
-def latent2image(vae, latents):
-    latents = 1 / 0.18215 * latents
-    image = vae.decode(latents)['sample']
-    image = (image / 2 + 0.5).clamp(0, 1)
-    image = image.cpu().permute(0, 2, 3, 1).numpy()
-    image = (image * 255).astype(np.uint8)
-    return image
-
-
-def init_latent(latent, model, height, width, generator, batch_size):
-    if latent is None:
-        latent = torch.randn(
-            (1, model.unet.in_channels, height // 8, width // 8),
-            generator=generator,
-        )
-    latents = latent.expand(batch_size,  model.unet.in_channels, height // 8, width // 8).to(model.device)
-    return latent, latents
-
-
-@torch.no_grad()
-def text2image_ldm(
-    model,
-    prompt:  List[str],
-    controller,
-    num_inference_steps: int = 50,
-    guidance_scale: Optional[float] = 7.,
-    generator: Optional[torch.Generator] = None,
-    latent: Optional[torch.FloatTensor] = None,
-):
-    register_attention_control(model, controller)
-    height = width = 256
-    batch_size = len(prompt)
-    
-    uncond_input = model.tokenizer([""] * batch_size, padding="max_length", max_length=77, return_tensors="pt")
-    uncond_embeddings = model.bert(uncond_input.input_ids.to(model.device))[0]
-    
-    text_input = model.tokenizer(prompt, padding="max_length", max_length=77, return_tensors="pt")
-    text_embeddings = model.bert(text_input.input_ids.to(model.device))[0]
-    latent, latents = init_latent(latent, model, height, width, generator, batch_size)
-    context = torch.cat([uncond_embeddings, text_embeddings])
-    
-    model.scheduler.set_timesteps(num_inference_steps)
-    for t in tqdm(model.scheduler.timesteps):
-        latents = diffusion_step(model, controller, latents, context, t, guidance_scale)
-    
-    image = latent2image(model.vqvae, latents)
-   
-    return image, latent
-
-
-
-@torch.no_grad()
-def text2image_ldm_stable(
-    model,
-    prompt: List[str],
-    controller,
-    num_inference_steps: int = 50,
-    guidance_scale: float = 7.5,
-    generator: Optional[torch.Generator] = None,
-    latent: Optional[torch.FloatTensor] = None,
-    low_resource: bool = False,
-):
-    register_attention_control(model, controller)
-    height = width = 512
-    batch_size = len(prompt)
-
-    text_input = model.tokenizer(
-        prompt,
-        padding="max_length",
-        max_length=model.tokenizer.model_max_length,
-        truncation=True,
-        return_tensors="pt",
-    )
-    text_embeddings = model.text_encoder(text_input.input_ids.to(model.device))[0]
-    max_length = text_input.input_ids.shape[-1]
-    uncond_input = model.tokenizer(
-        [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt"
-    )
-    uncond_embeddings = model.text_encoder(uncond_input.input_ids.to(model.device))[0]
-    
-    context = [uncond_embeddings, text_embeddings]
-    if not low_resource:
-        context = torch.cat(context)
-    latent, latents = init_latent(latent, model, height, width, generator, batch_size)
-    
-    # set timesteps
-    extra_set_kwargs = {"offset": 1}
-    model.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs)
-    for t in tqdm(model.scheduler.timesteps):
-        latents = diffusion_step(model, controller, latents, context, t, guidance_scale, low_resource)
-    
-    image = latent2image(model.vae, latents)
-  
-    return image, latent
+from typing import Optional, Union, Tuple, Dict
 
 
 def register_attention_control(model, controller):