lcm_sdhk_controlnet_canny_ip_adapter_full_face_paint_colab.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/latent-consistency-model-colab/blob/main/lcm_sdhk_controlnet_canny_ip_adapter_full_face_paint_colab.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VjYy0F2gZIPR"
      },
      "outputs": [],
      "source": [
        "!pip install git+https://github.com./huggingface/diffusers -U\n",
        "!pip install -q controlnet-aux transformers accelerate peft gradio==3.50.2\n",
        "!pip install -q https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl\n",
        "!pip install -q git+https://github.com./tencent-ailab/IP-Adapter einops\n",
        "\n",
        "!mkdir /content/image_encoder\n",
        "!wget https://huggingface.co/h94/IP-Adapter/raw/main/models/image_encoder/config.json -O /content/image_encoder/config.json\n",
        "!wget https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors?download=true -O /content/image_encoder/model.safetensors\n",
        "\n",
        "!mkdir /content/models\n",
        "!wget https://huggingface.co/h94/IP-Adapter/resolve/main/models/ip-adapter-full-face_sd15.safetensors?download=true -O /content/models/ip-adapter-full-face_sd15.safetensors\n",
        "\n",
        "from diffusers import StableDiffusionControlNetPipeline, UNet2DConditionModel, ControlNetModel, LCMScheduler\n",
        "import torch\n",
        "from ip_adapter import IPAdapterFull\n",
        "\n",
        "controlnet = ControlNetModel.from_pretrained(\"lllyasviel/control_v11p_sd15_canny\", torch_dtype=torch.float16)\n",
        "pipe = StableDiffusionControlNetPipeline.from_pretrained(\"ckpt/SDHK\", controlnet=controlnet, torch_dtype=torch.float16, safety_checker=None,).to(\"cuda\")\n",
        "pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)\n",
        "pipe.load_lora_weights(\"latent-consistency/lcm-lora-sdv1-5\")\n",
        "ip_model = IPAdapterFull(pipe, \"/content/image_encoder\", \"/content/models/ip-adapter-full-face_sd15.safetensors\", \"cuda\", num_tokens=257)\n",
        "\n",
        "import cv2\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "import gradio as gr\n",
        "\n",
        "def generate_canny(input_image, threshold1, threshold2):\n",
        "    input_image = np.array(input_image)\n",
        "    input_image = cv2.Canny(input_image, threshold1, threshold2)\n",
        "    input_image = input_image[:, :, None]\n",
        "    input_image = np.concatenate([input_image, input_image, input_image], axis=2)\n",
        "    canny_image = Image.fromarray(input_image)\n",
        "    return canny_image.resize((512, 768))\n",
        "\n",
        "def generate(prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image):\n",
        "    face_image.resize((256, 256))\n",
        "    image = ip_model.generate(prompt=prompt, negative_prompt=negative_prompt, width=512, height=768, pil_image=face_image, image=canny_image, num_samples=1, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, scale=controlnet_conditioning_scale)[0]\n",
        "    config = \" | \".join([f\"prompt: {prompt}\", f\"negative_prompt: {negative_prompt}\", f\"controlnet_conditioning_scale: {controlnet_conditioning_scale}\", f\"scale: {scale}\", f\"num_inference_steps: {num_inference_steps}\", f\"guidance_scale: {guidance_scale}\"])\n",
        "    return image.resize((512, 768)), config\n",
        "\n",
        "with gr.Blocks(title=f\"Realtime Latent Consistency Model\") as demo:\n",
        "    with gr.Box(scale=23):\n",
        "      with gr.Row():\n",
        "        with gr.Column():\n",
        "            with gr.Row():\n",
        "              prompt = gr.Textbox(show_label=False, value=\"1girl red dress\")\n",
        "            with gr.Row():\n",
        "              negative_prompt = gr.Textbox(show_label=False, value=\"blurry\")\n",
        "        with gr.Column():\n",
        "            with gr.Row():\n",
        "              scale = gr.Slider(minimum=0, maximum=1, step=0.1, value=1, label=\"lora_scale\")\n",
        "              threshold1 = gr.Slider(minimum=0, maximum=500, step=1, value=100, label=\"threshold1\")\n",
        "              threshold2 = gr.Slider(minimum=0, maximum=500, step=1, value=200, label=\"threshold2\")\n",
        "            with gr.Row():\n",
        "              num_inference_steps = gr.Slider(minimum=1, maximum=50, step=1, value=4, label=\"num_inference_steps\")\n",
        "              guidance_scale = gr.Slider(minimum=0, maximum=10, step=0.5, value=1, label=\"guidance_scale\")\n",
        "              controlnet_conditioning_scale = gr.Slider(minimum=0, maximum=1, step=0.1, value=0.5, label=\"controlnet_scale\")\n",
        "    with gr.Row():\n",
        "        input_image = gr.Image(\n",
        "            show_label=False,\n",
        "            type=\"pil\",\n",
        "            tool=\"color-sketch\",\n",
        "            source=\"upload\",\n",
        "            width=512,\n",
        "            height=768,\n",
        "            brush_radius=5.0,\n",
        "            interactive=True,\n",
        "        )\n",
        "        canny_image = gr.Image(\n",
        "            show_label=False,\n",
        "            type=\"pil\",\n",
        "            tool=\"color-sketch\",\n",
        "            source=\"upload\",\n",
        "            interactive=True,\n",
        "            width=512,\n",
        "            height=768,\n",
        "        )\n",
        "        output_image = gr.Image(\n",
        "            show_label=False,\n",
        "            type=\"pil\",\n",
        "            tool=\"color-sketch\",\n",
        "            source=\"upload\",\n",
        "            interactive=False,\n",
        "            width=512,\n",
        "            height=768,\n",
        "        )\n",
        "    with gr.Row():\n",
        "      config = gr.Label(show_label=False)\n",
        "      face_image = gr.Image(\n",
        "            show_label=False,\n",
        "            type=\"pil\",\n",
        "            source=\"upload\",\n",
        "            interactive=True,\n",
        "            width=256,\n",
        "            height=256,\n",
        "        )\n",
        "\n",
        "    input_image.change(fn=generate_canny, inputs=[input_image, threshold1, threshold2], outputs=[canny_image], show_progress=False)\n",
        "    threshold1.change(fn=generate_canny, inputs=[input_image, threshold1, threshold2], outputs=[canny_image], show_progress=False)\n",
        "    threshold2.change(fn=generate_canny, inputs=[input_image, threshold1, threshold2], outputs=[canny_image], show_progress=False)\n",
        "    prompt.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    negative_prompt.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    controlnet_conditioning_scale.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    scale.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    num_inference_steps.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    guidance_scale.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    canny_image.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "    face_image.change(fn=generate, inputs=[prompt, negative_prompt, controlnet_conditioning_scale, scale, num_inference_steps, guidance_scale, canny_image, face_image], outputs=[output_image, config], show_progress=False)\n",
        "\n",
        "demo.launch(inline=False, share=True, debug=True)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}