InstantID_IPAdapter_ControlNet_jupyter.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/InstantID-IPAdapter-ControlNet-jupyter/blob/main/InstantID_IPAdapter_ControlNet_jupyter.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VjYy0F2gZIPR"
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "!git clone -b totoro https://github.com./camenduru/ComfyUI /content/TotoroUI\n",
        "!git clone -b totoro_v2 https://github.com./camenduru/ComfyUI_IPAdapter_plus /content/TotoroUI/IPAdapter\n",
        "!git clone -b totoro https://github.com./camenduru/ComfyUI_InstantID /content/TotoroUI/InstantID\n",
        "\n",
        "!pip install -q torch==2.2.1+cu121 torchvision==0.17.1+cu121 torchaudio==2.2.1+cu121 torchtext==0.17.1 torchdata==0.7.1 --extra-index-url https://download.pytorch.org/whl/cu121\n",
        "!pip install -q torchsde einops diffusers accelerate xformers==0.0.25 insightface onnxruntime onnxruntime-gpu\n",
        "!apt -y install -qq aria2\n",
        "\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://civitai.com/api/download/models/470847 -d /content/TotoroUI/models -o raemuXL_v35Lightning.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/h94/IP-Adapter/resolve/main/models/image_encoder/model.safetensors -d /content/TotoroUI/models/clip_vision -o CLIP-ViT-H-14-laion2B-s32B-b79K.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/h94/IP-Adapter/resolve/main/sdxl_models/ip-adapter-plus-face_sdxl_vit-h.safetensors  -d /content/TotoroUI/models/ipadapter -o ip-adapter-plus-face_sdxl_vit-h.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lllyasviel/sd_control_collection/resolve/main/thibaud_xl_openpose.safetensors -d /content/TotoroUI/models/controlnet -o thibaud_xl_openpose.safetensors\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://github.com./Ttl/ComfyUi_NNLatentUpscale/raw/master/sdxl_resizer.pt -d /content/TotoroUI/models -o sdxl_resizer.pt\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/DIAMONIK7777/antelopev2/resolve/main/1k3d68.onnx -d /content/TotoroUI/models/insightface/models/antelopev2 -o 1k3d68.onnx\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/DIAMONIK7777/antelopev2/resolve/main/2d106det.onnx -d /content/TotoroUI/models/insightface/models/antelopev2 -o 2d106det.onnx\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/DIAMONIK7777/antelopev2/resolve/main/genderage.onnx -d /content/TotoroUI/models/insightface/models/antelopev2 -o genderage.onnx\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/DIAMONIK7777/antelopev2/resolve/main/glintr100.onnx -d /content/TotoroUI/models/insightface/models/antelopev2 -o glintr100.onnx\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/DIAMONIK7777/antelopev2/resolve/main/scrfd_10g_bnkps.onnx -d /content/TotoroUI/models/insightface/models/antelopev2 -o scrfd_10g_bnkps.onnx\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/InstantX/InstantID/resolve/main/ip-adapter.bin -d /content/TotoroUI/models/instantid -o ip-adapter.bin\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/InstantX/InstantID/resolve/main/ControlNetModel/diffusion_pytorch_model.safetensors -d /content/TotoroUI/models/controlnet/SDXL/instantid -o diffusion_pytorch_model.safetensors\n",
        "\n",
        "!wget https://huggingface.co/camenduru/IICF/resolve/main/test/anya.jpg -O /content/anya.jpg\n",
        "!wget https://huggingface.co/camenduru/IICF/resolve/main/test/pose_images.zip -O /content/pose_images.zip\n",
        "!unzip /content/pose_images.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "%cd /content/TotoroUI\n",
        "import torch\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "import totoro\n",
        "import nodes\n",
        "import sys\n",
        "sys.path.append('/content/TotoroUI/IPAdapter')\n",
        "import IPAdapterPlus\n",
        "sys.path.append('/content/TotoroUI/InstantID')\n",
        "import InstantID\n",
        "import scipy\n",
        "import model_management\n",
        "from latent_resizer import LatentResizer\n",
        "from totoro import model_management\n",
        "import gc\n",
        "import random\n",
        "\n",
        "def upscale(latent, upscale):\n",
        "  device = model_management.get_torch_device()\n",
        "  samples = latent.to(device=device, dtype=torch.float16)\n",
        "  model = LatentResizer.load_model('/content/TotoroUI/models/sdxl_resizer.pt', device, torch.float16)\n",
        "  model.to(device=device)\n",
        "  latent_out = (model(0.13025 * samples, scale=upscale) / 0.13025)\n",
        "  latent_out = latent_out.to(device=\"cpu\")\n",
        "  model.to(device=model_management.vae_offload_device())\n",
        "  return ({\"samples\": latent_out},)\n",
        "\n",
        "with torch.no_grad():\n",
        "    model_patcher, clip, vae, clipvision = totoro.sd.load_checkpoint_guess_config(\"/content/TotoroUI/models/raemuXL_v35Lightning.safetensors\", output_vae=True, output_clip=True, embedding_directory=None)\n",
        "    IPAdapterPlus_model = IPAdapterPlus.IPAdapterUnifiedLoader().load_models(model_patcher, 'PLUS FACE (portraits)', lora_strength=0.0, provider=\"CUDA\")\n",
        "    instantid = InstantID.InstantIDModelLoader().load_model(\"/content/TotoroUI/models/instantid/ip-adapter.bin\")\n",
        "    insightface = InstantID.InstantIDFaceAnalysis().load_insight_face(\"CUDA\")\n",
        "    instantid_control_net = totoro.controlnet.load_controlnet(\"/content/TotoroUI/models/controlnet/SDXL/instantid/diffusion_pytorch_model.safetensors\")\n",
        "    output_image, output_mask = nodes.LoadImage().load_image(\"/content/anya.jpg\") \n",
        "    image_kps, image_kps_mask = nodes.LoadImage().load_image(\"/content/pose_images/pose_kps_00008_.png\")\n",
        "    image_dw, image_dw_mask = nodes.LoadImage().load_image(\"/content/pose_images/pose_dw_pose_00008_.png\")\n",
        "    ip_model_patcher = IPAdapterPlus.IPAdapterAdvanced().apply_ipadapter(IPAdapterPlus_model[0], IPAdapterPlus_model[1], image=output_image, weight_type=\"style transfer\")\n",
        "    tokens = clip.tokenize(\"1girl\")\n",
        "    cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True)\n",
        "    cond = [[cond, {\"pooled_output\": pooled}]]\n",
        "    n_tokens = clip.tokenize(\"(nsfw:1.5), nipple, nude, naked, lowres, child, getty, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, artist name, trademark, watermark, title, multiple view, reference sheet, mutated hands and fingers, poorly drawn face, mutation, deformed, ugly, bad proportions, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, tatoo, amateur drawing, odd eyes, uneven eyes, unnatural face, uneven nostrils, crooked mouth, bad teeth, crooked teeth, photoshop, video game, censor, censored, ghost, b&w, weird colors, gradient background, spotty background, blurry background, ugly background, simple background, realistic, out of frame, extra objects, gross, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of focus, blurry, very long body, duplicate, morbid, mutilated, extra fingers, mutated hands, poorly drawn eyes, cloned face, disfigured, deformed, cross-eye, extra limbs, missing limb, malformed hands, mutated, morbid, mutilated, disfigured, extra arms, extra hands, mangled fingers, contorted, conjoined, mismatched limbs, mismatched parts, bad perspective, black and white, oversaturated, undersaturated, bad shadow, cropped image, draft, grainy, pixelated\")\n",
        "    n_cond, n_pooled = clip.encode_from_tokens(n_tokens, return_pooled=True)\n",
        "    n_cond = [[n_cond, {\"pooled_output\": n_pooled}]]\n",
        "    work_model, instantid_cond, instantid_n_cond = InstantID.ApplyInstantID().apply_instantid(instantid=instantid[0], insightface=insightface[0], control_net=instantid_control_net, image=output_image, model=ip_model_patcher[0], positive=cond, negative=n_cond, start_at=0.0, end_at=1.0, weight=0.80, image_kps=image_kps)\n",
        "    openpose_control_net = totoro.controlnet.load_controlnet(\"/content/TotoroUI/models/controlnet/thibaud_xl_openpose.safetensors\")\n",
        "    openpose_cond = nodes.ControlNetApply().apply_controlnet(conditioning=instantid_cond, control_net=openpose_control_net, image=image_dw, strength=0.90)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "instantid = None\n",
        "insightface = None\n",
        "instantid_control_net = None\n",
        "ip_model_patcher = None\n",
        "cond = None\n",
        "n_cond = None\n",
        "model_patcher = None\n",
        "clip = None\n",
        "clipvision = None\n",
        "IPAdapterPlus_model = None\n",
        "model_management.cleanup_models()\n",
        "gc.collect()\n",
        "model_management.soft_empty_cache()\n",
        "\n",
        "ran = random.randint(0, 65535)\n",
        "print(ran)\n",
        "\n",
        "with torch.no_grad():\n",
        "    latent = {\"samples\":torch.zeros([1, 4, 1024 // 8, 1024 // 8])}\n",
        "    sample = nodes.common_ksampler(model=work_model, \n",
        "                          seed=ran, \n",
        "                          steps=4, \n",
        "                          cfg=1.3, \n",
        "                          sampler_name=\"dpmpp_sde_gpu\", \n",
        "                          scheduler=\"karras\", \n",
        "                          positive=openpose_cond[0], \n",
        "                          negative=instantid_n_cond,\n",
        "                          latent=latent, \n",
        "                          denoise=0.95)\n",
        "\n",
        "    with torch.inference_mode():\n",
        "        sample = sample[0][\"samples\"].to(torch.float16)\n",
        "        vae.first_stage_model.cuda()\n",
        "        decoded = vae.decode_tiled(sample).detach()\n",
        "\n",
        "Image.fromarray(np.array(decoded*255, dtype=np.uint8)[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "  latent = upscale(sample, 1.5)\n",
        "  sample = nodes.common_ksampler(model=work_model,\n",
        "                            seed=ran,\n",
        "                            steps=4, \n",
        "                            cfg=1.3, \n",
        "                            sampler_name=\"dpmpp_sde_gpu\", \n",
        "                            scheduler=\"karras\", \n",
        "                            positive=openpose_cond[0], \n",
        "                            negative=instantid_n_cond,\n",
        "                            latent=latent[0],\n",
        "                            denoise=0.55)\n",
        "  with torch.inference_mode():\n",
        "    sample = sample[0][\"samples\"].to(torch.float16)\n",
        "    vae.first_stage_model.cuda()\n",
        "    decoded = vae.decode_tiled(sample).detach()\n",
        "Image.fromarray(np.array(decoded*255, dtype=np.uint8)[0])"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}