StreamDiffusion_colab.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/StreamDiffusion-colab/blob/main/StreamDiffusion_colab.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VjYy0F2gZIPR"
      },
      "outputs": [],
      "source": [
        "!pip install -q git+https://github.com./cumulo-autumn/StreamDiffusion.git@main#egg=streamdiffusion\n",
        "!python -m streamdiffusion.tools.install-tensorrt\n",
        "!pip install -q https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl accelerate\n",
        "\n",
        "import torch\n",
        "from diffusers import AutoencoderTiny, StableDiffusionPipeline\n",
        "\n",
        "from streamdiffusion import StreamDiffusion\n",
        "from streamdiffusion.image_utils import postprocess_image\n",
        "\n",
        "pipe = StableDiffusionPipeline.from_pretrained(\"KBlueLeaf/kohaku-v2.1\").to(\n",
        "    device=torch.device(\"cuda\"),\n",
        "    dtype=torch.float16,\n",
        ")\n",
        "\n",
        "# Wrap the pipeline in StreamDiffusion\n",
        "# Requires more long steps (len(t_index_list)) in text2image\n",
        "# You should use cfg_type=\"none\" when text2image\n",
        "stream = StreamDiffusion(\n",
        "    pipe,\n",
        "    t_index_list=[0, 16, 32, 45],\n",
        "    torch_dtype=torch.float16,\n",
        "    cfg_type=\"none\",\n",
        ")\n",
        "\n",
        "# If the loaded model is not LCM, merge LCM\n",
        "stream.load_lcm_lora()\n",
        "stream.fuse_lora()\n",
        "# Use Tiny VAE for further acceleration\n",
        "stream.vae = AutoencoderTiny.from_pretrained(\"madebyollin/taesd\").to(device=pipe.device, dtype=pipe.dtype)\n",
        "# Enable acceleration\n",
        "pipe.enable_xformers_memory_efficient_attention()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "prompt = \"1girl with dog hair, thick frame glasses\"\n",
        "# Prepare the stream\n",
        "stream.prepare(prompt)\n",
        "\n",
        "# Warmup >= len(t_index_list) x frame_buffer_size\n",
        "for _ in range(4):\n",
        "    stream()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "x_output = stream.txt2img()\n",
        "image = postprocess_image(x_output, output_type=\"pil\")[0]\n",
        "image"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}