app.py

import spaces
import gradio as gr
import torch
from diffusers import StableDiffusionXLPipeline, UNet2DConditionModel 
from tdd_scheduler import TDDScheduler
from safetensors.torch import load_file
from PIL import Image
import transformers
transformers.utils.move_cache()

SAFETY_CHECKER = True

loaded_acc = None
device = "cuda"

ACC_lora={
    "TDD":"sdxl_tdd_wo_adv_lora.safetensors",
    "TDD_adv":"sdxl_tdd_lora_weights.safetensors",
}

if torch.cuda.is_available():
    base1 = UNet2DConditionModel.from_pretrained(
        "stabilityai/stable-diffusion-xl-base-1.0", subfolder="unet", torch_dtype=torch.float16
    ).to(device)
    base2 = UNet2DConditionModel.from_pretrained(
        "frankjoshua/realvisxlV40_v40Bakedvae", subfolder="unet", torch_dtype=torch.float16
    ).to(device)
    pipe_sdxl = StableDiffusionXLPipeline.from_pretrained(
        "stabilityai/stable-diffusion-xl-base-1.0",
        unet=base1,
        torch_dtype=torch.float16,
        variant="fp16",
    ).to(device)

    pipe_sdxl.load_lora_weights("RED-AIGC/TDD", weight_name=ACC_lora["TDD"], adapter_name="TDD")
    pipe_sdxl.load_lora_weights("RED-AIGC/TDD", weight_name=ACC_lora["TDD_adv"], adapter_name="TDD_adv")
    pipe_sdxl.scheduler = TDDScheduler.from_config(pipe_sdxl.scheduler.config)

    pipe_sdxl_real = StableDiffusionXLPipeline.from_pretrained(
        "stabilityai/stable-diffusion-xl-base-1.0",
        unet=base2,
        torch_dtype=torch.float16,
        variant="fp16",
    ).to(device)
    pipe_sdxl_real.load_lora_weights("RED-AIGC/TDD", weight_name=ACC_lora["TDD"], adapter_name="TDD")
    pipe_sdxl_real.load_lora_weights("RED-AIGC/TDD", weight_name=ACC_lora["TDD_adv"], adapter_name="TDD_adv")
    pipe_sdxl_real.scheduler = TDDScheduler.from_config(pipe_sdxl.scheduler.config)

def update_base_model(ckpt):
    if torch.cuda.is_available():
        pipe_sdxl = StableDiffusionXLPipeline.from_pretrained(
            "stabilityai/stable-diffusion-xl-base-1.0",
            torch_dtype=torch.float16,
            variant="fp16",
        ).to(device)
    return pipe_sdxl


if SAFETY_CHECKER:
    from safety_checker import StableDiffusionSafetyChecker
    from transformers import CLIPFeatureExtractor

    safety_checker = StableDiffusionSafetyChecker.from_pretrained(
        "CompVis/stable-diffusion-safety-checker"
    ).to(device)
    feature_extractor = CLIPFeatureExtractor.from_pretrained(
        "openai/clip-vit-base-patch32"
    )

    def check_nsfw_images(
        images: list[Image.Image],
    ) -> tuple[list[Image.Image], list[bool]]:
        safety_checker_input = feature_extractor(images, return_tensors="pt").to(device)
        has_nsfw_concepts = safety_checker(
            images=[images], clip_input=safety_checker_input.pixel_values.to(device)
        )

        return images, has_nsfw_concepts


#@spaces.GPU(enable_queue=True, duration=5)
@spaces.GPU(enable_queue=True)
def generate_image(
    prompt,
    negative_prompt,
    ckpt,
    acc,
    num_inference_steps,
    guidance_scale,
    eta,
    seed,
    progress=gr.Progress(track_tqdm=True),
):
    global loaded_acc
    #pipe = pipe_sdxl #if mode == "sdxl" else pipe_sd15

    if ckpt == "Real":
        pipe = pipe_sdxl_real
    else:
        pipe = pipe_sdxl

    if loaded_acc != acc:
        #pipe.load_lora_weights(ACC_lora[acc], adapter_name=acc)
        pipe.set_adapters([acc], adapter_weights=[1.0])
        print(pipe.get_active_adapters())
        loaded_acc = acc

    results = pipe(
        prompt=prompt,
        negative_prompt=negative_prompt,
        num_inference_steps=num_inference_steps,
        guidance_scale=guidance_scale,
        eta=eta,
        generator=torch.Generator(device=pipe.device).manual_seed(seed),
    )

    if SAFETY_CHECKER:
        images, has_nsfw_concepts = check_nsfw_images(results.images)
        if any(has_nsfw_concepts):
            gr.Warning("NSFW content detected.")
            return Image.new("RGB", (512, 512))
        return images[0]
    return results.images[0]

css = """
h1 {
    text-align: center;
    display:block;
}
.gradio-container {
  max-width: 70.5rem !important;
}
"""

with gr.Blocks(css=css) as demo:
    gr.Markdown(
        """
# ✨Target-Driven Distillation✨

Target-Driven Distillation (TDD) is a state-of-the-art consistency distillation model that largely accelerates the inference processes of diffusion models. Using its delicate strategies of *target timestep selection* and *decoupled guidance*, models distilled by TDD can generated highly detailed images with only a few steps.

[**Project Page**](https://redaigc.github.io/TDD/) **|** [**Paper**](https://arxiv.org/abs/2409.01347) **|** [**Code**](https://github.com/RedAIGC/Target-Driven-Distillation) **|** [**Model**](https://huggingface.co/RED-AIGC/TDD) **|** [🤗 **TDD-SDXL Demo**](https://huggingface.co/spaces/RED-AIGC/TDD) **|** [🤗 **TDD-SVD Demo**](https://huggingface.co/spaces/RED-AIGC/SVD-TDD)
"""
    )
    with gr.Row():
        with gr.Column(scale=1):
            with gr.Group():
                with gr.Row():
                    prompt = gr.Textbox(label="Prompt")
                with gr.Row():
                    negative_prompt = gr.Textbox(label="Negative Prompt")
                with gr.Row():
                    steps = gr.Slider(
                        label="Sampling Steps",
                        minimum=4,
                        maximum=8,
                        step=1,
                        value=4,
                    )
                with gr.Row():
                    guidance_scale = gr.Slider(
                        label="CFG Scale",
                        minimum=1,
                        maximum=4,
                        step=0.1,
                        value=2.0,
                    )
                with gr.Row():
                    eta = gr.Slider(
                        label="eta",
                        minimum=0,
                        maximum=0.3,
                        step=0.01,
                        value=0.2,
                    )
                with gr.Row():
                    seed = gr.Number(label="Seed", value=-1)

                with gr.Row():

                    ckpt = gr.Dropdown(
                        label="Base Model",
                        choices=["SDXL-1.0", "Real"],
                        value="SDXL-1.0",
                    )

                    acc = gr.Dropdown(
                        label="Accelerate Lora",
                        choices=["TDD", "TDD_adv"],
                        value="TDD_adv",
                    )

        with gr.Column(scale=1):
            with gr.Group():
                img = gr.Image(label="TDD Image", value="cat.png")
                submit_sdxl = gr.Button("Run on SDXL")
    gr.Examples(
        examples=[
            ["A photo of a cat made of water.", "", "SDXL-1.0", "TDD_adv", 4, 1.7, 0.2, 546237],
            ["Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", "NSFW, low quality, low resolution, normal quality", "Real", "TDD_adv", 7, 2.0, 0.3, 68436],
            ["panda mad scientist mixing sparkling chemicals", "", "Real", "TDD_adv", 5, 1.7, 0.2, 3853],
            ["a dog in snow, 8k", "", "SDXL-1.0", "TDD", 6, 1.3, 0.2, 12138]
        ],
        inputs=[prompt, negative_prompt, ckpt, acc, steps, guidance_scale, eta, seed],
        outputs=[img],
        fn=generate_image,
        cache_examples="lazy",
    )

    gr.on(
        fn=generate_image,
        triggers=[ckpt.change, prompt.submit, submit_sdxl.click],
        inputs=[prompt, negative_prompt, ckpt, acc, steps, guidance_scale, eta, seed],
        outputs=[img],
    )

demo.queue(api_open=False).launch(show_api=False)