Diffusers代码学习-ControlNet(SDXL）_sdxl diffusers controlnet

目前没有太多的ControlNet模型与Stable Diffusion XL（SDXL）兼容，这里为SDXL训练了两个全尺寸ControlNet模型，这些模型以Canny边缘检测和深度图为条件进行推理。

# 以下代码为程序运行进行设置

import os

os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"

# 以下代码引入与SDXL兼容的ControlNet管道和模型

from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL

from diffusers.utils import load_image, make_image_grid
from PIL import Image
import cv2
import numpy as np
import torch

# 以下代码加载初始图像，并通过Canny边缘检测生成控制图片

original_image = load_image(
    "https://hf-mirror.com/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png"
)

image = np.array(original_image)

low_threshold = 100
high_threshold = 200

image = cv2.Canny(image, low_threshold, high_threshold)
image = image[:, :, None]
image = np.concatenate([image, image, image], axis=2)
canny_image = Image.fromarray(image)
make_image_grid([original_image, canny_image], rows=1, cols=2)

# 以下代码加载与SDXL兼容的ControlNet模型

controlnet = ControlNetModel.from_pretrained(
"diffusers/controlnet-canny-sdxl-1.0",
torch_dtype=torch.float16,
use_safetensors=True
)
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16, use_safetensors=True)
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
"stabilityai/stable-diffusion-xl-base-1.0",
controlnet=controlnet,
vae=vae,
torch_dtype=torch.float16,
use_safetensors=True
)
pipe.enable_model_cpu_offload()

# 以下代码由正负提示词及边缘检测形成的控制图片生成图片

prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting"
negative_prompt = 'low quality, bad quality, sketches'

image = pipe(
    prompt,
    negative_prompt=negative_prompt,
    image=canny_image,
    controlnet_conditioning_scale=0.5,
).images[0]

image.show()

以下为原始图像

以下为生成的图片（有些酷炫的效果）