1<!doctype html> 2<!--suppress HtmlRequiredAltAttribute --> 3<html lang="en"> 4<head> 5 <!-- Google tag (gtag.js) --> 6
6<script> 7
vendor: 154 bytes, lines 7-10
7window.dataLayer = window.dataLayer || []; 8 function gtag() { dataLayer.push(arguments); } 9 gtag('js', new Date()); 10 gtag('config', '
10G-F6S751BVJ5
vendor: 8 bytes, lines 10-11
10'); 11
11</script>
11 12 <!-- <meta charset="utf-8"> --> 13 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> 14 <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> 15 <meta name="description" content=""> 16 <meta name="author" content=""> 17 <link rel="icon" href="https://deepimagination.cc/eDiff-I/images/favicon.ico"> 18 <title>eDiff-I: Text-to-Image Diffusion Models with Ensemble of Expert Denoisers</title> 19 <link rel="canonical" href="https://deepimagination.cc/eDiff-I/" /> 20 <!-- Bootstrap core CSS --> 21 <link rel="stylesheet" href="../bootstrap/4.0.0/css/bootstrap.min.css" > 22 <!-- Custom styles for this template --> 23 <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Source+Sans+Pro"> 24 <link rel="dns-prefetch" href="https://cdn.jsdelivr.net/"/> 25 <link href="../npm/prismjs/themes/prism.min.css" rel="stylesheet"/> 26 <link href="../gh/jablonczay/code-box-copy/code-box-copy/css/code-box-copy.min.css" rel="stylesheet"/> 27 <style> 28 body { 29 font-family: 'Source Sans Pro', sans-serif; 30 padding-bottom: 50px; 31 } 32 hr { 33 background: #80808083; 34 } 35 .grid-container { 36 display: grid; 37 grid-template-columns: 25% 25% 25% 25%; 38 padding: 2px; 39 } 40 .grid-container-video { 41 display: grid; 42 grid-template-columns: 100%; 43 padding: 2px; 44 } 45 .grid-item { 46 padding: 2px; 47 font-size: 30px; 48 text-align: center; 49 align-items: center; 50 justify-content: center; 51 display: flex; 52 } 53 /*input[type="checkbox"] {*/ 54 /* display: none;*/ 55 /*}*/ 56 img { 57 -webkit-filter: grayscale(0%); 58 filter: grayscale(0%); 59 } 60 label { 61 border: 1px solid #000; 62 padding: 0; 63 display: inline-block; 64 position: relative; 65 margin: 0; 66 cursor: pointer; 67 -webkit-touch-callout: none; 68 -webkit-user-select: none; 69 -khtml-user-select: none; 70 -moz-user-select: none; 71 -ms-user-select: none; 72 user-select: none; 73 } 74 .embed-responsive-2by1 { 75 padding-bottom: 50%; 76 } 77 .embed-responsive-4by1 { 78 padding-bottom: 25%; 79 } 80 .embed-responsive-6by1 { 81 padding-bottom: 16.67%; 82 } 83 .embed-responsive-8by1 { 84 padding-bottom: 12.50%; 85 } 86 .embed-responsive-teaser { 87 padding-bottom: 29%; 88 } 89 .float-button { 90 position: fixed; 91 right: 1%; 92 z-index: 9999; 93 } 94 .vert { 95 transform-origin: 50% 50%; 96 transform: rotate(180deg); 97 writing-mode: vertical-rl; 98 margin: 0; 99 } 100 .caption { 101 font-size: 0.9rem; 102 margin-top: 0; 103 margin-bottom: 5px; 104 font-weight: bold; 105 } 106 /*.row {*/ 107 /* display: grid;*/ 108 /* grid-auto-flow: row;*/ 109 /* gap: 5%;*/ 110 /*}*/ 111 .carousel .carousel-indicators li { 112 width: 10px; 113 height: 10px; 114 border-radius: 100%; 115 } 116 .carousel .carousel-indicators { 117 visibility: hidden; 118 } 119 .carousel:hover .carousel-indicators { 120 visibility: visible; 121 } 122 .w-85 { 123 width: 85% !important; 124 } 125 /*#videoGallery { 126 margin-left: 50px; 127 margin-right: 50px; 128 }*/ 129 .carousel-control-prev { 130 width: 7%; 131 filter: invert(100%); 132 } 133 .carousel-control-next { 134 width: 7%; 135 filter: invert(100%); 136 } 137 </style> 138</head> 139<body> 140 <main role="main"> 141 <section class="jumbotron text-center" style="padding: 2%; padding-bottom: 1%; background-color: #e6e9ec;"> 142 <div class="container"> 143 <h2 class="jumbotron-heading">eDiff-I: Text-to-Image Diffusion Models with Ensemble of Expert Denoisers 144 </h2> 145 </div> 146 <div class="container" style="max-width: 840px;"> 147 <div class="row"> 148 <div class="col-md"> 149 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="http://yogeshbalaji.github.io/" target="_blank">Yogesh Balaji<span style="color: #76b900;"></span></a></h5> 150 </div> 151 <div class="col-md"> 152 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://seungjunnah.github.io/" target="_blank">Seungjun Nah<span style="color: #76b900;"></span></a></h5> 153 </div> 154 <div class="col-md"> 155 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://xunhuang.me/" target="_blank">Xun Huang<span style="color: #76b900;"></span></a></h5> 156 </div> 157 <div class="col-md"> 158 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="http://latentspace.cc/" target="_blank">Arash Vahdat<span style="color: #76b900;"></span></a></h5> 159 </div> 160 <div class="col-md"> 161 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://tsong.me/" target="_blank">Jiaming Song<span style="color: #76b900;"></span></a></h5> 162 </div> 163 <div class="col-md"> 164 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://karstenkreis.github.io/" target="_blank">Karsten Kreis<span style="color: #76b900;"></span></a></h5> 165 </div> 166 </div> 167 <div class="row"> 168 <div class="col-md"> 169 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://people.csail.mit.edu/miika/" target="_blank">Miika Aittala<span style="color: #76b900;"></span></a></h5> 170 </div> 171 <div class="col-md"> 172 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://research.nvidia.com/person/timo-aila" target="_blank">Timo Aila<span style="color: #76b900;"></span></a></h5> 173 </div> 174 <div class="col-md"> 175 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://users.aalto.fi/~laines9/" target="_blank">Samuli Laine<span style="color: #76b900;"></span></a></h5> 176 </div> 177 <div class="col-md"> 178 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://ctnzr.io/" target="_blank">Bryan Catanzaro<span style="color: #76b900;"></span></a></h5> 179 </div> 180 <div class="col-md"> 181 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://research.nvidia.com/person/tero-karras" target="_blank">Tero Karras<span style="color: #76b900;"></span></a></h5> 182 </div> 183 <div class="col-md"> 184 <h6 class="text-center" style="margin: 0;"><a class="text-center" href="http://mingyuliu.net/" target="_blank">Ming-Yu Liu<span style="color: #76b900;"></span></a></h5> 185 </div> 186 </div> 187 <div class="row" style="margin-top:0.5rem; margin-left: auto; margin-right: auto;"> 188 <div class="col-md"> 189 <h5>NVIDIA Corporation</h5> 190 </div> 191 </div> 192 </div> 193 <div class="buttons"> 194 <a href="../pdf/2211.01324.pdf" target="_blank" class="btn btn-primary my-2">Paper 195 (arxiv)</a> 196 <a href="https://www.youtube.com/watch?v=WbaVvlgxbl4" target="_blank" class="btn btn-primary my-2">Video</a> 197 </div> 198 <div class="container" style="max-width: 900px;"> 199 <div class="row"> 200 <div class="col-md"> 201 <h5>TL;DR: eDiff-I is a new generation of generative AI content 202 creation tool that offers unprecedented text-to-image synthesis with instant style transfer 203 and intuitive painting with words capabilities.</h5> 204 </div> 205 </div> 206 <div class="row"> 207 <div class="col-md"> 208 <p> 209 We propose eDiff-I, a diffusion model for synthesizing images given text. Motivated by the 210 empirical observation that the behavior of diffusion models differ at different stages of 211 sampling, we propose to train an ensemble of expert denoising networks, each specializing 212 for a specific noise interval. Our model is conditioned on the T5 text embeddings, CLIP 213 image embeddings and CLIP text embeddings. Our approach can generate photorealistic images 214 correponding to any input text prompt. In addition to text-to-image synthesis, we present 215 two additional capabilies - (1) style transfer, which enables us to control the style of the 216 generated sample using a reference style image, and (2) "Paint with words" - an application 217 where the user can generate images by painting segmentation maps on canvas, which is very 218 handy for crafting the desired image in mind. 219 </p> 220 </div> 221 </div> 222 <div class="row"> 223 <div class="column" style="margin: auto;width: 33%;"> 224 <img id="teaser_portal" src="../eDiff-I/images/teaser/portal.jpg" style="margin: 8px;width: 100%;"> 225 </div> 226 <div class="column" style="margin: auto;width: 33%;"> 227 <img id="teaser_cat" src="../eDiff-I/images/teaser/cat_witch.jpg" style="margin: 8px;width: 100%;"> 228 </div> 229 <div class="column" style="margin: auto;width: 33%;"> 230 <img id="teaser_landscape" src="../eDiff-I/images/teaser/landscape.jpg" style="margin: 8px;width: 100%;"> 231 </div> 232 </div> 233 <div class="row"> 234 <div class="col-md" style="margin: auto;width: 33%;"> 235 <p class="caption text-center"><i>A highly detailed digital painting of a portal in a mystic 236 forest with many beautiful trees. A person is standing in front of the portal</i></p> 237 </div> 238 <div class="col-md" style="margin: auto;width: 33%;"> 239 <p class="caption text-center"><i>A highly detailed zoomed-in digital painting of a cat dressed 240 as a witch wearing a wizard hat in a haunted house, artstation</i></p> 241 </div> 242 <div class="col-md" style="margin: auto;width: 33%;"> 243 <p class="caption text-center"><i>An image of a beautiful landscape of an ocean. There is a huge 244 rock in the middle of the ocean. There is a mountain in the background. Sun is 245 setting.</i></p> 246 </div> 247 </div> 248 <div class="row"> 249 <div class="column" style="margin-right:4px; margin-left:8px; width: 15%"> 250 <img id="teaser" src="../eDiff-I/images/teaser/animal_scrape.jpg" style="width: 100%;"> 251 <p>Style Reference</p> 252 <p class="caption text-center"><i>A photo of a duckling wearing a medieval soldier helmet and 253 riding a skateboard.</i></p> 254 </div> 255 <div class="column" style="margin-left:0px;width: 30%;"> 256 <img id="teaser" src="../eDiff-I/images/teaser/duck.jpg" style="width: 100%;"> 257 </div> 258 <div class="column" style="margin-left: 20px;width: 50%;"> 259 <div class="row"> 260 <img id="teaser" src="../eDiff-I/images/paint_with_words/aurora_text.jpg" style="margin: 2px; width: 48%;"> 261 <img id="teaser" src="../eDiff-I/images/paint_with_words/aurora_out.jpg" style="margin: 2px;width: 48%;"> 262 </div> 263 <div class="col-md" style="margin: auto;width: 100%;"> 264 <p class="caption text-center"><i>
264A digital painting of a half-frozen lake near mountains 265 under a full 266 moon and aurora. A boat is in the middle of the lake. Highly detailed.</i></p> 267 </div> 268 </div> 269 </div> 270 </section> 271 </main> 272 <!-- Summary video --> 273 <div class="container" style="max-width: 768px;"> 274 <div class="grid-container-video" style="width: 90%; margin-left: auto; margin-right: auto"> 275 <div class="grid-item"> 276 <div> 277 <h5>Video</h5> 278 </div> 279 </div> 280 <div class="embed-responsive embed-responsive-16by9" style="margin: 0;"> 281 <iframe class="embed-responsive-item" src="https://www.youtube.com/embed/WbaVvlgxbl4" allowfullscreen></iframe> 282 </div> 283 </div> 284 <div class="row"> 285 <div class="col-md-12"> 286 <hr style="max-width: 768px;"> 287 <h5 class="text-center">Pipeline</h5> 288 <p> 289 Our pipeline consists of a cascade of three diffusion models - a base model which can synthesize 290 samples of 64x64 resolution, and two super-resolution stacks that can upsample the images 291 progressively to 256x256 and 1024x1024 resolution respectively. Our models take an input caption 292 and first compute T5 XXL embedding and text embedding. We optionally use CLIP image encodings 293 which are computed from a reference image. These image embeddings can serve as a style vector. 294 These embeddings are then fed into our cascaded diffusion models which progressively generate 295 images of resolution 1024x1024. 296 </p> 297 <div class="row justify-content-center"> 298 <img src="../eDiff-I/images/pipeline/model_arch2.jpg" style="height: 500px;margin-bottom: 8px;"> 299 </div> 300 <p class="caption text-center">Pipeline</p> 301 </div> 302 </div> 303 <div class="row"> 304 <div class="col-md-12"> 305 <hr style="max-width: 768px;"> 306 <h5 class="text-center">Denoising experts</h5> 307 <p> 308 In diffusion models, image synthesis happens via an iterative denoising process that gradually 309 generates images from random noise. In the figure shown below, we start from a complete random 310 noise which is then gradually denoised in multiple steps to finally produce an image of a panda 311 riding a bike. In conventional diffusion model training, a single model is trained to denoise 312 the whole noise distribution. In our framework, we instead train an ensemble of expert denoisers 313 that are specialized for denoising in different intervals of the generative process. The use of 314 such expert denoisers leads to improved synthesis capabilities. 315 </p> 316 <div class="row justify-content-center"> 317 <img src="../eDiff-I/images/pipeline/motivation2_fig.jpg" style="height: 500px;margin-bottom: 8px;"> 318 </div> 319 <p class="caption text-center">Pipeline</p> 320 </div> 321 </div> 322 <!-- Outputs. --> 323 <hr style="max-width: 768px;"> 324 <div class="container" style="max-width: 768px;"> 325 <div class="text-center"> 326 <h5>Results</h5> 327 </div> 328 <div class="row"> 329 <div class="col-md-12"> 330 <p> Compared to the open-source text to image methods (<a href="https://huggingface.co/spaces/stabilityai/stable-diffusion" target="_blank">Stable 331 diffusion</a>) and (<a href="../pdf/2204.06125.pdf" target="_blank">DALL-E2</a>), our model conistently leads to improved synthesis quality. 332 </p> 333 </div> 334 </div> 335 <div class="row"> 336 <div class="column" style="margin: auto;width: 33%;"> 337 <b> 338 <p class="caption text-center"> Stable diffusion </p> 339 </b> 340 </div> 341 <div class="column" style="margin: auto;width: 33%;"> 342 <p class="caption text-center"> DALL-E2 </p> 343 </div> 344 <div class="column" style="margin: auto;width: 33%;"> 345 <p class="caption text-center"> eDiff-I (ours) </p> 346 </div> 347 </div> 348 <div class="row"> 349 <div class="column" style="margin: auto;width: 33%;"> 350 <img id="comparison_stable_teapot" src="../eDiff-I/images/comparison_multiple_entitles/stable_diffusion/stable_teapots.jpg" style="margin: 8px;width: 100%;"> 351 </div> 352 <div class="column" style="margin: auto;width: 33%;"> 353 <img id="comparison_dalle_teapot" src="../eDiff-I/images/comparison_multiple_entitles/dalle2/dalle_teapots.png" style="margin: 8px;width: 100%;"> 354 </div> 355 <div class="column" style="margin: auto;width: 33%;"> 356 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/comparison_multiple_entitles/ediffi/ours_teapots.jpg" style="margin: 8px;width: 100%;"> 357 </div> 358 </div> 359 <p class="caption text-center"><i>There are two Chinese teapots on a table. One pot has a painting of a 360 dragon, 361 while the other pot has a painting of a panda.</i></p> 362 <div class="row"> 363 <div class="column" style="margin: auto;width: 33%;"> 364 <img id="comparison_stable_grizzly" src="../eDiff-I/images/comparison_multiple_entitles/stable_diffusion/stable_grizzly.jpg" style="margin: 8px;width: 100%;"> 365 </div> 366 <div class="column" style="margin: auto;width: 33%;"> 367 <img id="comparison_dalle_grizzly" src="../eDiff-I/images/comparison_multiple_entitles/dalle2/dalle_grizzly.png" style="margin: 8px;width: 100%;"> 368 </div> 369 <div class="column" style="margin: auto;width: 33%;"> 370 <img id="comparison_eDiff-I_grizzly" src="../eDiff-I/images/comparison_multiple_entitles/ediffi/ours_grizzly.jpg" style="margin: 8px;width: 100%;"> 371 </div> 372 </div> 373 <p class="caption text-center"><i>A photo of two cute teddy bears sitting on top of a grizzly bear in a 374 beautiful forest. Highly detailed fantasy art, 4k, artstation</i></p> 375 <div class="row"> 376 <div class="column" style="margin: auto;width: 33%;"> 377 <img id="text_stable_nvidia" src="../eDiff-I/images/comparison_text/stable_diffusion/stable_diffusion_nvidia_rocks.jpg" style="margin: 8px;width: 100%;"> 378 </div> 379 <div class="column" style="margin: auto;width: 33%;"> 380 <img id="comparison_dalle_nvidia" src="../eDiff-I/images/comparison_text/dalle2/dalle_nvidia_rocks.png" style="margin: 8px;width: 100%;"> 381 </div> 382 <div class="column" style="margin: auto;width: 33%;"> 383 <img id="comparison_eDiff-I_nvidia" src="../eDiff-I/images/comparison_text/ediffi/ours_nvidia_rocks.jpg" style="margin: 8px;width: 100%;"> 384 </div> 385 </div> 386 <p class="caption text-center"><i>A photo of a golden retriever puppy wearing a green shirt. The shirt 387 has text that says âNVIDIA rocksâ. Backgroun
387d office. 4k dslr</i></p> 388 <div class="row"> 389 <div class="column" style="margin: auto;width: 33%;"> 390 <img id="text_stable_monkey_friends" src="../eDiff-I/images/comparison_text/stable_diffusion/stable_diffusion_two_monkeys.jpg" style="margin: 8px;width: 100%;"> 391 </div> 392 <div class="column" style="margin: auto;width: 33%;"> 393 <img id="comparison_dalle_monkey_friends" src="../eDiff-I/images/comparison_text/dalle2/dalle_best_friends.png" style="margin: 8px;width: 100%;"> 394 </div> 395 <div class="column" style="margin: auto;width: 33%;"> 396 <img id="comparison_eDiff-I_monkey_friends" src="../eDiff-I/images/comparison_text/ediffi/ours_two_monkeys.jpg" style="margin: 8px;width: 100%;"> 397 </div> 398 </div> 399 <p class="caption text-center"><i>A photo of two monkeys sitting on a tree. They are holding a wooden 400 board that says âBest friendsâ, 4K dslr.</i></p> 401 <div class="row"> 402 <div class="column" style="margin: auto;width: 33%;"> 403 <img id="comparison_stable_dish" src="../eDiff-I/images/comparison_long/stable_diffusion/stable_restaurant.jpg" style="margin: 8px;width: 100%;"> 404 </div> 405 <div class="column" style="margin: auto;width: 33%;"> 406 <img id="comparison_dalle_dish" src="../eDiff-I/images/comparison_long/dalle2/dalle_restaurant.png" style="margin: 8px;width: 100%;"> 407 </div> 408 <div class="column" style="margin: auto;width: 33%;"> 409 <img id="comparison_eDiff-I_dish" src="../eDiff-I/images/comparison_long/ediffi/ours_restaurant.jpg" style="margin: 8px;width: 100%;"> 410 </div> 411 </div> 412 <p class="caption text-center"><i>A photo of a plate at a restaurant table with spaghetti and red sauce. 413 There is sushi on top of the spaghetti. The dish is garnished with mint leaves. On the side, 414 there is a glass with a purple drink, photorealistic, dslr.</i></p> 415 <div class="row"> 416 <div class="column" style="margin: auto;width: 33%;"> 417 <img id="comparison_stable_cat_scooter" src="../eDiff-I/images/comparison_long/stable_diffusion/stable_cat_scooter.jpg" style="margin: 8px;width: 100%;"> 418 </div> 419 <div class="column" style="margin: auto;width: 33%;"> 420 <img id="comparison_dalle_cat_scooter" src="../eDiff-I/images/comparison_long/dalle2/dalle_cat_scooter.png" style="margin: 8px;width: 100%;"> 421 </div> 422 <div class="column" style="margin: auto;width: 33%;"> 423 <img id="comparison_eDiff-I_cat_scooter" src="../eDiff-I/images/comparison_long/ediffi/ours_cat_scooter.jpg" style="margin: 8px;width: 100%;"> 424 </div> 425 </div> 426 <p class="caption text-center"><i>A close-up 4k dslr photo of a cat riding a scooter. It is wearing a 427 plain shirt and has a bandana around its neck. It is wearing a scooter helmet. There are palm 428 trees in the background.</i></p> 429 </div> 430 <!-- Style transfer. --> 431 <hr style="max-width: 768px;"> 432 <div class="container" style="max-width: 768px;"> 433 <div class="text-center"> 434 <h5>Style transfer</h5> 435 </div> 436 <div class="row"> 437 <div class="col-md-12"> 438 <p> Our method enables style transfer when the CLIP image embeddings are used. From a reference 439 style image, we first extract the CLIP image emebddings which can be used as a style 440 reference vector. The left panel in the figure shown below is a style reference. The middle 441 panel shows the results when style conditioning is enabled. The panel on the right shows the 442 results when style conditioning is disabled. When style conditioning is used, our model 443 generates outputs that are faithful both to the input style and also the input caption. When 444 style conditioning is disabled, we generate images in natural style. 445 </p> 446 </div> 447 </div> 448 <div class="row"> 449 <div class="column" style="margin: auto;width: 33%;"> 450 <b> 451 <p class="caption text-center"> Reference style </p> 452 </b> 453 </div> 454 <div class="column" style="margin: auto;width: 33%;"> 455 <p class="caption text-center"> Style conditioning enabled </p> 456 </div> 457 <div class="column" style="margin: auto;width: 33%;"> 458 <p class="caption text-center"> Style conditioning disabled </p> 459 </div> 460 </div> 461 <div class="row"> 462 <div class="column" style="margin: auto;width: 33%;"> 463 <img id="comparison_style_two_pandas_reference" src="../eDiff-I/images/clip_img_conditioning/two_pandas_reference.jpg" style="margin: 8px;width: 100%;"> 464 </div> 465 <div class="column" style="margin: auto;width: 33%;"> 466 <img id="comparison_style_two_pandas_style" src="../eDiff-I/images/clip_img_conditioning/two_pandas_conditioned.jpg" style="margin: 8px;width: 100%;"> 467 </div> 468 <div class="column" style="margin: auto;width: 33%;"> 469 <img id="comparison_style_two_pandas_nostyle" src="../eDiff-I/images/clip_img_conditioning/two_pandas.jpg" style="margin: 8px;width: 100%;"> 470 </div> 471 </div> 472 <p class="caption text-center"><i>A photo of two pandas walking on a road.</i></p> 473 <div class="row"> 474 <div class="column" style="margin: auto;width: 33%;"> 475 <img id="comparison_style_rabbit_reference" src="../eDiff-I/images/clip_img_conditioning/rabbit_queen_reference.jpg" style="margin: 8px;width: 100%;"> 476 </div> 477 <div class="column" style="margin: auto;width: 33%;"> 478 <img id="comparison_style_rabbit_style" src="../eDiff-I/images/clip_img_conditioning/rabbit_queen_conditioned.jpg" style="margin: 8px;width: 100%;"> 479 </div> 480 <div class="column" style="margin: auto;width: 33%;"> 481 <img id="comparison_style_rabbit_nostyle" src="../eDiff-I/images/clip_img_conditioning/rabbit_queen.jpg" style="margin: 8px;width: 100%;"> 482 </div> 483 </div> 484 <p class="caption text-center"><i>A detailed oil painting of a beautiful rabbit queen wearing a royal 485 gown in a palace. She is looking outside the window, artistic.</i></p> 486 <div class="row"> 487 <div class="column" style="margin: auto;width: 33%;"> 488 <img id="comparison_style_rabbit_reference" src="../eDiff-I/images/clip_img_conditioning/dog_trumpet_reference.jpg" style="margin: 8px;width: 100%;"> 489 </div> 490 <div class="column" style="margin: auto;width: 33%;"> 491 <img id="comparison_style_rabbit_style" src="../eDiff-I/images/clip_img_conditioning/dog_trumpet_conditioned.jpg" style="margin: 8px;width: 100%;"> 492 </div> 493 <div class="column" style="margin: auto;width: 33%;"> 494 <img id="comparison_style_rabbit_nostyle" src="../eDiff-I/images/clip_img_conditioning/dog_trumpet.jpg" style="margin: 8px;width: 100%;"> 495 </div> 496 </div> 497 <p class="caption text-center"><i>A dslr photo of a dog playing trumpet from the top of a mountain.</i> 498 </p> 499 <div class="row"> 500 <div class="column" style="margin: auto;width: 33%;"> 501 <img id="comparison_style_teddy_bear_reference" src="../eDiff-I/images/clip_img_conditioning/teddy_bear_reference.jpg" style="margin: 8px;width: 100%;"> 502 </div> 503 <div class="column" style="margin: auto;width: 33%;"> 504 <img id="comparison_style_teddy_bear_style" src="../eDiff-I/images/clip_img_conditioning/teddy_bear_conditioned.jpg" style="margin: 8px;width: 100%;"> 505 </div> 506 <div class="column" style="margin: auto;width: 33%;"> 507 <img id="comparison_style_teddy_bear_nostyle" src="../eDiff-I/images/clip_img_conditioning/teddy_bear.jpg" style="margin: 8px;width: 100%;"> 508 </div> 509 </div> 510 <p class="caption text-center"><i>A photo of a teddy bear wearing a casual plain white shirt surfing in 511 the ocean.</i> 512 </p> 513 </div> 514 <!-- Paint with words. --> 515 <hr style="max-width: 768px;"> 516 <div class="container" style="max-width: 768px;"> 517 <div class="text-center"> 518 <h5>Paint with words</h5> 519 </div> 520 <div class="row"> 521 <div class="col-md-12"> 522 <p> Our method allows users to control the location of objects mentioned in the text prompt by 523 selecting phrases and scribbling them on the image. The model then makes use of the prompt 524 along with the maps to generate images that can consistent both with the caption and the 525 input map. 526 </p> 527 </div> 528 </div> 529 <div class="row"> 530 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 531 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/portal.mp4" type="video/mp4"> 532 </video> 533 </div> 534 <div class="row"> 535 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 536 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/halloween.mp4" type="video/mp4"> 537 </video> 538 </div> 539 <div class="row"> 540 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 541 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/aurora.mp4" type="video/mp4"> 542 </video> 543 </div> 544 <div class="row"> 545 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 546 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/monster.mp4" type="video/mp4"> 547 </video> 548 </div> 549 <div class="row"> 550 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 551 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/rabbit_mage.mp4" type="video/mp4"> 552 </video> 553 </div> 554 <div class="row"> 555 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 556 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/rainbow.mp4" type="video/mp4"> 557 </video> 558 </div> 559 <div class="row"> 560 <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto"> 561 <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/squirrel.mp4" type="video/mp4"> 562 </video> 563 </div> 564 </div> 565 <!-- Effect of using denoising experts. --> 566 <hr style="max-width: 768px;"> 567 <div class="container" style="max-width: 768px;"> 568 <div class="text-center"> 569 <h5>Benefit of using denoising experts</h5> 570 </div> 571 <div class="row"> 572 <div class="col-md-12"> 573 <p> We illustrate the benefit of using denoising experts by visualizing samples generated by 574 our approach and comparing it with a baseline that does not use any denoising experts. 575 The left two panels on the figure below denotes a baseline not using any experts, while 576 the right two panels show the results obtaining by using our expert models. We find that 577 we can greately improve on faithfullness to the input text by using denoising experts. 578 </p> 579 </div> 580 </div> 581 <div class="row"> 582 <div class="column" style="margin: auto;width: 50%;"> 583 <b> 584 <p class="caption text-center"> Baseline </p> 585 </b> 586 </div> 587 <div class="column" style="margin: auto;width: 50%;"> 588 <p class="caption text-center"> eDiff-I </p> 589 </div> 590 </div> 591 <div class="row"> 592 <div class="column" style="margin: auto;width: 24%;"> 593 <img id="comparison_stable_teapot" src="../eDiff-I/images/model_size_comparison/base/teddy_bear/img_0.jpg" style="margin: 8px;width: 100%;"> 594 </div> 595 <div class="column" style="margin: auto;width: 24%;"> 596 <img id="comparison_dalle_teapot" src="../eDiff-I/images/model_size_comparison/base/teddy_bear/img_1.jpg" style="margin: 8px;width: 100%;"> 597 </div> 598 <div class="column" style="margin: auto;width: 24%;"> 599 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/teddy_bear/img_0.jpg" style="margin: 8px;width: 100%;"> 600 </div> 601 <div class="column" style="margin: auto;width: 24%;"> 602 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/teddy_bear/img_1.jpg" style="margin: 8px;width: 100%;"> 603 </div> 604 </div> 605 <p class="caption text-center"><i>A 4k dslr photo of two teddy bears wearing a sports jersey 606 with the text âeDiffiâ written on it. They are on a soccer field.</i></p> 607 <div class="row"> 608 <div class="column" style="margin: auto;width: 24%;"> 609 <img id="comparison_stable_teapot" src="../eDiff-I/images/model_size_comparison/base/monkey_bike/img_0.jpg" style="margin: 8px;width: 100%;"> 610 </div> 611 <div class="column" style="margin: auto;width: 24%;"> 612 <img id="comparison_dalle_teapot" src="../eDiff-I/images/model_size_comparison/base/monkey_bike/img_1.jpg" style="margin: 8px;width: 100%;"> 613 </div> 614 <div class="column" style="margin: auto;width: 24%;"> 615 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/monkey_bike/img_0.jpg" style="margin: 8px;width: 100%;"> 616 </div> 617 <div class="column" style="margin: auto;width: 24%;"> 618 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/monkey_bike/img_1.jpg" style="margin: 8px;width: 100%;"> 619 </div> 620 </div> 621 <p class="caption text-center"><i>An origami of a monkey dressed as a monk riding a bike on a 622 mountain.</i></p> 623 </div> 624 <!-- Comparison between T5 and CLIP embeddigs. --> 625 <hr style="max-width: 768px;"> 626 <div class="container" style="max-width: 768px;"> 627 <div class="text-center"> 628 <h5>Compairson between T5 and CLIP text embeddings</h5> 629 </div> 630 <div class="row"> 631 <div class="col-md-12"> 632 <p> We study the effect of using CLIP text embeddings and T5 embeddings in isolation, and 633 compare wi
633th the full system that uses CLIP + T5 embeddings. We observe that images 634 generated in the CLIP-text-only setting often contain correct foreground objects and 635 they tend to miss fine-grain details. Images generated in the T5-text-only setting are 636 of higher quality, but they sometimes contain incorrect objects. Using CLIP+T5 results 637 in best performance. 638 </p> 639 </div> 640 </div> 641 <div class="row"> 642 <div class="column" style="margin: auto;width: 33%;"> 643 <b> 644 <p class="caption text-center"> CLIP text only </p> 645 </b> 646 </div> 647 <div class="column" style="margin: auto;width: 33%;"> 648 <p class="caption text-center"> T5 only </p> 649 </div> 650 <div class="column" style="margin: auto;width: 33%;"> 651 <p class="caption text-center"> CLIP + T5 </p> 652 </div> 653 </div> 654 <div class="row"> 655 <div class="column" style="margin: auto;width: 33%;"> 656 <img id="comparison_stable_teapot" src="../eDiff-I/images/t5_vs_clip/corgi_text_clip.jpg" style="margin: 8px;width: 100%;"> 657 </div> 658 <div class="column" style="margin: auto;width: 33%;"> 659 <img id="comparison_dalle_teapot" src="../eDiff-I/images/t5_vs_clip/corgi_text_t5.jpg" style="margin: 8px;width: 100%;"> 660 </div> 661 <div class="column" style="margin: auto;width: 33%;"> 662 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/t5_vs_clip/corgi_text.jpg" style="margin: 8px;width: 100%;"> 663 </div> 664 </div> 665 <p class="caption text-center"><i>A photo of a cute corgi wearing a beret holding a sign 666 that 667 says 668 âDiffusion Modelsâ. There is Eiffel tower in the background.</i></p> 669 <div class="row"> 670 <div class="column" style="margin: auto;width: 33%;"> 671 <img id="comparison_stable_teapot" src="../eDiff-I/images/t5_vs_clip/parrots_singing_clip.jpg" style="margin: 8px;width: 100%;"> 672 </div> 673 <div class="column" style="margin: auto;width: 33%;"> 674 <img id="comparison_dalle_teapot" src="../eDiff-I/images/t5_vs_clip/parrots_singing_t5.jpg" style="margin: 8px;width: 100%;"> 675 </div> 676 <div class="column" style="margin: auto;width: 33%;"> 677 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/t5_vs_clip/parrots_singing.jpg" style="margin: 8px;width: 100%;"> 678 </div> 679 </div> 680 <p class="caption text-center"><i>A photo of a red parrot, a blue parrot and a green parrot singing 681 at a concert in front of a microphone. Colorful lights in the background.</i></p> 682 </div> 683 <!-- Style variations. --> 684 <hr style="max-width: 768px;"> 685 <div class="container" style="max-width: 768px;"> 686 <div class="text-center"> 687 <h5>Style Variations</h5> 688 </div> 689 <div class="row"> 690 <div class="col-md-12"> 691 <p> Our method can also generate images with different styles, which can be specified in the 692 input caption. 693 </p> 694 </div> 695 </div> 696 <p class="caption text-center"><i>A photo of penguin working as a fruit vendor in a tropical 697 village.</i></p> 698 <div class="row"> 699 <div class="column" style="margin: auto;width: 33%;"> 700 <img id="comparison_stable_teapot" src="../eDiff-I/images/style_variants/real.jpg" style="margin: 8px;width: 100%;"> 701 </div> 702 <div class="column" style="margin: auto;width: 33%;"> 703 <img id="comparison_dalle_teapot" src="../eDiff-I/images/style_variants/rembrandt.jpg" style="margin: 8px;width: 100%;"> 704 </div> 705 <div class="column" style="margin: auto;width: 33%;"> 706 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/style_variants/sketch.jpg" style="margin: 8px;width: 100%;"> 707 </div> 708 </div> 709 <div class="row"> 710 <div class="column" style="margin: auto;width: 33%;"> 711 <b> 712 <p class="caption text-center"> Real photo </p> 713 </b> 714 </div> 715 <div class="column" style="margin: auto;width: 33%;"> 716 <p class="caption text-center"> Rembrandt </p> 717 </div> 718 <div class="column" style="margin: auto;width: 33%;"> 719 <p class="caption text-center"> Pencil sketch </p> 720 </div> 721 </div> 722 <div class="row"> 723 <div class="column" style="margin: auto;width: 33%;"> 724 <img id="comparison_stable_teapot" src="../eDiff-I/images/style_variants/van_gogh.jpg" style="margin: 8px;width: 100%;"> 725 </div> 726 <div class="column" style="margin: auto;width: 33%;"> 727 <img id="comparison_dalle_teapot" src="../eDiff-I/images/style_variants/hieroglyphics.jpg" style="margin: 8px;width: 100%;"> 728 </div> 729 <div class="column" style="margin: auto;width: 33%;"> 730 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/style_variants/abstract_cubism.jpg" style="margin: 8px;width: 100%;"> 731 </div> 732 </div> 733 <div class="row"> 734 <div class="column" style="margin: auto;width: 33%;"> 735 <b> 736 <p class="caption text-center"> Vincent van Gogh </p> 737 </b> 738 </div> 739 <div class="column" style="margin: auto;width: 33%;"> 740 <p class="caption text-center"> Egyptian tomb hieroglyphics </p> 741 </div> 742 <div class="column" style="margin: auto;width: 33%;"> 743 <p class="caption text-center"> Abstract cubism </p> 744 </div> 745 </div> 746 <div class="row"> 747 <div class="column" style="margin: auto;width: 33%;"> 748 <img id="comparison_stable_teapot" src="../eDiff-I/images/style_variants/pixel_art.jpg" style="margin: 8px;width: 100%;"> 749 </div> 750 <div class="column" style="margin: auto;width: 33%;"> 751 <img id="comparison_dalle_teapot" src="../eDiff-I/images/style_variants/hokusai.jpg" style="margin: 8px;width: 100%;"> 752 </div> 753 <div class="column" style="margin: auto;width: 33%;"> 754 <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/style_variants/mahubani.jpg" style="margin: 8px;width: 100%;"> 755 </div> 756 </div> 757 <div class="row"> 758 <div class="column" style="margin: auto;width: 33%;"> 759 <b> 760 <p class="caption text-center"> Pixel art </p> 761 </b> 762 </div> 763 <div class="column" style="margin: auto;width: 33%;"> 764 <p class="caption text-center"> Hokusai </p> 765 </div> 766 <div class="column" style="margin: auto;width: 33%;"> 767 <p class="caption text-center"> Madhubani </p> 768 </div> 769 </div> 770 </div> 771 <!-- Citation. --> 772 <hr style="max-width: 768px;"> 773 <div class="container" style="max-width: 768px;"> 774 <div class="row"> 775 <div class="col-md-12"> 776 <h2>Citation</h2> 777 </div> 778 </div> 779 <div class="code-box-copy" style="font-size: 0.9rem !important; max-width: 768px; margin: 0 auto;"> 780 <button class="code-box-copy__btn" data-clipboard-target="#citation" title="Copy" style="opacity: 1;"></button> 781 <pre><code class="language-bib" style="font-size: 0.9rem;" id="citation">@article{balaji2022eDiff-I, 782 title={eDiff-I: Text-to-Image Diffusion Models with Ensemble of Expert Denoisers}, 783 author={Yogesh Balaji and Seungjun Nah and Xun Huang and Arash Vahdat and Jiaming Song and Karsten Kreis and Miika Aittala and Timo Aila and
783Samuli Laine and Bryan Catanzaro and Tero Karras and Ming-Yu Liu}, 784 journal={arXiv preprint arXiv:2211.01324}, 785 year={2022} 786 </code></pre> 787 </div> 788 </div> 789 </div> 790 <footer class="text-muted"> 791 <div class="container"> 792 <p class="float-right"> 793 <a href="#">Back to top</a> 794 </p> 795 </div> 796 </footer>
797<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"947599f4388942eba3109cff3410e701","r":1,"spa":2}' crossorigin="anonymous"></script>
797 798</body> 799</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.