PageSourceSearch

https://deepimagination.cc/eDiff-I/

html deepimagination.cc collected 2026-10-03 22:02:00 UTC 47,114 bytes, 799 lines download raw bytes

1<!doctype html>
2<!--suppress HtmlRequiredAltAttribute -->
3<html lang="en">
4<head>
5    <!-- Google tag (gtag.js) -->
6    
6<script>
7        
vendor: 154 bytes, lines 7-10
7window.dataLayer = window.dataLayer || [];
8        function gtag() { dataLayer.push(arguments); }
9        gtag('js', new Date());
10        gtag('config', '
10G-F6S751BVJ5
vendor: 8 bytes, lines 10-11
10');
11    
11</script>
11
12    <!-- <meta charset="utf-8"> -->
13    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
14    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
15    <meta name="description" content="">
16    <meta name="author" content="">
17    <link rel="icon" href="https://deepimagination.cc/eDiff-I/images/favicon.ico">
18    <title>eDiff-I: Text-to-Image Diffusion Models with Ensemble of Expert Denoisers</title>
19	<link rel="canonical" href="https://deepimagination.cc/eDiff-I/" />
20    <!-- Bootstrap core CSS -->
21    <link rel="stylesheet" href="../bootstrap/4.0.0/css/bootstrap.min.css" >
22    <!-- Custom styles for this template -->
23    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Source+Sans+Pro">
24    <link rel="dns-prefetch" href="https://cdn.jsdelivr.net/"/>
25    <link href="../npm/prismjs/themes/prism.min.css" rel="stylesheet"/>
26    <link href="../gh/jablonczay/code-box-copy/code-box-copy/css/code-box-copy.min.css" rel="stylesheet"/>
27    <style>
28        body {
29            font-family: 'Source Sans Pro', sans-serif;
30            padding-bottom: 50px;
31        }
32        hr {
33            background: #80808083;
34        }
35        .grid-container {
36            display: grid;
37            grid-template-columns: 25% 25% 25% 25%;
38            padding: 2px;
39        }
40        .grid-container-video {
41            display: grid;
42            grid-template-columns: 100%;
43            padding: 2px;
44        }
45        .grid-item {
46            padding: 2px;
47            font-size: 30px;
48            text-align: center;
49            align-items: center;
50            justify-content: center;
51            display: flex;
52        }
53        /*input[type="checkbox"] {*/
54        /*    display: none;*/
55        /*}*/
56        img {
57            -webkit-filter: grayscale(0%);
58            filter: grayscale(0%);
59        }
60        label {
61            border: 1px solid #000;
62            padding: 0;
63            display: inline-block;
64            position: relative;
65            margin: 0;
66            cursor: pointer;
67            -webkit-touch-callout: none;
68            -webkit-user-select: none;
69            -khtml-user-select: none;
70            -moz-user-select: none;
71            -ms-user-select: none;
72            user-select: none;
73        }
74        .embed-responsive-2by1 {
75            padding-bottom: 50%;
76        }
77        .embed-responsive-4by1 {
78            padding-bottom: 25%;
79        }
80        .embed-responsive-6by1 {
81            padding-bottom: 16.67%;
82        }
83        .embed-responsive-8by1 {
84            padding-bottom: 12.50%;
85        }
86        .embed-responsive-teaser {
87            padding-bottom: 29%;
88        }
89        .float-button {
90            position: fixed;
91            right: 1%;
92            z-index: 9999;
93        }
94        .vert {
95            transform-origin: 50% 50%;
96            transform: rotate(180deg);
97            writing-mode: vertical-rl;
98            margin: 0;
99        }
100        .caption {
101            font-size: 0.9rem;
102            margin-top: 0;
103            margin-bottom: 5px;
104            font-weight: bold;
105        }
106        /*.row {*/
107        /*    display: grid;*/
108        /*    grid-auto-flow: row;*/
109        /*    gap: 5%;*/
110        /*}*/
111        .carousel .carousel-indicators li {
112            width: 10px;
113            height: 10px;
114            border-radius: 100%;
115        }
116        .carousel .carousel-indicators {
117            visibility: hidden;
118        }
119        .carousel:hover .carousel-indicators {
120            visibility: visible;
121        }
122        .w-85 {
123            width: 85% !important;
124        }
125        /*#videoGallery {
126            margin-left: 50px;
127            margin-right: 50px;
128        }*/
129        .carousel-control-prev {
130            width: 7%;
131            filter: invert(100%);
132        }
133        .carousel-control-next {
134            width: 7%;
135            filter: invert(100%);
136        }
137    </style>
138</head>
139<body>
140    <main role="main">
141        <section class="jumbotron text-center" style="padding: 2%; padding-bottom: 1%; background-color: #e6e9ec;">
142            <div class="container">
143                <h2 class="jumbotron-heading">eDiff-I: Text-to-Image Diffusion Models with Ensemble of Expert Denoisers
144                </h2>
145            </div>
146            <div class="container" style="max-width: 840px;">
147                <div class="row">
148                    <div class="col-md">
149                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="http://yogeshbalaji.github.io/" target="_blank">Yogesh Balaji<span style="color: #76b900;"></span></a></h5>
150                    </div>
151                    <div class="col-md">
152                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://seungjunnah.github.io/" target="_blank">Seungjun Nah<span style="color: #76b900;"></span></a></h5>
153                    </div>
154                    <div class="col-md">
155                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://xunhuang.me/" target="_blank">Xun Huang<span style="color: #76b900;"></span></a></h5>
156                    </div>
157                    <div class="col-md">
158                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="http://latentspace.cc/" target="_blank">Arash Vahdat<span style="color: #76b900;"></span></a></h5>
159                    </div>
160                    <div class="col-md">
161                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://tsong.me/" target="_blank">Jiaming Song<span style="color: #76b900;"></span></a></h5>
162                    </div>
163                    <div class="col-md">
164                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://karstenkreis.github.io/" target="_blank">Karsten Kreis<span style="color: #76b900;"></span></a></h5>
165                    </div>
166                </div>
167                <div class="row">
168                    <div class="col-md">
169                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://people.csail.mit.edu/miika/" target="_blank">Miika Aittala<span style="color: #76b900;"></span></a></h5>
170                    </div>
171                    <div class="col-md">
172                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://research.nvidia.com/person/timo-aila" target="_blank">Timo Aila<span style="color: #76b900;"></span></a></h5>
173                    </div>
174                    <div class="col-md">
175                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://users.aalto.fi/~laines9/" target="_blank">Samuli Laine<span style="color: #76b900;"></span></a></h5>
176                    </div>
177                    <div class="col-md">
178                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://ctnzr.io/" target="_blank">Bryan Catanzaro<span style="color: #76b900;"></span></a></h5>
179                    </div>
180                    <div class="col-md">
181                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="https://research.nvidia.com/person/tero-karras" target="_blank">Tero Karras<span style="color: #76b900;"></span></a></h5>
182                    </div>
183                    <div class="col-md">
184                        <h6 class="text-center" style="margin: 0;"><a class="text-center" href="http://mingyuliu.net/" target="_blank">Ming-Yu Liu<span style="color: #76b900;"></span></a></h5>
185                    </div>
186                </div>
187                <div class="row" style="margin-top:0.5rem; margin-left: auto; margin-right: auto;">
188                    <div class="col-md">
189                        <h5>NVIDIA Corporation</h5>
190                    </div>
191                </div>
192            </div>
193            <div class="buttons">
194                <a href="../pdf/2211.01324.pdf" target="_blank" class="btn btn-primary my-2">Paper
195                    (arxiv)</a>
196                <a href="https://www.youtube.com/watch?v=WbaVvlgxbl4" target="_blank" class="btn btn-primary my-2">Video</a>
197            </div>
198            <div class="container" style="max-width: 900px;">
199                <div class="row">
200                    <div class="col-md">
201                        <h5>TL;DR: eDiff-I is a new generation of generative AI content
202                            creation tool that offers unprecedented text-to-image synthesis with instant style transfer
203                            and intuitive painting with words capabilities.</h5>
204                    </div>
205                </div>
206                <div class="row">
207                    <div class="col-md">
208                        <p>
209                            We propose eDiff-I, a diffusion model for synthesizing images given text. Motivated by the
210                            empirical observation that the behavior of diffusion models differ at different stages of
211                            sampling, we propose to train an ensemble of expert denoising networks, each specializing
212                            for a specific noise interval. Our model is conditioned on the T5 text embeddings, CLIP
213                            image embeddings and CLIP text embeddings. Our approach can generate photorealistic images
214                            correponding to any input text prompt. In addition to text-to-image synthesis, we present
215                            two additional capabilies - (1) style transfer, which enables us to control the style of the
216                            generated sample using a reference style image, and (2) "Paint with words" - an application
217                            where the user can generate images by painting segmentation maps on canvas, which is very
218                            handy for crafting the desired image in mind.
219                        </p>
220                    </div>
221                </div>
222                <div class="row">
223                    <div class="column" style="margin: auto;width: 33%;">
224                        <img id="teaser_portal" src="../eDiff-I/images/teaser/portal.jpg" style="margin: 8px;width: 100%;">
225                    </div>
226                    <div class="column" style="margin: auto;width: 33%;">
227                        <img id="teaser_cat" src="../eDiff-I/images/teaser/cat_witch.jpg" style="margin: 8px;width: 100%;">
228                    </div>
229                    <div class="column" style="margin: auto;width: 33%;">
230                        <img id="teaser_landscape" src="../eDiff-I/images/teaser/landscape.jpg" style="margin: 8px;width: 100%;">
231                    </div>
232                </div>
233                <div class="row">
234                    <div class="col-md" style="margin: auto;width: 33%;">
235                        <p class="caption text-center"><i>A highly detailed digital painting of a portal in a mystic
236                                forest with many beautiful trees. A person is standing in front of the portal</i></p>
237                    </div>
238                    <div class="col-md" style="margin: auto;width: 33%;">
239                        <p class="caption text-center"><i>A highly detailed zoomed-in digital painting of a cat dressed
240                                as a witch wearing a wizard hat in a haunted house, artstation</i></p>
241                    </div>
242                    <div class="col-md" style="margin: auto;width: 33%;">
243                        <p class="caption text-center"><i>An image of a beautiful landscape of an ocean. There is a huge
244                                rock in the middle of the ocean. There is a mountain in the background. Sun is
245                                setting.</i></p>
246                    </div>
247                </div>
248                <div class="row">
249                    <div class="column" style="margin-right:4px; margin-left:8px; width: 15%">
250                        <img id="teaser" src="../eDiff-I/images/teaser/animal_scrape.jpg" style="width: 100%;">
251                        <p>Style Reference</p>
252                        <p class="caption text-center"><i>A photo of a duckling wearing a medieval soldier helmet and
253                                riding a skateboard.</i></p>
254                    </div>
255                    <div class="column" style="margin-left:0px;width: 30%;">
256                        <img id="teaser" src="../eDiff-I/images/teaser/duck.jpg" style="width: 100%;">
257                    </div>
258                    <div class="column" style="margin-left: 20px;width: 50%;">
259                        <div class="row">
260                            <img id="teaser" src="../eDiff-I/images/paint_with_words/aurora_text.jpg" style="margin: 2px; width: 48%;">
261                            <img id="teaser" src="../eDiff-I/images/paint_with_words/aurora_out.jpg" style="margin: 2px;width: 48%;">
262                        </div>
263                        <div class="col-md" style="margin: auto;width: 100%;">
264                            <p class="caption text-center"><i>
264A digital painting of a half-frozen lake near mountains
265                                    under a full
266                                    moon and aurora. A boat is in the middle of the lake. Highly detailed.</i></p>
267                        </div>
268                    </div>
269                </div>
270        </section>
271    </main>
272    <!-- Summary video -->
273    <div class="container" style="max-width: 768px;">
274        <div class="grid-container-video" style="width: 90%; margin-left: auto; margin-right: auto">
275            <div class="grid-item">
276                <div>
277                    <h5>Video</h5>
278                </div>
279            </div>
280            <div class="embed-responsive embed-responsive-16by9" style="margin: 0;">
281                <iframe class="embed-responsive-item" src="https://www.youtube.com/embed/WbaVvlgxbl4" allowfullscreen></iframe>
282            </div>
283        </div>
284        <div class="row">
285            <div class="col-md-12">
286                <hr style="max-width: 768px;">
287                <h5 class="text-center">Pipeline</h5>
288                <p>
289                    Our pipeline consists of a cascade of three diffusion models - a base model which can synthesize
290                    samples of 64x64 resolution, and two super-resolution stacks that can upsample the images
291                    progressively to 256x256 and 1024x1024 resolution respectively. Our models take an input caption
292                    and first compute T5 XXL embedding and text embedding. We optionally use CLIP image encodings
293                    which are computed from a reference image. These image embeddings can serve as a style vector.
294                    These embeddings are then fed into our cascaded diffusion models which progressively generate
295                    images of resolution 1024x1024.
296                </p>
297                <div class="row justify-content-center">
298                    <img src="../eDiff-I/images/pipeline/model_arch2.jpg" style="height: 500px;margin-bottom: 8px;">
299                </div>
300                <p class="caption text-center">Pipeline</p>
301            </div>
302        </div>
303        <div class="row">
304            <div class="col-md-12">
305                <hr style="max-width: 768px;">
306                <h5 class="text-center">Denoising experts</h5>
307                <p>
308                    In diffusion models, image synthesis happens via an iterative denoising process that gradually
309                    generates images from random noise. In the figure shown below, we start from a complete random
310                    noise which is then gradually denoised in multiple steps to finally produce an image of a panda
311                    riding a bike. In conventional diffusion model training, a single model is trained to denoise
312                    the whole noise distribution. In our framework, we instead train an ensemble of expert denoisers
313                    that are specialized for denoising in different intervals of the generative process. The use of
314                    such expert denoisers leads to improved synthesis capabilities.
315                </p>
316                <div class="row justify-content-center">
317                    <img src="../eDiff-I/images/pipeline/motivation2_fig.jpg" style="height: 500px;margin-bottom: 8px;">
318                </div>
319                <p class="caption text-center">Pipeline</p>
320            </div>
321        </div>
322        <!-- Outputs. -->
323        <hr style="max-width: 768px;">
324        <div class="container" style="max-width: 768px;">
325            <div class="text-center">
326                <h5>Results</h5>
327            </div>
328            <div class="row">
329                <div class="col-md-12">
330                    <p> Compared to the open-source text to image methods (<a href="https://huggingface.co/spaces/stabilityai/stable-diffusion" target="_blank">Stable
331                            diffusion</a>) and (<a href="../pdf/2204.06125.pdf" target="_blank">DALL-E2</a>), our model conistently leads to improved synthesis quality.
332                    </p>
333                </div>
334            </div>
335            <div class="row">
336                <div class="column" style="margin: auto;width: 33%;">
337                    <b>
338                        <p class="caption text-center"> Stable diffusion </p>
339                    </b>
340                </div>
341                <div class="column" style="margin: auto;width: 33%;">
342                    <p class="caption text-center"> DALL-E2 </p>
343                </div>
344                <div class="column" style="margin: auto;width: 33%;">
345                    <p class="caption text-center"> eDiff-I (ours) </p>
346                </div>
347            </div>
348            <div class="row">
349                <div class="column" style="margin: auto;width: 33%;">
350                    <img id="comparison_stable_teapot" src="../eDiff-I/images/comparison_multiple_entitles/stable_diffusion/stable_teapots.jpg" style="margin: 8px;width: 100%;">
351                </div>
352                <div class="column" style="margin: auto;width: 33%;">
353                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/comparison_multiple_entitles/dalle2/dalle_teapots.png" style="margin: 8px;width: 100%;">
354                </div>
355                <div class="column" style="margin: auto;width: 33%;">
356                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/comparison_multiple_entitles/ediffi/ours_teapots.jpg" style="margin: 8px;width: 100%;">
357                </div>
358            </div>
359            <p class="caption text-center"><i>There are two Chinese teapots on a table. One pot has a painting of a
360                    dragon,
361                    while the other pot has a painting of a panda.</i></p>
362            <div class="row">
363                <div class="column" style="margin: auto;width: 33%;">
364                    <img id="comparison_stable_grizzly" src="../eDiff-I/images/comparison_multiple_entitles/stable_diffusion/stable_grizzly.jpg" style="margin: 8px;width: 100%;">
365                </div>
366                <div class="column" style="margin: auto;width: 33%;">
367                    <img id="comparison_dalle_grizzly" src="../eDiff-I/images/comparison_multiple_entitles/dalle2/dalle_grizzly.png" style="margin: 8px;width: 100%;">
368                </div>
369                <div class="column" style="margin: auto;width: 33%;">
370                    <img id="comparison_eDiff-I_grizzly" src="../eDiff-I/images/comparison_multiple_entitles/ediffi/ours_grizzly.jpg" style="margin: 8px;width: 100%;">
371                </div>
372            </div>
373            <p class="caption text-center"><i>A photo of two cute teddy bears sitting on top of a grizzly bear in a
374                    beautiful forest. Highly detailed fantasy art, 4k, artstation</i></p>
375            <div class="row">
376                <div class="column" style="margin: auto;width: 33%;">
377                    <img id="text_stable_nvidia" src="../eDiff-I/images/comparison_text/stable_diffusion/stable_diffusion_nvidia_rocks.jpg" style="margin: 8px;width: 100%;">
378                </div>
379                <div class="column" style="margin: auto;width: 33%;">
380                    <img id="comparison_dalle_nvidia" src="../eDiff-I/images/comparison_text/dalle2/dalle_nvidia_rocks.png" style="margin: 8px;width: 100%;">
381                </div>
382                <div class="column" style="margin: auto;width: 33%;">
383                    <img id="comparison_eDiff-I_nvidia" src="../eDiff-I/images/comparison_text/ediffi/ours_nvidia_rocks.jpg" style="margin: 8px;width: 100%;">
384                </div>
385            </div>
386            <p class="caption text-center"><i>A photo of a golden retriever puppy wearing a green shirt. The shirt
387                    has text that says “NVIDIA rocks”. Backgroun
387d office. 4k dslr</i></p>
388            <div class="row">
389                <div class="column" style="margin: auto;width: 33%;">
390                    <img id="text_stable_monkey_friends" src="../eDiff-I/images/comparison_text/stable_diffusion/stable_diffusion_two_monkeys.jpg" style="margin: 8px;width: 100%;">
391                </div>
392                <div class="column" style="margin: auto;width: 33%;">
393                    <img id="comparison_dalle_monkey_friends" src="../eDiff-I/images/comparison_text/dalle2/dalle_best_friends.png" style="margin: 8px;width: 100%;">
394                </div>
395                <div class="column" style="margin: auto;width: 33%;">
396                    <img id="comparison_eDiff-I_monkey_friends" src="../eDiff-I/images/comparison_text/ediffi/ours_two_monkeys.jpg" style="margin: 8px;width: 100%;">
397                </div>
398            </div>
399            <p class="caption text-center"><i>A photo of two monkeys sitting on a tree. They are holding a wooden
400                    board that says “Best friends”, 4K dslr.</i></p>
401            <div class="row">
402                <div class="column" style="margin: auto;width: 33%;">
403                    <img id="comparison_stable_dish" src="../eDiff-I/images/comparison_long/stable_diffusion/stable_restaurant.jpg" style="margin: 8px;width: 100%;">
404                </div>
405                <div class="column" style="margin: auto;width: 33%;">
406                    <img id="comparison_dalle_dish" src="../eDiff-I/images/comparison_long/dalle2/dalle_restaurant.png" style="margin: 8px;width: 100%;">
407                </div>
408                <div class="column" style="margin: auto;width: 33%;">
409                    <img id="comparison_eDiff-I_dish" src="../eDiff-I/images/comparison_long/ediffi/ours_restaurant.jpg" style="margin: 8px;width: 100%;">
410                </div>
411            </div>
412            <p class="caption text-center"><i>A photo of a plate at a restaurant table with spaghetti and red sauce.
413                    There is sushi on top of the spaghetti. The dish is garnished with mint leaves. On the side,
414                    there is a glass with a purple drink, photorealistic, dslr.</i></p>
415            <div class="row">
416                <div class="column" style="margin: auto;width: 33%;">
417                    <img id="comparison_stable_cat_scooter" src="../eDiff-I/images/comparison_long/stable_diffusion/stable_cat_scooter.jpg" style="margin: 8px;width: 100%;">
418                </div>
419                <div class="column" style="margin: auto;width: 33%;">
420                    <img id="comparison_dalle_cat_scooter" src="../eDiff-I/images/comparison_long/dalle2/dalle_cat_scooter.png" style="margin: 8px;width: 100%;">
421                </div>
422                <div class="column" style="margin: auto;width: 33%;">
423                    <img id="comparison_eDiff-I_cat_scooter" src="../eDiff-I/images/comparison_long/ediffi/ours_cat_scooter.jpg" style="margin: 8px;width: 100%;">
424                </div>
425            </div>
426            <p class="caption text-center"><i>A close-up 4k dslr photo of a cat riding a scooter. It is wearing a
427                    plain shirt and has a bandana around its neck. It is wearing a scooter helmet. There are palm
428                    trees in the background.</i></p>
429        </div>
430        <!-- Style transfer. -->
431        <hr style="max-width: 768px;">
432        <div class="container" style="max-width: 768px;">
433            <div class="text-center">
434                <h5>Style transfer</h5>
435            </div>
436            <div class="row">
437                <div class="col-md-12">
438                    <p> Our method enables style transfer when the CLIP image embeddings are used. From a reference
439                        style image, we first extract the CLIP image emebddings which can be used as a style
440                        reference vector. The left panel in the figure shown below is a style reference. The middle
441                        panel shows the results when style conditioning is enabled. The panel on the right shows the
442                        results when style conditioning is disabled. When style conditioning is used, our model
443                        generates outputs that are faithful both to the input style and also the input caption. When
444                        style conditioning is disabled, we generate images in natural style.
445                    </p>
446                </div>
447            </div>
448            <div class="row">
449                <div class="column" style="margin: auto;width: 33%;">
450                    <b>
451                        <p class="caption text-center"> Reference style </p>
452                    </b>
453                </div>
454                <div class="column" style="margin: auto;width: 33%;">
455                    <p class="caption text-center"> Style conditioning enabled </p>
456                </div>
457                <div class="column" style="margin: auto;width: 33%;">
458                    <p class="caption text-center"> Style conditioning disabled </p>
459                </div>
460            </div>
461            <div class="row">
462                <div class="column" style="margin: auto;width: 33%;">
463                    <img id="comparison_style_two_pandas_reference" src="../eDiff-I/images/clip_img_conditioning/two_pandas_reference.jpg" style="margin: 8px;width: 100%;">
464                </div>
465                <div class="column" style="margin: auto;width: 33%;">
466                    <img id="comparison_style_two_pandas_style" src="../eDiff-I/images/clip_img_conditioning/two_pandas_conditioned.jpg" style="margin: 8px;width: 100%;">
467                </div>
468                <div class="column" style="margin: auto;width: 33%;">
469                    <img id="comparison_style_two_pandas_nostyle" src="../eDiff-I/images/clip_img_conditioning/two_pandas.jpg" style="margin: 8px;width: 100%;">
470                </div>
471            </div>
472            <p class="caption text-center"><i>A photo of two pandas walking on a road.</i></p>
473            <div class="row">
474                <div class="column" style="margin: auto;width: 33%;">
475                    <img id="comparison_style_rabbit_reference" src="../eDiff-I/images/clip_img_conditioning/rabbit_queen_reference.jpg" style="margin: 8px;width: 100%;">
476                </div>
477                <div class="column" style="margin: auto;width: 33%;">
478                    <img id="comparison_style_rabbit_style" src="../eDiff-I/images/clip_img_conditioning/rabbit_queen_conditioned.jpg" style="margin: 8px;width: 100%;">
479                </div>
480                <div class="column" style="margin: auto;width: 33%;">
481                    <img id="comparison_style_rabbit_nostyle" src="../eDiff-I/images/clip_img_conditioning/rabbit_queen.jpg" style="margin: 8px;width: 100%;">
482                </div>
483            </div>
484            <p class="caption text-center"><i>A detailed oil painting of a beautiful rabbit queen wearing a royal
485                    gown in a palace. She is looking outside the window, artistic.</i></p>
486            <div class="row">
487                <div class="column" style="margin: auto;width: 33%;">
488                    <img id="comparison_style_rabbit_reference" src="../eDiff-I/images/clip_img_conditioning/dog_trumpet_reference.jpg" style="margin: 8px;width: 100%;">
489                </div>
490                <div class="column" style="margin: auto;width: 33%;">
491                    <img id="comparison_style_rabbit_style" src="../eDiff-I/images/clip_img_conditioning/dog_trumpet_conditioned.jpg" style="margin: 8px;width: 100%;">
492                </div>
493                <div class="column" style="margin: auto;width: 33%;">
494                    <img id="comparison_style_rabbit_nostyle" src="../eDiff-I/images/clip_img_conditioning/dog_trumpet.jpg" style="margin: 8px;width: 100%;">
495                </div>
496            </div>
497            <p class="caption text-center"><i>A dslr photo of a dog playing trumpet from the top of a mountain.</i>
498            </p>
499            <div class="row">
500                <div class="column" style="margin: auto;width: 33%;">
501                    <img id="comparison_style_teddy_bear_reference" src="../eDiff-I/images/clip_img_conditioning/teddy_bear_reference.jpg" style="margin: 8px;width: 100%;">
502                </div>
503                <div class="column" style="margin: auto;width: 33%;">
504                    <img id="comparison_style_teddy_bear_style" src="../eDiff-I/images/clip_img_conditioning/teddy_bear_conditioned.jpg" style="margin: 8px;width: 100%;">
505                </div>
506                <div class="column" style="margin: auto;width: 33%;">
507                    <img id="comparison_style_teddy_bear_nostyle" src="../eDiff-I/images/clip_img_conditioning/teddy_bear.jpg" style="margin: 8px;width: 100%;">
508                </div>
509            </div>
510            <p class="caption text-center"><i>A photo of a teddy bear wearing a casual plain white shirt surfing in
511                    the ocean.</i>
512            </p>
513        </div>
514        <!-- Paint with words. -->
515        <hr style="max-width: 768px;">
516        <div class="container" style="max-width: 768px;">
517            <div class="text-center">
518                <h5>Paint with words</h5>
519            </div>
520            <div class="row">
521                <div class="col-md-12">
522                    <p> Our method allows users to control the location of objects mentioned in the text prompt by
523                        selecting phrases and scribbling them on the image. The model then makes use of the prompt
524                        along with the maps to generate images that can consistent both with the caption and the
525                        input map.
526                    </p>
527                </div>
528            </div>
529            <div class="row">
530                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
531                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/portal.mp4" type="video/mp4">
532                </video>
533            </div>
534            <div class="row">
535                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
536                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/halloween.mp4" type="video/mp4">
537                </video>
538            </div>
539            <div class="row">
540                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
541                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/aurora.mp4" type="video/mp4">
542                </video>
543            </div>
544            <div class="row">
545                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
546                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/monster.mp4" type="video/mp4">
547                </video>
548            </div>
549            <div class="row">
550                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
551                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/rabbit_mage.mp4" type="video/mp4">
552                </video>
553            </div>
554            <div class="row">
555                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
556                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/rainbow.mp4" type="video/mp4">
557                </video>
558            </div>
559            <div class="row">
560                <video height="350px" controls muted loop style="vertical-align: top; margin-left: auto; margin-right: auto">
561                    <source src="https://deepimagination.cc/eDiff-I/videos/paint_videos/squirrel.mp4" type="video/mp4">
562                </video>
563            </div>
564        </div>
565        <!-- Effect of using denoising experts. -->
566        <hr style="max-width: 768px;">
567        <div class="container" style="max-width: 768px;">
568            <div class="text-center">
569                <h5>Benefit of using denoising experts</h5>
570            </div>
571            <div class="row">
572                <div class="col-md-12">
573                    <p> We illustrate the benefit of using denoising experts by visualizing samples generated by
574                        our approach and comparing it with a baseline that does not use any denoising experts.
575                        The left two panels on the figure below denotes a baseline not using any experts, while
576                        the right two panels show the results obtaining by using our expert models. We find that
577                        we can greately improve on faithfullness to the input text by using denoising experts.
578                    </p>
579                </div>
580            </div>
581            <div class="row">
582                <div class="column" style="margin: auto;width: 50%;">
583                    <b>
584                        <p class="caption text-center"> Baseline </p>
585                    </b>
586                </div>
587                <div class="column" style="margin: auto;width: 50%;">
588                    <p class="caption text-center"> eDiff-I </p>
589                </div>
590            </div>
591            <div class="row">
592                <div class="column" style="margin: auto;width: 24%;">
593                    <img id="comparison_stable_teapot" src="../eDiff-I/images/model_size_comparison/base/teddy_bear/img_0.jpg" style="margin: 8px;width: 100%;">
594                </div>
595                <div class="column" style="margin: auto;width: 24%;">
596                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/model_size_comparison/base/teddy_bear/img_1.jpg" style="margin: 8px;width: 100%;">
597                </div>
598                <div class="column" style="margin: auto;width: 24%;">
599                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/teddy_bear/img_0.jpg" style="margin: 8px;width: 100%;">
600                </div>
601                <div class="column" style="margin: auto;width: 24%;">
602                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/teddy_bear/img_1.jpg" style="margin: 8px;width: 100%;">
603                </div>
604            </div>
605            <p class="caption text-center"><i>A 4k dslr photo of two teddy bears wearing a sports jersey
606                    with the text “eDiffi” written on it. They are on a soccer field.</i></p>
607            <div class="row">
608                <div class="column" style="margin: auto;width: 24%;">
609                    <img id="comparison_stable_teapot" src="../eDiff-I/images/model_size_comparison/base/monkey_bike/img_0.jpg" style="margin: 8px;width: 100%;">
610                </div>
611                <div class="column" style="margin: auto;width: 24%;">
612                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/model_size_comparison/base/monkey_bike/img_1.jpg" style="margin: 8px;width: 100%;">
613                </div>
614                <div class="column" style="margin: auto;width: 24%;">
615                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/monkey_bike/img_0.jpg" style="margin: 8px;width: 100%;">
616                </div>
617                <div class="column" style="margin: auto;width: 24%;">
618                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/model_size_comparison/ediffi/monkey_bike/img_1.jpg" style="margin: 8px;width: 100%;">
619                </div>
620            </div>
621            <p class="caption text-center"><i>An origami of a monkey dressed as a monk riding a bike on a
622                    mountain.</i></p>
623        </div>
624        <!-- Comparison between T5 and CLIP embeddigs. -->
625        <hr style="max-width: 768px;">
626        <div class="container" style="max-width: 768px;">
627            <div class="text-center">
628                <h5>Compairson between T5 and CLIP text embeddings</h5>
629            </div>
630            <div class="row">
631                <div class="col-md-12">
632                    <p> We study the effect of using CLIP text embeddings and T5 embeddings in isolation, and
633                        compare wi
633th the full system that uses CLIP + T5 embeddings. We observe that images
634                        generated in the CLIP-text-only setting often contain correct foreground objects and
635                        they tend to miss fine-grain details. Images generated in the T5-text-only setting are
636                        of higher quality, but they sometimes contain incorrect objects. Using CLIP+T5 results
637                        in best performance.
638                    </p>
639                </div>
640            </div>
641            <div class="row">
642                <div class="column" style="margin: auto;width: 33%;">
643                    <b>
644                        <p class="caption text-center"> CLIP text only </p>
645                    </b>
646                </div>
647                <div class="column" style="margin: auto;width: 33%;">
648                    <p class="caption text-center"> T5 only </p>
649                </div>
650                <div class="column" style="margin: auto;width: 33%;">
651                    <p class="caption text-center"> CLIP + T5 </p>
652                </div>
653            </div>
654            <div class="row">
655                <div class="column" style="margin: auto;width: 33%;">
656                    <img id="comparison_stable_teapot" src="../eDiff-I/images/t5_vs_clip/corgi_text_clip.jpg" style="margin: 8px;width: 100%;">
657                </div>
658                <div class="column" style="margin: auto;width: 33%;">
659                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/t5_vs_clip/corgi_text_t5.jpg" style="margin: 8px;width: 100%;">
660                </div>
661                <div class="column" style="margin: auto;width: 33%;">
662                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/t5_vs_clip/corgi_text.jpg" style="margin: 8px;width: 100%;">
663                </div>
664            </div>
665            <p class="caption text-center"><i>A photo of a cute corgi wearing a beret holding a sign
666                    that
667                    says
668                    “Diffusion Models”. There is Eiffel tower in the background.</i></p>
669            <div class="row">
670                <div class="column" style="margin: auto;width: 33%;">
671                    <img id="comparison_stable_teapot" src="../eDiff-I/images/t5_vs_clip/parrots_singing_clip.jpg" style="margin: 8px;width: 100%;">
672                </div>
673                <div class="column" style="margin: auto;width: 33%;">
674                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/t5_vs_clip/parrots_singing_t5.jpg" style="margin: 8px;width: 100%;">
675                </div>
676                <div class="column" style="margin: auto;width: 33%;">
677                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/t5_vs_clip/parrots_singing.jpg" style="margin: 8px;width: 100%;">
678                </div>
679            </div>
680            <p class="caption text-center"><i>A photo of a red parrot, a blue parrot and a green parrot singing
681                    at a concert in front of a microphone. Colorful lights in the background.</i></p>
682        </div>
683        <!-- Style variations. -->
684        <hr style="max-width: 768px;">
685        <div class="container" style="max-width: 768px;">
686            <div class="text-center">
687                <h5>Style Variations</h5>
688            </div>
689            <div class="row">
690                <div class="col-md-12">
691                    <p> Our method can also generate images with different styles, which can be specified in the
692                        input caption.
693                    </p>
694                </div>
695            </div>
696            <p class="caption text-center"><i>A photo of penguin working as a fruit vendor in a tropical
697                    village.</i></p>
698            <div class="row">
699                <div class="column" style="margin: auto;width: 33%;">
700                    <img id="comparison_stable_teapot" src="../eDiff-I/images/style_variants/real.jpg" style="margin: 8px;width: 100%;">
701                </div>
702                <div class="column" style="margin: auto;width: 33%;">
703                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/style_variants/rembrandt.jpg" style="margin: 8px;width: 100%;">
704                </div>
705                <div class="column" style="margin: auto;width: 33%;">
706                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/style_variants/sketch.jpg" style="margin: 8px;width: 100%;">
707                </div>
708            </div>
709            <div class="row">
710                <div class="column" style="margin: auto;width: 33%;">
711                    <b>
712                        <p class="caption text-center"> Real photo </p>
713                    </b>
714                </div>
715                <div class="column" style="margin: auto;width: 33%;">
716                    <p class="caption text-center"> Rembrandt </p>
717                </div>
718                <div class="column" style="margin: auto;width: 33%;">
719                    <p class="caption text-center"> Pencil sketch </p>
720                </div>
721            </div>
722            <div class="row">
723                <div class="column" style="margin: auto;width: 33%;">
724                    <img id="comparison_stable_teapot" src="../eDiff-I/images/style_variants/van_gogh.jpg" style="margin: 8px;width: 100%;">
725                </div>
726                <div class="column" style="margin: auto;width: 33%;">
727                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/style_variants/hieroglyphics.jpg" style="margin: 8px;width: 100%;">
728                </div>
729                <div class="column" style="margin: auto;width: 33%;">
730                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/style_variants/abstract_cubism.jpg" style="margin: 8px;width: 100%;">
731                </div>
732            </div>
733            <div class="row">
734                <div class="column" style="margin: auto;width: 33%;">
735                    <b>
736                        <p class="caption text-center"> Vincent van Gogh </p>
737                    </b>
738                </div>
739                <div class="column" style="margin: auto;width: 33%;">
740                    <p class="caption text-center"> Egyptian tomb hieroglyphics </p>
741                </div>
742                <div class="column" style="margin: auto;width: 33%;">
743                    <p class="caption text-center"> Abstract cubism </p>
744                </div>
745            </div>
746            <div class="row">
747                <div class="column" style="margin: auto;width: 33%;">
748                    <img id="comparison_stable_teapot" src="../eDiff-I/images/style_variants/pixel_art.jpg" style="margin: 8px;width: 100%;">
749                </div>
750                <div class="column" style="margin: auto;width: 33%;">
751                    <img id="comparison_dalle_teapot" src="../eDiff-I/images/style_variants/hokusai.jpg" style="margin: 8px;width: 100%;">
752                </div>
753                <div class="column" style="margin: auto;width: 33%;">
754                    <img id="comparison_eDiff-I_teapot" src="../eDiff-I/images/style_variants/mahubani.jpg" style="margin: 8px;width: 100%;">
755                </div>
756            </div>
757            <div class="row">
758                <div class="column" style="margin: auto;width: 33%;">
759                    <b>
760                        <p class="caption text-center"> Pixel art </p>
761                    </b>
762                </div>
763                <div class="column" style="margin: auto;width: 33%;">
764                    <p class="caption text-center"> Hokusai </p>
765                </div>
766                <div class="column" style="margin: auto;width: 33%;">
767                    <p class="caption text-center"> Madhubani </p>
768                </div>
769            </div>
770        </div>
771        <!-- Citation. -->
772        <hr style="max-width: 768px;">
773        <div class="container" style="max-width: 768px;">
774            <div class="row">
775                <div class="col-md-12">
776                    <h2>Citation</h2>
777                </div>
778            </div>
779            <div class="code-box-copy" style="font-size: 0.9rem !important; max-width: 768px; margin: 0 auto;">
780                <button class="code-box-copy__btn" data-clipboard-target="#citation" title="Copy" style="opacity: 1;"></button>
781                <pre><code class="language-bib" style="font-size: 0.9rem;" id="citation">@article{balaji2022eDiff-I,
782    title={eDiff-I: Text-to-Image Diffusion Models with Ensemble of Expert Denoisers},
783    author={Yogesh Balaji and Seungjun Nah and Xun Huang and Arash Vahdat and Jiaming Song and Karsten Kreis and Miika Aittala and Timo Aila and 
783Samuli Laine and Bryan Catanzaro and Tero Karras and Ming-Yu Liu},
784    journal={arXiv preprint arXiv:2211.01324},
785    year={2022}
786                    </code></pre>
787            </div>
788        </div>
789    </div>
790    <footer class="text-muted">
791        <div class="container">
792            <p class="float-right">
793                <a href="#">Back to top</a>
794            </p>
795        </div>
796    </footer>
797<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"947599f4388942eba3109cff3410e701","r":1,"spa":2}' crossorigin="anonymous"></script>
797
798</body>
799</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.