1<!DOCTYPE html> 2<!-- saved from url=(0027)https://llava-vl.github.io/ --> 3<html class="fontawesome-i2svg-active fontawesome-i2svg-complete" lang="en-US"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> 4 5 <meta name="description" content="Vision-Language Feedback"> 6 <meta name="keywords" content="multimodal chatbot"> 7 <meta name="viewport" content="width=device-width, initial-scale=1"> 8 <title>Silkie</title> 9 10 <style type="text/css">svg:not(:root).svg-inline--fa{overflow:visible}.svg-inline--fa{display:inline-block;font-size:inherit;height:1em;overflow:visible;vertical-align:-.125em}.svg-inline--fa.fa-lg{vertical-align:-.225em}.svg-inline--fa.fa-w-1{width:.0625em}.svg-inline--fa.fa-w-2{width:.125em}.svg-inline--fa.fa-w-3{width:.1875em}.svg-inline--fa.fa-w-4{width:.25em}.svg-inline--fa.fa-w-5{width:.3125em}.svg-inline--fa.fa-w-6{width:.375em}.svg-inline--fa.fa-w-7{width:.4375em}.svg-inline--fa.fa-w-8{width:.5em}.svg-inline--fa.fa-w-9{width:.5625em}.svg-inline--fa.fa-w-10{width:.625em}.svg-inline--fa.fa-w-11{width:.6875em}.svg-inline--fa.fa-w-12{width:.75em}.svg-inline--fa.fa-w-13{width:.8125em}.svg-inline--fa.fa-w-14{width:.875em}.svg-inline--fa.fa-w-15{width:.9375em}.svg-inline--fa.fa-w-16{width:1em}.svg-inline--fa.fa-w-17{width:1.0625em}.svg-inline--fa.fa-w-18{width:1.125em}.svg-inline--fa.fa-w-19{width:1.1875em}.svg-inline--fa.fa-w-20{width:1.25em}.svg-inline--fa.fa-pull-left{margin-right:.3em;width:auto}.svg-inline--fa.fa-pull-right{margin-left:.3em;width:auto}.svg-inline--fa.fa-border{height:1.5em}.svg-inline--fa.fa-li{width:2em}.svg-inline--fa.fa-fw{width:1.25em}.fa-layers svg.svg-inline--fa{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.fa-layers{display:inline-block;height:1em;position:relative;text-align:center;vertical-align:-.125em;width:1em}.fa-layers svg.svg-inline--fa{-webkit-transform-origin:center center;transform-origin:center center}.fa-layers-counter,.fa-layers-text{display:inline-block;position:absolute;text-align:center}.fa-layers-text{left:50%;top:50%;-webkit-transform:translate(-50%,-50%);transform:translate(-50%,-50%);-webkit-transform-origin:center center;transform-origin:center center}.fa-layers-counter{background-color:#ff253a;border-radius:1em;-webkit-box-sizing:border-box;box-sizing:border-box;color:#fff;height:1.5em;line-height:1;max-width:5em;min-width:1.5em;overflow:hidden;padding:.25em;right:0;text-overflow:ellipsis;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top right;transform-origin:top right}.fa-layers-bottom-right{bottom:0;right:0;top:auto;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:bottom right;transform-origin:bottom right}.fa-layers-bottom-left{bottom:0;left:0;right:auto;top:auto;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:bottom left;transform-origin:bottom left}.fa-layers-top-right{right:0;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top right;transform-origin:top right}.fa-layers-top-left{left:0;right:auto;top:0;-webkit-transform:scale(.25);transform:scale(.25);-webkit-transform-origin:top left;transform-origin:top left}.fa-lg{font-size:1.3333333333em;line-height:.75em;vertical-align:-.0667em}.fa-xs{font-size:.75em}.fa-sm{font-size:.875em}.fa-1x{font-size:1em}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-6x{font-size:6em}.fa-7x{font-size:7em}.fa-8x{font-size:8em}.fa-9x{font-size:9em}.fa-10x{font-size:10em}.fa-fw{text-align:center;width:1.25em}.fa-ul{list-style-type:none;margin-left:2.5em;padding-left:0}.fa-ul>li{position:relative}.fa-li{left:-2em;position:absolute;text-align:center;width:2em;line-height:inherit}.fa-border{border:solid .08em #eee;border-radius:.1em;padding:.2em .25em .15em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left,.fab.fa-pull-left,.fal.fa-pull-left,.far.fa-pull-left,.fas.fa-pull-left{margin-right:.3em}.fa.fa-pull-right,.fab.fa-pull-right,.fal.fa-pull-right,.far.fa-pull-right,.fas.fa-pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.fa-rotate-90{-webkit-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-webkit-transform:rotate(180deg);transform:rotate(180deg)}
10.fa-rotate-270{-webkit-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-webkit-transform:scale(-1,1);transform:scale(-1,1)}.fa-flip-vertical{-webkit-transform:scale(1,-1);transform:scale(1,-1)}.fa-flip-both,.fa-flip-horizontal.fa-flip-vertical{-webkit-transform:scale(-1,-1);transform:scale(-1,-1)}:root .fa-flip-both,:root .fa-flip-horizontal,:root .fa-flip-vertical,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-rotate-90{-webkit-filter:none;filter:none}.fa-stack{display:inline-block;height:2em;position:relative;width:2.5em}.fa-stack-1x,.fa-stack-2x{bottom:0;left:0;margin:auto;position:absolute;right:0;top:0}.svg-inline--fa.fa-stack-1x{height:1em;width:1.25em}.svg-inline--fa.fa-stack-2x{height:2em;width:2.5em}.fa-inverse{color:#fff}.sr-only{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.sr-only-focusable:active,.sr-only-focusable:focus{clip:auto;height:auto;margin:0;overflow:visible;position:static;width:auto}.svg-inline--fa .fa-primary{fill:var(--fa-primary-color,currentColor);opacity:1;opacity:var(--fa-primary-opacity,1)}.svg-inline--fa .fa-secondary{fill:var(--fa-secondary-color,currentColor);opacity:.4;opacity:var(--fa-secondary-opacity,.4)}.svg-inline--fa.fa-swap-opacity .fa-primary{opacity:.4;opacity:var(--fa-secondary-opacity,.4)}.svg-inline--fa.fa-swap-opacity .fa-secondary{opacity:1;opacity:var(--fa-primary-opacity,1)}.svg-inline--fa mask .fa-primary,.svg-inline--fa mask .fa-secondary{fill:#000}.fad.fa-inverse{color:#fff}</style><link rel="stylesheet" href="./LLaVA_files/css"> 11 <link rel="stylesheet" href="./LLaVA_files/bulma.min.css"> 12 <link rel="stylesheet" href="./LLaVA_files/bootstrap.min.css"> 13 <link rel="stylesheet" href="./LLaVA_files/academicons.min.css"> 14 <link rel="stylesheet" href="./LLaVA_files/all.min.css"> 15 <link rel="stylesheet" href="./LLaVA_files/index.css"> 16 <link rel="icon" href="./LLaVA_files/silkie.png"> 17 <link href="./LLaVA_files/icon" rel="stylesheet"> 18 19 20
20<script src="./LLaVA_files/jquery.min.js"></script>
20 21
21<script defer="" src="./LLaVA_files/all.min.js"></script>
21 22
22<script type="module" src="./LLaVA_files/gradio.js"></script>
22 23<style> 24 .expandable-card .card-text-container { 25 max-height: 200px; 26 overflow-y: hidden; 27 position: relative; 28 } 29 30 .expandable-card.expanded .card-text-container { 31 max-height: none; 32 } 33 34 .expand-btn { 35 position: relative; 36 display: none; 37 background-color: rgba(255, 255, 255, 0.8); 38 /* margin-top: -20px; */ 39 /* justify-content: center; */ 40 color: #510c75; 41 border-color: transparent; 42 } 43 44 .expand-btn:hover { 45 background-color: rgba(200, 200, 200, 0.8); 46 text-decoration: none; 47 border-color: transparent; 48 color: #510c75; 49 } 50 51 .expand-btn:focus { 52 outline: none; 53 text-decoration: none; 54 } 55 56 .expandable-card:not(.expanded) .card-text-container:after { 57 content: ""; 58 position: absolute; 59 bottom: 0; 60 left: 0; 61 width: 100%; 62 height: 90px; 63 background: linear-gradient(rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 1)); 64 } 65 66 .expandable-card:not(.expanded) .expand-btn { 67 margin-top: -40px; 68 } 69 70 .card-body { 71 padding-bottom: 5px; 72 } 73 74 .vertical-flex-layout { 75 justify-content: center; 76 align-items: center; 77 height: 100%; 78 display: flex; 79 flex-direction: column; 80 gap: 5px; 81 } 82 83 .figure-img { 84 max-width: 100%; 85 height: auto; 86 } 87 88 .adjustable-font-size { 89 font-size: calc(0.5rem + 2vw); 90 } 91 92 .chat-history { 93 flex-grow: 1; 94 overflow-y: auto; 95 /* overflow-x: hidden; */ 96 padding: 5px; 97 border-bottom: 1px solid #ccc; 98 margin-bottom: 10px; 99 } 100 101 #gradio pre { 102 background-color: transparent; 103 } 104</style>
104<script type="module" crossorigin="" src="./LLaVA_files/index-9405f928.js"></script>
104<link rel="stylesheet" href="./LLaVA_files/index-8f5e8d2d.css"><link rel="stylesheet" href="./LLaVA_files/theme.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Blocks-b7e1d3bc.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Button-496612d6.js"><link rel="stylesheet" href="./LLaVA_files/Button-3657eefc.css"><link rel="stylesheet" href="./LLaVA_files/Blocks-005a10ea.css"><link rel="stylesheet" href="./LLaVA_files/css2"><link rel="stylesheet" href="./LLaVA_files/css2(1)"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-7da00dd0.js"><link rel="modulepreload" as="script" crossorigin="" href="./LLaVA_files/index-9405f928.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-e350ed90.js"><link rel="stylesheet" href="./LLaVA_files/index-edf307d2.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-9454ebde.js"><link rel="stylesheet" href="./LLaVA_files/index-93c91554.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-29aa4c6e.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Column-56337312.js"><link rel="stylesheet" href="./LLaVA_files/Column-2853eb31.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-6c061571.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/BlockTitle-3b63dcc6.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Info-88fa40fc.js"><link rel="stylesheet" href="./LLaVA_files/ColorPicker-41813019.css"><link rel="stylesheet" href="./LLaVA_files/DropdownArrow-5fa4dd09.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-dc4f9a55.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/BlockLabel-eae79103.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Image-0296a7e2.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/StaticImage.svelte_svelte_type_style_lang-943673fc.js"><link rel="stylesheet" href="./LLaVA_files/StaticImage-ede66243.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/ModifyUpload-3ceaec64.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/ModifyUpload.svelte_svelte_type_style_lang-ba6baa96.js"><link rel="stylesheet" href="./LLaVA_files/ModifyUpload-77b0d4b2.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Upload-0b7d7edd.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Empty-0bf01d93.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Download-836e8a5d.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/UploadText-67055cc3.js"><link rel="stylesheet" href="./LLaVA_files/UploadText-33d53a1c.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Image-4f594cb8.js"><link rel="stylesheet" href="./LLaVA_files/Image-003ee87c.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-fb7b6d6a.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-4360065a.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/_commonjsHelpers-042e6b4d.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/csv-b0b7514a.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/dsv-576afacd.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Model3D-49e4759c.js"><link rel="stylesheet" href="./LLaVA_files/Model3D-98fc2b2c.css"><link rel="stylesheet" href="./LLaVA_files/index-4a8edf2e.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-544cdc5f.js">
104<link rel="stylesheet" href="./LLaVA_files/index-8f1feca1.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-efacc33c.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-5ff6e9da.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Form-e2ba6e12.js"><link rel="stylesheet" href="./LLaVA_files/Form-a4a7741e.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-74280621.js"><link rel="stylesheet" href="./LLaVA_files/index-d9aad8e1.css"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-f67a924c.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Textbox-91bfa043.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/Copy-4761247d.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-00a3ebda.js"><link rel="modulepreload" as="script" crossorigin="" href="https://gradio.s3-us-west-2.amazonaws.com/3.27.0/assets/index-1c9d35e7.js"><link rel="stylesheet" href="./LLaVA_files/index-3ca142e0.css">
104<script async="" defer="" src="./LLaVA_files/js"></script>
104</head> 105 106 107 108<body data-new-gr-c-s-check-loaded="14.1126.0" data-gr-ext-installed=""> 109 110 <nav class="navbar" role="navigation" aria-label="main navigation"> 111 <div class="navbar-brand"> 112 <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false"> 113 <span aria-hidden="true"></span> 114 <span aria-hidden="true"></span> 115 <span aria-hidden="true"></span> 116 </a> 117 </div> 118 <div class="navbar-menu"> 119 <div class="navbar-start" style="flex-grow: 1; justify-content: center;"> 120 <div class="navbar-item has-dropdown is-hoverable"> 121 <a class="navbar-link"> 122 ð¥ More Research 123 </a> 124 <div class="navbar-dropdown"> 125 <a class="navbar-item" href="https://vl-rewardbench.github.io"> 126 VL-RewardBench 127 </a> 128 <a class="navbar-item" href="https://video-mme.github.io/home_page.html"> 129 Video-MME 130 </a> 131 <a class="navbar-item" href="https://video-t3.github.io/"> 132 Temporal Transfer from Text 133 </a> 134 <a class="navbar-item" href="https://huggingface.co/spaces/lyx97/TempCompass"> 135 TempCompass 136 </a> 137 <a class="navbar-item" href="https://vlf-silkie.github.io/"> 138 VLFeedback 139 </a> 140 <a class="navbar-item" href="https://m3-it.github.io/"> 141 M3IT 142 </a> 143 </div> 144 </div> 145 </div> 146 </div> 147 </nav> 148 149 <section class="hero"> 150 <div class="hero-body"> 151 <div class="container is-max-desktop"> 152 <div class="columns is-centered"> 153 <div class="column has-text-centered"> 154 <!-- <h1 class="title is-1 publication-title">VLFeedback and Silkie</h1> --> 155 <!-- <h1 class="title is-1 publication-title"> 156 <img src="./LLaVA_files/silkie.png" alt="VLFeedback and Silkie" style="height: 50px; width: 50px;"> VLFeedback and Silkie 157 </h1> --> 158 <h1 class="title is-1 publication-title"> 159 <span style="display: inline-block; vertical-align: middle;">VLFeedback</span> 160 <span style="display: inline-block; vertical-align: middle; margin-top: -15px; margin-left: -7px;"> 161 <img src="./LLaVA_files/silkie.png" alt="VLFeedback and Silkie" style="height: 50px; width: 50px;"> 162 </span> 163 </h1> 164 <h3 class="title is-3 publication-title">A Large-Scale AI Feedback Dataset for Large Vision-Language Models Alignment</h3> 165 <div class="is-size-5"> 166 <span class="author-block"> 167 <a href="https://lilei-nlp.github.io" style="color:#008AD7;font-weight:normal;">Lei Li<sup>*</sup></a>, 168 </span> 169 <span class="author-block"> 170 <a href="https://zhxie.site/" style="color:#008AD7;font-weight:normal;">Zhihui Xie<sup>*</sup></a>, 171 </span> 172 <span class="author-block"> 173 <a href="https://scholar.google.com/citations?user=BizedOAAAAAJ" style="color:#008AD7;font-weight:normal;">Mukai Li</a>, 174 </span> <br> 175 <span class="author-block"> 176 <a href="https://github.com/Shunian-Chen" style="color:#F2A900;font-weight:normal;">Shunian Chen</a>, 177 </span> 178 <span class="author-block"> 179 <a href="https://scholar.google.com.tw/citations?user=K0uQ3ygAAAAJ" style="color:#94070A;font-weight:normal;">Peiyi Wang</a>, 180 </span> 181 <span class="author-block"> 182 <a href="https://chenllliang.github.io/about/" style="color:#94070A;font-weight:normal;">Liang Chen</a>, 183 </span> <br> 184 <span class="author-block"> 185 <a href="https://scholar.google.com/citations?user=SAeMbW4AAAAJ&hl=en" style="color:#008AD7;font-weight:normal;">Yazheng Yang</a>, 186 </span> 187 <span class="author-block"> 188 <a href="https://wabyking.github.io/old.html" style="color:#F2A900;font-weight:normal;">Benyou Wang</a>, 189 </span>
190 <span class="author-block"> 191 <a href="https://ikekonglp.github.io/" style="color:#008AD7;font-weight:normal;">Lingpeng Kong</a>, 192 </span> 193 <span class="author-block"> 194 <a href="https://leuchine.github.io/" style="color:#008AD7;font-weight:normal;">Qi Liu</a> 195 </span> 196 197 </span> 198 </div> 199 200 <br> 201 <div class="is-size-5 publication-authors"> 202 <span class="author-block"><b style="color:#008AD7; font-weight:normal">â¶ </b>The University of Hong Kong</span> 203 <br> 204 <span class="author-block"> <b style="color:#F2A900; font-weight:normal"> â¶ </b> The Chinese University of Hong Kong, Shenzhen </span> <br> 205 <span class="author-block"><b style="color:#94070A; font-weight:normal">â¶ </b>Peking University</span> <br> 206 <span class="author-block"> <sup>*</sup>Equal Contribution</span> 207 </div> 208 209 210 <div class="column has-text-centered"> 211 <div class="publication-links"> 212 <span class="link-block"> 213 <a href="https://arxiv.org/abs/2410.09421" target="_blank" class="external-link button is-normal is-rounded is-dark"> 214 <span class="icon"> 215 <!-- <i class="ai-arxiv"></i> --> 216 </span> 217 <span>arXiv</span> 218 </a> 219 </span> 220 <span class="link-block"> 221 <a href="https://github.com/vlf-silkie/VLFeedback" target="_blank" class="external-link button is-normal is-rounded is-dark"> 222 <span class="icon"> 223 <svg class="svg-inline--fa fa-github fa-w-16" aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512" data-fa-i2svg=""><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg><!-- <i class="fab fa-github"></i> Font Awesome fontawesome.com --> 224 </span> 225 <span>Code</span> 226 </a> 227 </span> 228 <!-- <span class="link-block"> 229 <a href="http://pitt.lti.cs.cmu.edu:7890/" target="_blank" class="external-link button is-normal is-rounded is-dark"> 230 <span class="icon"> 231 <svg class="svg-inline--fa fa-images fa-w-18" aria-hidden="true" focusable="false" data-prefix="far" data-icon="images" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" data-fa-i2svg=""><path fill="currentColor" d="M480 416v16c0 26.51-21.49 48-48 48H48c-26.51 0-48-21.49-48-48V176c0-26.51 21.49-48 48-48h16v48H54a6 6 0 0 0-6 6v244a6 6 0 0 0 6 6h372a6 6 0 0 0 6-6v-10h48zm42-336H150a6 6 0 0 0-6 6v244a6 6 0 0 0 6 6h372a6 6 0 0 0 6-6V86a6 6 0 0 0-6-6zm6-48c26.51 0 48 21.49 48 48v256c0 26.51-21.49 48-48 48H144c-26.51 0-48-21.49-48-48V80c0-26.51 21.49-48 48-48h384zM264 144c0 22.091-17.909 40-40 40s-40-17.909-40-40 17.909-40 40-40 40 17.909 40 40zm-72 96l39.515-39.515c4.686-4.686 12.284-4.686 16.971 0L288 240l103.515-103.515c4.686-4.686 12.284-4.686 16.971 0L480 208v80H192v-48z"></path></svg><!-- <i class="far fa-images"></i> Font Awesome fontawesome.com --> 232 <!-- </span> --> 233 <!-- <span>Demo</span> --> 234 <!-- </a> --> 235 <!-- </span> -->
236 <span class="link-block"> 237 <a href="https://huggingface.co/datasets/MMInstruction/VLFeedback" target="_blank" class="external-link button is-normal is-rounded is-dark"> 238 <span class="icon"> 239 <svg class="svg-inline--fa fa-database fa-w-14" aria-hidden="true" focusable="false" data-prefix="fas" data-icon="database" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" data-fa-i2svg=""><path fill="currentColor" d="M448 73.143v45.714C448 159.143 347.667 192 224 192S0 159.143 0 118.857V73.143C0 32.857 100.333 0 224 0s224 32.857 224 73.143zM448 176v102.857C448 319.143 347.667 352 224 352S0 319.143 0 278.857V176c48.125 33.143 136.208 48.572 224 48.572S399.874 209.143 448 176zm0 160v102.857C448 479.143 347.667 512 224 512S0 479.143 0 438.857V336c48.125 33.143 136.208 48.572 224 48.572S399.874 369.143 448 336z"></path></svg><!-- <i class="fas fa-database"></i> Font Awesome fontawesome.com --> 240 </span> 241 <span>Dataset (VLFeedback)</span> 242 </a> 243 </span> 244 245 246 <span class="link-block"> 247 <a href="https://huggingface.co/MMInstruction/Silkie" target="_blank" class="external-link button is-normal is-rounded is-dark"> 248 <span class="icon"> 249 <svg class="svg-inline--fa fa-share-square fa-w-18" aria-hidden="true" focusable="false" data-prefix="fas" data-icon="share-square" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" data-fa-i2svg=""><path fill="currentColor" d="M568.482 177.448L424.479 313.433C409.3 327.768 384 317.14 384 295.985v-71.963c-144.575.97-205.566 35.113-164.775 171.353 4.483 14.973-12.846 26.567-25.006 17.33C155.252 383.105 120 326.488 120 269.339c0-143.937 117.599-172.5 264-173.312V24.012c0-21.174 25.317-31.768 40.479-17.448l144.003 135.988c10.02 9.463 10.028 25.425 0 34.896zM384 379.128V448H64V128h50.916a11.99 11.99 0 0 0 8.648-3.693c14.953-15.568 32.237-27.89 51.014-37.676C185.708 80.83 181.584 64 169.033 64H48C21.49 64 0 85.49 0 112v352c0 26.51 21.49 48 48 48h352c26.51 0 48-21.49 48-48v-88.806c0-8.288-8.197-14.066-16.011-11.302a71.83 71.83 0 0 1-34.189 3.377c-7.27-1.046-13.8 4.514-13.8 11.859z"></path></svg><!-- <i class="fas fa-share-square"></i> Font Awesome fontawesome.com --> 250 </span> 251 <span>Model</span> 252 </a> 253 </span> 254 255 256 <!-- <span class="link-block"> 257 <a href="#" 258 class="external-link button is-normal is-rounded is-dark"> 259 <span class="icon"> 260 <i class="fab fa-youtube"></i> 261 </span> 262 <span>Video</span> 263 </a> 264 </span> --> 265 </div> 266 </div> 267 </div> 268 </div> 269 </div> 270 </div> 271 </section> 272 273 <section class="hero teaser"> 274 <div class="container is-max-desktop"> 275 <div class="hero-body"> 276 277 <h4 class="subtitle has-text-centered" style="color:rgb(0, 4, 255)"> 278 Accepted by EMNLP 2024 Main Conference 279 </h4> 280 <h4 class="subtitle has-text-centered"> 281 <!-- LLaVA-RLHF represents the first open-source <strong>RLHF-trained</strong> large multimodal model for general-purpose visual and language understanding, achieving impressive visual reasoning and perception capabilities mimicking spirits of the multimodal GPT-4 and setting a new state-of-the-art accuracy on LLaVA-Bench, MMBench, and MMHal-Bench. <br> --> 282 <!-- We propose a new alignment algorithm called <strong>Factually Augmented RLHF (Fact-RLHF)</strong> that augments the reward model with additional factual information such as image captions and ground-truth multi-choice options, which alleviates the reward hacking phenomenon in RLHF and further improves the performance. <br> --> 283 <!-- LLaVA-RLHF combines a CLIP vision encoder and Vicuna, is fine-tuned with high-quality vision instruction tuning data and Fact-RLHF, and is presented to be <strong>more helpful</strong> and <strong>less hallucinated</strong> than LLaVA or other open-sourced LMMs. --> 284 VLFeedback is the first open-sourced <strong>GPT-4V annotated vision-language preference dataset</strong>, covering 80k instructions sampled from various sources with responses decoded from 12 large language vision models such as GPT-4V, LLaVA-series and Qwen-VL. <br> 285 Based on Qwen-VL-Chat, we present Silkie, by performing DPO on our VLFeedback. Compared with the original model, Silkile 286 <strong>achieves 6.9% and 9.5% relative improvement on the MME benchmark regarding the perception and cognition capabilities</strong>, respectively. 287 Besides, Silkie sets a new <strong>state-of-the-art score of 3.02 on MMHal-Bench</strong> regarding hallucination evaluation. 288 </h4> 289 </div> 290 </div> 291 </section> 292 293 294 295 <section class="section" style="background-color:#efeff081"> 296 <div class="container is-max-desktop"> 297 <!-- Abstract. --> 298 <div class="columns is-centered has-text-centered"> 299 <div class="column is-six-fifths"> 300 <h2 class="title is-3">Abstract</h2> 301 <div class="content has-text-justified"> 302 <p> 303 As large vision-language models (LVLMs) evolve rapidly, the demand for high-quality and diverse data to align these models becomes increasingly crucial. 304 However, the creation of such data with human supervision proves costly and time-intensive. 305 In this paper, we investigate the efficacy of AI feedback to scale supervision for aligning LVLMs. 306 We introduce VLFeedback, the first large-scale vision-language feedback dataset, comprising over 82K multi-modal instructions and comprehensive rationales generated by off-the-shelf models without human annotations. 307 To evaluate the effectiveness of AI feedback for vision-language alignment, we train Silkie, an LVLM fine-tuned via direct preference optimization on VLFeedback. 308 Silkie showcases exceptional performance regarding helpfulness, visual faithfulness, and safety metrics. It outperforms its base model by 6.9% and 9.5% in perception and cognition tasks, reduces hallucination issues on MMHal-Bench, and exhibits enhanced resilience against red-teaming attacks. 309 Furthermore, our analysis underscores the advantage of AI feedback, particularly in fostering preference diversity to deliver more comprehensive improvements. 310 <p></p> 311 312 </div> 313 </div> 314 </div> 315 316 </div> 317 </section> 318 319 320 321<section class="section"> 322 <!-- Results. --> 323 <div class="columns is-centered has-text-centered"> 324 <div class="column is-six-fifths"> 325 <h2 class="title is-3"><img id="painting_icon" width="3%" src="./LLaVA_files/5886212.png"> Multimodal Instructions and AI Preference Data</h2> 326 </div> 327 </div> 328 <!-- </div> --> 329 <!--/ Results. --> 330<div class="container is-max-desktop"> 331 332 <div class="columns is-centered"> 333 <div class="column is-full-width"> 334 <div class="content has-text-justified"> 335 <p> 336 We sample multi-modal instructions from various souces, covering different capabilities of LVLMs. We further build a model pool consisting of 12 LVLMs. </a>. 337 <centering> 338 <div style="text-align: center;"> 339 <img id="teaser" width="80%" src="./LLaVA_files/instruction_source.png"> 340 </div> 341 <br> 342 We further use GPT-4V as the annoator to assess the quality of each response regarding helpfulessn, visual faithfulness, and ethical considerations. 343 <br> 344 <br> 345 346 <div style="text-align: center;"> 347 <img id="teaser" width="90%" src="./LLaVA_files/vdpov2.png"> 348 </div> 349 <br> 350 <br> 351 (Left) The overall score distribution of three aspects. (Right) The comparison of models in our pool. 352 </centering> 353 <div style="text-align: center;"> 354 <img id="teaser" width="90%" src="./LLaVA_files/scores_of_vlm.png"> 355 </div> 356 </div> 357 </div> 358 359 360</div></section> 361 362 363<section class="section"> 364 <!-- Results. --> 365 <div class="columns is-centered has-text-centered"> 366 <div class="column is-six-fifths"> 367 <h2 class="title is-3"><img id="painting_icon" width="5%" src="./LLaVA_files/silkie.png"> Silkie: A Better Aligned LVLM </h2> 368 <!-- <h2 class="title is-3"><img id="painting_icon" width="3%" src="./LLaVA_files/silkie.png"> Silkie: A Better Aligned LVLM </h2> --> 369 </div> 370 </div> 371 <!-- </div> --> 372 <!--/ Results. --> 373<div class="container is-max-desktop"> 374 375 <div class="columns is-centered"> 376 <div class="column is-full-width"> 377 <div class="content has-text-justified"> 378 <p> 379 <!-- LLaVa-RLHF connects pre-trained <a href="https://openai.com/research/clip">CLIP ViT-L/14</a> visual encoder and large language model <a href="https://github.com/lm-sys/FastChat">Vicuna</a>, using a simple projection matrix and a LoRA module. We consider a three-stage alignment procedure: --> 380 We improve Qwen-VL-Chat by performing DPO on our VLFeedback, using the efficient LoRA tuning method. After DPO training, the resulting model Silkie achieves promising results compared with other models with similar-sized LLMs as the backbone. 381 <!-- </p><ul type="1"> 382 <li><b>Stage 1: Supervised Fine-tuning.</b> <span style="font-size: 95%;"></span> 383 <ul type="1"> 384 <li> Following LLaVA, we conduct pre-training for Feature Alignment. Only the projection matrix is updated, based on a subset of CC3M.
385 <li> Visual Chat and HQ Multimodal Instruction: LLaVA-SFT<sup>+</sup> is fine-tuned on 90k LLaVA-Instruct task, 83k VQA-v2 and 16k A-OKVQA multi-round QA task, and 23k Flickr30k Spotting Caption task. 386 </ul> 387 </li> 388 <li><b>Stage 2: Human Preference Collection & Preference Modeling. 389 </b> <span style="font-size: 95%;"> 390 <ul type="1"> 391 <li> We collect 10k human preferences where human annotators are asked to compare two responses and pinpoint the more hallucinated one. 392 </ul> 393 <li><b>Stage 3: Factually-Augmented RLHF</b>. <span style="font-size: 95%;"> 394 <ul type="1"> 395 <li> Only the LoRA module on top of LLaVA-SFT<sup>+</sup> is fine-tuned to get the Reward Model on 10k human preference data and the RL Model via reinforcement learning (PPO) from simulated human preferences. 396 </ul> 397 </span></li></ul> --> 398 <!-- Please check out our <a href="https://huggingface.co/zhiqings/LLaVA-RLHF-13b-v1.5-336">[LLaVA-RLHF-13bx336-v1.5]</a> model checkpoint. --> 399 <p></p> 400 </div> 401 <centering> 402 <div style="text-align: center;"> 403 <img id="teaser" width="90%" src="./LLaVA_files/new_ret.png"> 404 </div> 405 <br> 406 407 (Left) In-depth analysis on the MME benchmark for the performance improvements. 408 Our VLFeedback dataset brings clearer gains in OCR recognition and code reasoning tasks. 409 <br> 410 411 (Right) Relative performance improvement by performing DPO with RLHF-V preference data and a subset of our VLFeedback dataset. Our GPT-4V annotated preference dataset brings more consistent improvements on four benchmarks. 412 <br> 413 <br> 414 <div style="text-align: center;"> 415 <img id="teaser" width="95%" src="./LLaVA_files/analysis.png"> 416 </div> 417 </centering> 418 </div> 419 </div> 420 421 422</div></section> 423 424 425 426 427 428<section class="section"> 429 430 <div class="columns is-centered has-text-centered"> 431 <div class="column is-six-fifths"> 432 <h3 class="title is-3"> Qualitative Examples</h3> 433 </div> 434 </div> 435 436 <div class="columns is-centered has-text-centered"> 437 <div class="column is-six-fifths"> 438 <!-- <h2 class="title is-4">Visual Reasoning on two examples from <a href="https://arxiv.org/abs/2303.08774">OpenAI GPT-4 Technical Report</a></h2> --> 439 </div> 440 </div> 441 442 <div class="columns is-centered has-text-centered"> 443 444 <div class="column is-six-fifths"> 445 Our Silkie locates the wooden stools with a red flower without giving misleading assertions (Left), and correctly answers the scientific-related question (Right), exhibiting better perception and cognition capabilities. 446 <img id="teaser" width="70%" src="./LLaVA_files/case_study1.png"> 447 <br> 448 <br> 449 On a challenging query asking the model to generate a report for the diagram of weather forecast process, Silkie generates a well-structured report satisfying the word requirement. 450 <br> 451 <br> 452 453 454 <img id="teaser" width="70%" src="./LLaVA_files/case_study2.png"> 455 456 </div> 457 </div> 458 459 460 461 462 463 464 465 466 <div class="container mt-5"> 467 <!-- <h2 class="text-center mb-5">Who's GPT-4's favorite? Battles between State-of-the-Art Chatbots</h2> --> 468 <!-- Selection --> 469 <!-- <div class="form-row" style="justify-content: flex-end;"> 470 <div class="form-group col-md-1"> 471 <div class="col-md-2" style="width: 100%"><label> </label></div> 472 <div class="btn-group" role="group" aria-label="Left and Right Controller" style="width: 100%;align-items: flex-end;justify-content: center;flex-direction: row;display: flex;"> 473 <button type="button" class="form-control btn btn-primary" id="prev-question"><i class="material-icons">keyboard_arrow_left</i></button> 474 <button type="button" class="form-control btn btn-primary" id="next-question"><i class="material-icons">keyboard_arrow_right</i></button> 475 476 </div> 477 </div> 478 </div> --> 479 480 <!-- Question Card --> 481 <div style="display: flex; justify-content: center; align-items: center;"> 482 <div class="card mb-4" style="width: 100%; display: flex; align-items: center;"> 483 <!-- <p><b>Description:</b> Monalisa is a famous painting by Leonardo da Vinci. </p> --> 484 485 <!-- <div class="card-body" id="selected-question" style="display: flex; height: 80vh;"> --> 486 <!-- <div class="chat-history"><article class="media"><figure class="media-left"><span class="icon is-large"><svg class="svg-inline--fa fa-user fa-w-14 fa-2x" aria-hidden="true" focusable="false" data-prefix="fas" data-icon="user" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" data-fa-i2svg="">
486<path fill="currentColor" d="M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"></path></svg><i class="fas fas fa-2x fa-user "></i> Font Awesome fontawesome.com</span></figure><div class="media-content"><div class="content"><p><strong>User</strong><br><pre style="background-color: white; font-size: 18px; font-family: Arial; padding: 0px; margin: 0px; white-space: pre-wrap; overflow-wrap: break-word;"></pre><img src="./LLaVA_files/monalisa.jpg" style="max-width: 100%; max-height: 300px;"></p></div></div></article><article class="media"><figure class="media-left"><span class="icon is-large"><svg class="svg-inline--fa fa-user fa-w-14 fa-2x" aria-hidden="true" focusable="false" data-prefix="fas" data-icon="user" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" data-fa-i2svg=""><path fill="currentColor" d="M224 256c70.7 0 128-57.3 128-128S294.7 0 224 0 96 57.3 96 128s57.3 128 128 128zm89.6 32h-16.7c-22.2 10.2-46.9 16-72.9 16s-50.6-5.8-72.9-16h-16.7C60.2 288 0 348.2 0 422.4V464c0 26.5 21.5 48 48 48h352c26.5 0 48-21.5 48-48v-41.6c0-74.2-60.2-134.4-134.4-134.4z"></path></svg><i class="fas fas fa-2x fa-user "></i> Font Awesome fontawesome.com</span></figure><div class="media-content"><div class="content"><p><strong>User</strong><br><pre style="background-color: white; font-size: 18px; font-family: Arial; padding: 0px; margin: 0px; white-space: pre-wrap; overflow-wrap: break-word;">Do you know who drew this painting?â</pre></p></div></div></article><article class="media"><figure class="media-left"><span class="icon is-large"><svg class="svg-inline--fa fa-robot fa-w-20 fa-2x" aria-hidden="true" focusable="false" data-prefix="fas" data-icon="robot" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512" data-fa-i2svg=""><path fill="currentColor" d="M32,224H64V416H32A31.96166,31.96166,0,0,1,0,384V256A31.96166,31.96166,0,0,1,32,224Zm512-48V448a64.06328,64.06328,0,0,1-64,64H160a64.06328,64.06328,0,0,1-64-64V176a79.974,79.974,0,0,1,80-80H288V32a32,32,0,0,1,64,0V96H464A79.974,79.974,0,0,1,544,176ZM264,256a40,40,0,1,0-40,40A39.997,39.997,0,0,0,264,256Zm-8,128H192v32h64Zm96,0H288v32h64ZM456,256a40,40,0,1,0-40,40A39.997,39.997,0,0,0,456,256Zm-8,128H384v32h64ZM640,256V384a31.96166,31.96166,0,0,1-32,32H576V224h32A31.96166,31.96166,0,0,1,640,256Z"></path></svg><i class="fas fas fa-2x fa-robot"></i> Font Awesome fontawesome.com</span></figure><div class="media-content"><div class="content"><p><strong>LLaVA</strong><br><pre style="background-color: white; font-size: 18px; font-family: Arial; padding: 0px; margin: 0px; white-space: pre-wrap; overflow-wrap: break-word;">The painting depicts a woman, commonly believed to be Mona Lisa, the famous artwork by Leonardo da Vinci. It is a portrait painting that showcases the woman's enigmatic smile and has become one of the most famous and iconic art pieces in the world. The original work is displayed in the Louvre Museum in Paris, and it is known for its intricate details, use of oil paint, and the artist's innovative techniques that contributed to its enduring appeal and mystery.ââ</pre></p></div></div></article></div> --> 487 488 <!-- </div> --> 489 <!-- </div> --> 490 <!-- </div> --> 491 492 </div> 493</section> 494 495 <section class="section" id="BibTeX"> 496 <div class="container is-max-desktop content"> 497 <h2 class="title">BibTeX</h2> 498 <pre><code> 499@inproceedings{li-etal-2024-vlfeedback, 500 title = "{VLF}eedback: A Large-Scale {AI} Feedback Dataset for Large Vision-Language Models Alignment", 501 author = "Li, Lei and 502 Xie, Zhihui and 503 Li, Mukai and 504 Chen, Shunian and 505 Wang, Peiyi and 506 Chen, Liang and 507 Yang, Yazheng and 508 Wang, Benyou and 509 Kong, Lingpeng and 510 Liu, Qi", 511 booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing", 512 year = "2024", 513 url = "https://aclanthology.org/2024.emnlp-main.358", 514 pages = "6227--6246" 515} 516 </code></pre> 517 </div> 518 </section> 519 520 <section class="section" id="Acknowledgement"> 521 <div class="container is-max-desktop content"> 522 <h2 class="title">Acknowledgement</h2> 523 <p> 524 This website is adapted from <a href="https://github.com/nerfies/nerfies.github.io">Nerfies</a> and <a href="https://llava-rlhf.github.io/">LLaVA-RLHF</a>, licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative 525 Commons Attribution-ShareAlike 4.0 International License</a>. 526 We thank the authors of the multi-modal instruction tuning datasets and open-source projects, including LLaVA, LLaVA-RLHF and Qwen-VL. 527 We would thank <a href="https://runxinxu.github.io/aboutme/">Runxin Xu</a> for his great help on the project. 528 </p> 529 530 <p> 531<b>Usage and License Notices</b>: The data, code and checkpoint is intended and licensed for research use only. They are also restricted to uses that follow the license agreement of Qwen-VL and GPT-4. The dataset is CC BY NC 4.0 (allowing only non-commercial use) and models trained using the dataset should not be used outside of research purposes. 532</p> 533 534 <p> 535 <!-- <a href="https://github.com/Computer-Vision-in-the-Wild/"><img id="painting_icon" width="10%" src="./LLaVA_files/97258247"></a> --> 536 Related Links: 537 <a href="https://llava.hliu.cc/">[LLaVA] </a> 538 <a href="https://llava-rlhf.github.io/">[LLaVA-RLHF] </a> 539 <a href="https://github.com/QwenLM/Qwen-VL/">[Qwen-VL] </a> 540 541 </p> 542 </div> 543 </section> 544 545
545<script> 546 // Handle message showing 547 function createChatRow(sender, text, imageSrc) { 548 var article = document.createElement("article"); 549 article.className = "media" 550 551 var figure = document.createElement("figure"); 552 figure.className = "media-left"; 553 554 var span = document.createElement("span"); 555 span.className = "icon is-large"; 556 557 var icon = document.createElement("i"); 558 icon.className = "fas fas fa-2x" + (sender === "User" ? " fa-user " : sender === "LLaVA" ? " fa-robot" : ""); 559 560 var media = document.createElement("div"); 561 media.className = "media-content"; 562 563 var content = document.createElement("div"); 564 content.className = "content"; 565 566 var para = document.createElement("p"); 567 568 // wrap text in pre tag to preserve whitespace and line breaks 569 var pre_text = document.createElement("pre"); 570 pre_text.style = "background-color: white; font-size: 18px; font-family: Arial; padding: 0; margin: 0; white-space: pre-wrap; word-wrap: break-word;"; 571 var paraText = document.createTextNode(text); 572 pre_text.appendChild(paraText); 573 574 var strong = document.createElement("strong"); 575 strong.innerHTML = sender; 576 var br = document.createElement("br"); 577 578 para.appendChild(strong); 579 para.appendChild(br); 580 para.appendChild(pre_text); 581 582 // Add image if imageSrc is provided 583 if (imageSrc) { 584 var img = document.createElement("img"); 585 img.src = imageSrc; 586 img.style = "max-width: 100%; max-height: 300px;"; // Adjust the style as needed 587 para.appendChild(img); 588 } 589 590 content.appendChild(para); 591 media.appendChild(content); 592 span.appendChild(icon); 593 figure.appendChild(span); 594 if (sender !== "Description") { 595 article.appendChild(figure); 596 }; 597 article.appendChild(media); 598 return article; 599 } 600 601 function addMessageToChatHistory(sender, message, imageSrc) { 602 const chatHistory = document.querySelector('.chat-history'); 603 const chatRow = createChatRow(sender, message, imageSrc); 604 chatHistory.appendChild(chatRow); 605 chatHistory.scrollTop = chatHistory.scrollHeight; 606 } 607 608 function clearChatHistory() { 609 const chatHistory = document.querySelector('.chat-history'); 610 chatHistory.innerHTML = ""; 611 } 612 613 // 614 const conversations = [ 615 { 616 "description": "monalisa", 617 "turns": [ 618 // ["Description", "Please read the description and answer the question."], 619 ["User", "", "/static/images/monalisa.jpg"], 620 ["User", "Do you know who drew this painting?â"], 621 ["LLaVA", 622 "The painting depicts a woman, commonly believed to be Mona Lisa, the famous artwork by Leonardo da Vinci. " + 623 "It is a portrait painting that showcases the woman's enigmatic smile and has become one of the most famous and iconic art pieces in the world. " + 624 "The original work is displayed in the Louvre Museum in Paris, and it is known for its intricate details, use of oil paint, and the artist's innovative techniques that contributed to its enduring appeal and mystery.ââ" 625 ] 626 ] 627 }, 628 { 629 "description": "monalisa with dog head", 630 "turns": [ 631 ["User", "", "/static/images/monalisa-fun.jpg"], 632 ["User", "Do you know who drew this painting?â"], 633 ["LLaVA", "The painting depicts a dog in a humorous situation, where it appears to be wearing a woman's dress and standing in front of a landscape. Although the artist of the painting is not explicitly identified, it could be a reference to the famous artist Da Vinci, known for his artistic works, including the Mona Lisa. The painting is likely a creative and comical take on the traditional portrait style, replacing a human figure with a dog dressed in a woman's clothing, making it an interesting and amusing visual piece.â"] 634 ] 635 }, 636 { 637 "description": "Titanic Rose and Jack", 638 "turns": [ 639 ["User", "", "/static/images/titanic.jpg"], 640 ["User", "What's the ending of this movie?ââ"], 641 ["LLaVA", "âIn the image, a young woman is leaning on a man in front of a beautiful sunset, with both of them standing on a boat. They appear to be enjoying a romantic moment together. The scene suggests that they could be depicted as the characters Jack and Rose from the popular movie Titanic. In the mov
641ie, their romance blossoms aboard the fictional ocean liner Titanic, with the pair ultimately sharing a passionate kiss as the ship sinks into the ocean. The ending of the movie is known for its emotional and heartfelt scenes, where the characters are forced to say goodbye to each other as they are separated by the sinking ship. The image evokes a sense of romance and nostalgia, paying tribute to the iconic love story portrayed in the movie Titanic.â"] 642 ] 643 }, 644 { 645 "description": "Fridge", 646 "turns": [ 647 ["User", "", "/static/images/fridge.jpg"], 648 ["User", "What are the meals that I can cook with these?ââ"], 649 ["LLaVA", "With the variety of food items stored in the refrigerator, you can prepare several meals. For example, you can create a fruit salad using the strawberries, blueberries, and carrots. Additionally, you can make a delicious smoothie using the fruits and vegetables, which would be a healthy and nutritious snack or breakfast option. The milk and eggs can be used to create various dishes, such as a milk-based smoothie, a baked casserole, or scrambled eggs. The juice can also be used to make a fresh fruit and juice mix or as a refreshing homemade popsicle. The possibilities are vast, and the meals you can create will depend on your preferences and dietary needs.ââ"], 650 ["User", "I would like to do the fruit salad. Show me the recipe.â"], 651 ["LLaVA", 652 `Certainly! Here's a simple fruit salad recipe using the items in your refrigerator:â 653 654Ingredients:â 655- 2 cups fresh strawberries, slicedâ 656- 1 cup fresh blueberriesâ 657- 1 large carrot, gratedâ 658- Juice of half a lemonâ 659- A handful of chopped fresh parsley or mintâ 660- A few ice cubesâ 661- Optional: honey or maple syrup (adjust to taste)â 662 663Instructions:â 6641. In a large bowl, combine the sliced strawberries, blueberries, and carrot. Gently toss them together to mix.â 6652. Drizzle the juice of half a lemon over the fruit and vegetables to prevent them from oxidizing and add a refreshing citrus flavor.â 6663. Sprinkle the chopped fresh parsley or mint over the fruit and gently toss once more.â 6674. If desired, add a drizzle of honey or maple syrup for a touch of sweetness. Adjust the amount according to your taste preferences.â 6685. Cover the bowl and refrigerate the fruit salad for at least 30 minutes or up to a few hours. This allows the flavors to meld and creates a more refreshing salad.â 6696. Serve the fruit salad in cups or bowls, garnished with a sprinkle of fresh parsley or mint, if desired.â 670 671This fruit salad recipe is perfect for a healthy snack or side dish, and with the variety of ingredients available in your refrigerator, it should be easy to put together quickly.`]] 672 }, 673 { 674 "description": "Beautiful View", 675 "turns": [ 676 ["User", "", "/static/images/view.jpg"], 677 ["User", "What are the things I should be cautious about when I visit here?ââ"], 678 ["LLaVA", "When visiting this peaceful location with a pier extending over a serene lake, surrounded by mountains, there are several things to be cautious about. First, pay attention to the weather conditions, as sudden changes in weather, especially strong winds or storms, can cause the water to become rough and pose risks to the stability of the pier. Second, be mindful of the structural integrity of the pier; avoid overloading it with too many people or excessive weight, as this could potentially damage the pier or cause it to collapse. Additionally, be aware of the surrounding wildlife and respect their space; do not harm or disturb them in any way. Finally, if you decide to go fishing, ensure that you adhere to any local fishing regulations and use appropriate fishing techniques to keep the lake ecosystem healthy and maintain the balance of the food chain.ââ"], 679 ["User", "Compose an engaging travel blog post about my trip to this place, highlighting the beautiful sceneries, joyful experiences, and include possible anecdotes.ââ"], 680 ["LLaVA", 681 `Traveling to this picturesque location with a serene lake and mountains in the background was a refreshing and rejuvenating experience. The pier extending over the calm water provided an excellent vantage point from which to appreciate the breathtaking scenery. As I stood on the pier, I couldn't help but feel a deep connection with nature, taking a moment to soak in the peaceful atmosphere.â 682 683One of the highlights of my trip was undoubtedly the fishing experience. The area is teeming with life, and I was thrilled to land a sizeable catch, despite the modest size of the lake. The joy of fishing in such a beautiful setting was truly rewarding, and it allowed me to take a break from my busy life and truly immerse myself in the tranquility of nature.â 684 685While exploring the surrounding area, I came across a family of birds that had built a nest close to the pier, adding to the charm of the location. Interacting with the wildlife and observing their behavior was an enriching experience that I'll cherish for years to come.â 686
687In conclusion, this idyllic destination provided me with an opportunity to unwind and recharge, surrounded by mesmerizing natural beauty and the gentle sounds of water and wildlife. If you're looking for a serene escape from the hustle and bustle of everyday life, this pier overlooking the lake is the perfect place to find solace and reconnect with nature.` 688 ] 689 ] 690 }, 691 ]; 692 693 // The current image index 694 let currentIndex = 0; 695 696 // The function to update the displayed chat history 697 function update_dialog_demo() { 698 // Clear the chat history 699 clearChatHistory(); 700 701 for (let i = 0; i < conversations[currentIndex].turns.length; i++) { 702 if (conversations[currentIndex].turns[i].length == 2) { 703 addMessageToChatHistory(conversations[currentIndex].turns[i][0], conversations[currentIndex].turns[i][1]); 704 } 705 else { 706 addMessageToChatHistory(conversations[currentIndex].turns[i][0], conversations[currentIndex].turns[i][1], conversations[currentIndex].turns[i][2]); 707 } 708 } 709 710 // scroll to the top of the chat history 711 document.querySelector('.chat-history').scrollTop = 0; 712 } 713 714 // Initialize the displayed image 715 update_dialog_demo(); 716 717 // Event listeners for the buttons 718 document.getElementById('prev-question').addEventListener('click', () => { 719 currentIndex = (currentIndex - 1 + conversations.length) % conversations.length; 720 update_dialog_demo(); 721 }); 722 723 document.getElementById('next-question').addEventListener('click', () => { 724 currentIndex = (currentIndex + 1) % conversations.length; 725 update_dialog_demo(); 726 }); 727 728 729 </script>
729 730 731 732 733</body><grammarly-desktop-integration data-grammarly-shadow-root="true"><template shadowrootmode="open"><style> 734 div.grammarly-desktop-integration { 735 position: absolute; 736 width: 1px; 737 height: 1px; 738 padding: 0; 739 margin: -1px; 740 overflow: hidden; 741 clip: rect(0, 0, 0, 0); 742 white-space: nowrap; 743 border: 0; 744 -moz-user-select: none; 745 -webkit-user-select: none; 746 -ms-user-select:none; 747 user-select:none; 748 } 749 750 div.grammarly-desktop-integration:before { 751 content: attr(data-content); 752 } 753 </style><div aria-label="grammarly-integration" role="group" tabindex="-1" class="grammarly-desktop-integration" data-content="{"mode":"full","isActive":true,"isUserDisabled":false}"></div></template></grammarly-desktop-integration></html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.