1<!DOCTYPE html> 2<html> 3<head> 4 <meta charset="utf-8"> 5 <!-- Meta tags for social media banners, these should be filled in appropriatly as they are your "business card" --> 6 <!-- Replace the content tag with appropriate information --> 7 <meta name="description" content="DESCRIPTION META TAG"> 8 <meta property="og:title" content="SOCIAL MEDIA TITLE TAG"/> 9 <meta property="og:description" content="SOCIAL MEDIA DESCRIPTION TAG TAG"/> 10 <meta property="og:url" content="URL OF THE WEBSITE"/> 11 <!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X630--> 12 <meta property="og:image" content="static/image/your_banner_image.png" /> 13 <meta property="og:image:width" content="1200"/> 14 <meta property="og:image:height" content="630"/> 15 16 17 <meta name="twitter:title" content="TWITTER BANNER TITLE META TAG"> 18 <meta name="twitter:description" content="TWITTER BANNER DESCRIPTION META TAG"> 19 <!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X600--> 20 <meta name="twitter:image" content="static/images/your_twitter_banner_image.png"> 21 <meta name="twitter:card" content="summary_large_image"> 22 <!-- Keywords for your paper to be indexed by--> 23 <meta name="keywords" content="KEYWORDS SHOULD BE PLACED HERE"> 24 <meta name="viewport" content="width=device-width, initial-scale=1"> 25 26 27 <title>Adaptive Image Quality Assessment via Teaching Large Multimodal Model to Compare</title> 28 <link rel="icon" type="image/x-icon" href="static/images/logo.pic.jpg"> 29 <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro" 30 rel="stylesheet"> 31 32 <link rel="stylesheet" href="static/css/bulma.min.css"> 33 <link rel="stylesheet" href="static/css/bulma-carousel.min.css"> 34 <link rel="stylesheet" href="static/css/bulma-slider.min.css"> 35 <link rel="stylesheet" href="static/css/fontawesome.all.min.css"> 36 <link rel="stylesheet" 37 href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css"> 38 <link rel="stylesheet" href="static/css/index.css"> 39 40
40<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
40 41
41<script src="https://documentcloud.adobe.com/view-sdk/main.js"></script>
41 42
42<script defer src="static/js/fontawesome.all.min.js"></script>
42 43
43<script src="static/js/bulma-carousel.min.js"></script>
43 44
44<script src="static/js/bulma-slider.min.js"></script>
44 45
45<script src="static/js/index.js"></script>
45 46
46<script type="module" src="https://gradio.s3-us-west-2.amazonaws.com/4.12.0/gradio.js"></script>
46 47<style> 48 .darkblue { 49 color: darkblue; 50 } 51 pre { 52 white-space: pre-wrap; /* Since CSS might be used, ensure that white-space is preserved */ 53 } 54</style> 55</head> 56<body> 57 58 59 <section class="hero"> 60 <div class="hero-body"> 61 <div class="container is-max-desktop"> 62 <div class="columns is-centered"> 63 <div class="column has-text-centered"> 64 <h1 class="title is-1 publication-title">Adaptive Image Quality Assessment via Teaching Large Multimodal Model to Compare</h1> 65 <div class="is-size-5 publication-authors"> 66<!-- Paper authors --> 67<span class="author-block"> 68 <a href="https://github.com/h4nwei" target="_blank">Hanwei Zhu</a><sup>1*</sup>, 69</span> 70<span class="author-block"> 71 <a href="https://teowu.github.io" target="_blank">Haoning Wu</a><sup>2*</sup>, 72</span> 73<span class="author-block"> 74 <a href="https://compare2score.github.com" target="_blank">Yixuan Li</a><sup>1</sup>, 75</span> 76<span class="author-block"> 77 <a href="https://github.com/zzc-1998" target="_blank">Zicheng Zhang</a><sup>3</sup>, 78</span> 79<span class="author-block"> 80 <a href="https://scholar.google.com/citations?hl=en&user=w_WL27oAAAAJ&view_op=list_works" target="_blank">Baoliang Chen</a><sup>1</sup>, 81</span> 82<span class="author-block"> 83 <a href="https://scholar.google.com/citations?hl=en&user=IhyTEDkAAAAJ" target="_blank">Lingyu Zhu</a><sup>1</sup>, 84</span> 85<span class="author-block"> 86 <a href="http://sim.jxufe.cn/JDMKL/ymfang_EN.html/" target="_blank">Yuming Fang</a><sup>4</sup>, 87</span> 88<span class="author-block"> 89 <a href="https://ee.sjtu.edu.cn/en/FacultyDetail.aspx?id=24&infoid=153&flag=153" target="_blank">Guangtao Zhai</a><sup>3</sup>, 90</span> 91<span class="author-block"> 92 <a href="https://personal.ntu.edu.sg/wslin/Home.html" target="_blank">Weisi Lin</a><sup>2</sup>, 93</span> 94<span class="author-block"> 95 <a href="https://www.cs.cityu.edu.hk/~shiqwang/" target="_blank">Shiqi Wang</a><sup>1</sup> 96</span> 97</div> 98 99 <div class="is-size-5 publication-authors"> 100 <span class="author-block"><sup>1</sup>City University of Hong Kong</span> 101 <span class="author-block"><sup>2</sup>Nanyang Technological University</span> 102 <span class="author-block"><sup>3</sup>Shanghai Jiao Tong University</span> 103 <span class="author-block"><sup>4</sup>Jiangxi University of Finance and Economics</span> 104 <span class="eql-cntrb"><small><br><sup>*</sup>Equal Contribution.</small></span> 105 <!-- <span class="eql-cntrb"><small><br><sup>#</sup>Equal Contribution.</small></span> --> 106 </div> 107 108 <div class="column has-text-centered"> 109 110 <div class="publication-links"> 111 <!-- Arxiv PDF link --> 112 <span class="link-block"> 113 <a href="https://arxiv.org/abs/2405.19298" target="_blank" 114 class="external-link button is-normal is-rounded is-dark"> 115 <span class="icon"> 116 <i class="fas fa-file-pdf"></i> 117 </span> 118 <span>ArXiv</span> 119 </a> 120 </span> 121 122 <!-- huggingface --> 123 <span class="link-block"> 124 <a href="https://huggingface.co/q-future/Compare2Score" target="_blank" 125 class="external-link button is-normal is-rounded is-dark"> 126 <span class="icon"> 127 <i class="fas fa-smile"></i> 128 </span> 129 <span>Compare2Score</span> 130 </a> 131 </span> 132 133 134 <!-- Github link --> 135 <span class="link-block"> 136 <a href="https://github.com/Q-Future/Compare2Score" target="_blank" 137 class="external-link button is-normal is-rounded is-dark"> 138 <span class="icon"> 139 <i class="fab fa-github"></i> 140 </span>
141 <span>Code</span> 142 </a> 143 </span> 144 145 <!-- huggingface --> 146 <!-- <span class="link-block"> 147 <a href="https://huggingface.co/spaces/q-future/Compare2Score" target="_blank" 148 class="external-link button is-normal is-rounded is-dark"> 149 <span class="icon"> 150 <i class="fas fa-smile"></i> 151 </span> 152 <span>HF Demo</span> 153 </a> 154 </span> --> 155 156 </div> 157 </div> 158 </div> 159 </div> 160 </div> 161 </div> 162</section> 163 164 165<!-- <section class="section" style="background-color:#efeff081"> 166 <div class="container is-max-desktop" id="gradio"> 167 <gradio-app src="https://q-future-compare2score.hf.space/"></gradio-app> 168 </div> 169</section> --> 170 171 172<!-- Paper abstract --> 173<section class="section hero is-light"> 174 <div class="container is-max-desktop"> 175 <div class="columns is-centered has-text-centered"> 176 <div class="column is-four-fifths"> 177 <h2 class="title is-3">Abstract</h2> 178 <div class="content has-text-justified"> 179 <p> 180 While recent advancements in large multimodal models (LMMs) have significantly improved their abilities in image quality assessment (IQA) relying on absolute quality rating, how to transfer reliable relative quality comparison outputs to continuous perceptual quality scores remains largely unexplored. To address this gap, we introduce <strong>Compare2Score</strong>âan all-around LMM-based no-reference IQA~(NR-IQA) model, which is capable of producing qualitatively comparative responses and effectively translating these discrete comparative levels into a continuous quality score. Specifically, during training, we present to generate scaled-up comparative instructions by comparing images from the same IQA dataset, allowing for more flexible integration of diverse IQA datasets. Utilizing the established large-scale training corpus, we develop a human-like visual quality comparator. During inference, moving beyond binary choices, we propose a soft comparison method that calculates the likelihood of the test image being preferred over multiple predefined anchor images. The quality score is further optimized by maximum a posteriori estimation with the resulting probability matrix. Extensive experiments on nine IQA datasets validate that the <strong>Compare2Score</strong> effectively bridges text-defined comparative levels during training with converted single image quality score for inference, surpassing state-of-the-art IQA models across diverse scenarios. Moreover, we verify that the probability-matrix-based inference conversion not only improves the rating accuracy of <strong>Compare2Score</strong> but also zero-shot general-purpose LMMs, suggesting its intrinsic effectiveness. 181 </p> 182 </div> 183 <!-- <img src="static/images/teaser.png" , width="800" /> --> 184 </div> 185 </div> 186 </div> 187</section> 188<!-- End paper abstract --> 189 190 191 192<!-- Teaser --> 193 <section class="hero is-small is-light"> 194 <div class="hero-body"> 195 <div class="columns is-centered has-text-centered"> 196 <div class="column is-three-fifths"> 197 <h5 class="title">Motivation</h5> 198 <div class="content has-text-justified"> 199 <p> 200 Illustrations of the motivation of this work. <strong>(a)</strong> Images with identical rescaled MOS from various IQA datasets exhibit significant variations in perceptual quality. <strong>(b)</strong> Images that cluster at the same rating level from different IQA datasets display mismatches due to differing subjective testing methodologies. <strong>(c)</strong> By comparing MOSs within the same dataset, it facilitates the flexible combination of multiple IQA datasets. 201 </p> 202 </div> 203 <img src="static/images/motivation.png" , width="1400" /> 204 </div> 205 </div> 206 </div> 207 </section> 208 209 <!-- Teaser --> 210 <section class="hero is-small is-light"> 211 <div class="hero-body"> 212 <div class="columns is-centered has-text-centered"> 213 <div class="column is-three-fifths"> 214 <h5 class="title">A repurposed training dataset.</strong></h5> 215 <div class="content has-text-justified"> 216 <p> 217 We present to generate scaled-up comparative instructions by comparing MOSs of images within each IQA dataset, 218 which allows for more flexible integration of diverse IQA datasets. Specifically, the approach simulates subjective 219 testing by posing the question, ``Compared with the first image, how is the quality of the second image?â. Responses are then generated based on the MOS comparisons of the image pairs. 220 Using the empirical rule, we categorize the image pairs into five distinct comparative levels: inferior, 221 worse, similar, better, superior. This method produces a comprehensive training 222 dataset that enables the LMM to effectively handle various distortion scenarios, resulting in a human-like 223 visual quality comparator. 224 </p> 225 </div> 226 <img src="static/images/framework.png" , width="1400" /> 227 </div> 228 </div> 229 </div> 230 </section> 231 232 <!-- Teaser --> 233 <section class="hero is-small is-light"> 234 <div class="hero-body"> 235 <div class="columns is-centered has-text-centered"> 236 <div class="column is-three-fifths"> 237 <h5 class="title">An inference conversion strategy</h5> 238 <div class="content has-text-justified"> 239 <p> 240 We develop an adaptive soft comparison scheme that efficiently translates discrete comparative levels into continuous 241 quality scores. Unlike traditional two-alternative forced choice (2AFC) methods, our approach calculates the 242 likelihood that an input image is preferred over multiple anchor images. This probability is derived from 243 a weighted summation of the softmax-transformed log probabilities across five comparative levels. Subsequently, 244 the quality score of the input image is calculated through maximum a posteriori (MAP) estimation based on the 245 resulting probability matrix. 246 </p> 247 </div> 248 <img src="static/images/dataset.png" , width="900" /> 249 </div> 250 </div> 251 </div> 252 </section> 253 254 <!-- Teaser --> 255 <!-- <section class="hero is-small is-light"> 256 <div class="hero-body"> 257 <div class="columns is-centered has-text-centered"> 258 <div class="column is-three-fifths"> 259 <h5 class="title">A state-of-the-art framework</h5> 260 <div class="content has-text-justified"> 261 <p> 262 We conduct extensive experiments to validate the effectiveness of teaching the relative quality ranking knowledge to LMM. 263 The proposed model, namely Compare2Score, consistently outperforms state-of-the-art NR-IQA models on both synthetic and 264 realistic distortions and shows enhanced generalization capability across different cross-distortion scenarios. 265 Furthermore, we demonstrate that the probability matrix-based inference conversion significantly enhances the rating accuracy 266 of Compare2Score and extends these improvements to zero-shot general-purpose LMMs. 267 </p> 268 </div> 269 <img src="static/images/micbench.png" , width="900" /> 270 </div> 271 </div> 272 </div> 273 </section> --> 274 275 <!-- Image carousel --> 276<section class="hero is-small"> 277 <div class="hero-body"> 278 <div class="container"> 279 <div class="columns is-centered has-text-centered"> 280 <div class="column is-three-fifths"> 281 <h5 class="title">Experiments</h5> 282 <div id="results-carousel" class="carousel results-carousel"> 283 <div class="item"> 284 <!-- Your image here --> 285 <img src="static/images/MainResult.png" alt="MY ALT TEXT"/> 286 <h2 class="subtitle has-text-centered"> 287 Performance comparison in terms of median SRCC and PLCC on six IQA datasets under the intra-dataset setup. 288 </h2> 289 </div> 290 <div class="item"> 291 <!-- Your image here --> 292 <img src="static/images/CrossDataset.png" alt="MY ALT TEXT"/> 293 <h2 class="subtitle has-text-centered"> 294 SRCC results on the three IQA datasets under the cross-dataset setup. 295 </h2> 296 </div> 297 <div class="item"> 298 <!-- Your image here --> 299 <img src="static/images/SoftComparison.png" alt="MY ALT TEXT"/> 300 <h2 class="subtitle has-text-centered"> 301 SRCC results of probability matrix and count matrix on four IQA datasets. Prob. stands for probability. 302 </h2>
303 </div> 304 <div class="item"> 305 <!-- Your image here --> 306 <img src="static/images/Accuracy.png" alt="MY ALT TEXT"/> 307 <h2 class="subtitle has-text-centered"> 308 Performance comparison in terms of prediction accuracy on six IQA datasets. 309 </h2> 310 </div> 311 </div> 312 </div> 313 </div> 314</div> 315</div> 316</section> 317<!-- End image carousel --> 318 319<!--BibTex citation --> 320 <section class="section" id="BibTeX"> 321 <div class="container is-max-desktop content"> 322 <h2 class="title">BibTeX</h2> 323 <pre><code>@misc{zhu2024adaptive, 324 title={Adaptive Image Quality Assessment via Teaching Large Multimodal Model to Compare}, 325 author={Hanwei Zhu and Haoning Wu and Yixuan Li and Zicheng Zhang and Baoliang Chen and Lingyu Zhu and Yuming Fang and Guangtao Zhai and Weisi Lin and Shiqi Wang}, 326 year={2024}, 327 eprint={2405.19298}, 328 archivePrefix={arXiv}, 329 primaryClass={cs.CV} 330} 331</code></pre> 332 </div> 333</section> 334<!--End BibTex citation --> 335 336 337 <footer class="footer"> 338 <div class="container"> 339 <div class="columns is-centered"> 340 <div class="column is-8"> 341 <div class="content"> 342 343 <p> 344 This page was built using the <a href="https://github.com/eliahuhorwitz/Academic-project-page-template" target="_blank">Academic Project Page Template</a>. 345 <br> This website is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/" target="_blank">Creative 346 Commons Attribution-ShareAlike 4.0 International License</a>. 347 </p> 348 349 </div> 350 </div> 351 </div> 352 </div> 353</footer> 354 355<!-- Statcounter tracking code --> 356 357<!-- You can add a tracker to track page visits by creating an account at statcounter.com --> 358 359 <!-- End of Statcounter Code --> 360 361 </body> 362 </html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.