1<!DOCTYPE html> 2<html> 3 4<head> 5 <meta charset="utf-8"> 6<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> 7<meta http-equiv="X-UA-Compatible" content="IE=edge"> 8 9<title>VideoLLaMB</title> 10<meta name="description" content="VideoLLaMB: Long Video Understanding with Recurrent Memory Bridges"> 11 12<!-- Open Graph --> 13 14 15<!-- Bulma & MDB --> 16<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/css/bulma.min.css"> 17<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bulma-carousel.min.css"> 18<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bulma-slider.min.css"> 19 20<!-- Fonts & Icons --> 21<link rel="stylesheet" 22 href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/all.min.css" 23 integrity="sha512-z3gLpd7yknf1YoNbCzqRKc4qyor8gaKU1qmn+CShxbuBusANI9QpRohGBreCFkKxLhei6S9CQXFEbbKuqLg0DA==" crossorigin="anonymous"> 24<link rel="stylesheet" 25 href="https://cdnjs.cloudflare.com/ajax/libs/academicons/1.9.4/css/academicons.min.css" 26 crossorigin="anonymous"> 27<link rel="stylesheet" type="text/css" 28 href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Roboto+Slab:100,300,400,500,700|Lora|Google+Sans|Material+Icons"> 29 30<!-- Code Syntax Highlighting --> 31<!-- <link rel="stylesheet" href="https://gitcdn.link/repo/jwarby/jekyll-pygments-themes/master/.css" />
31 --> 32 33<!-- Styles --> 34<link rel="shortcut icon" href="/assets/img/favicon.ico"> 35<link rel="stylesheet" href="/assets/css/main.css"> 36 37<link rel="canonical" href="/"> 38 39<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism-solarizedlight.min.css"/> 40 41 42<!-- Theming--> 43 44 45 46<!-- Global site tag (gtag.js) - Google Analytics -->
vendor: 64 bytes, lines 46-47
46 47<script async src="https://www.googletagmanager.com/gtag/js?id=
47G-XD4J271NBT
vendor: 12 bytes, line 47
47"></script>
48<script> 49
vendor: 144 bytes, lines 49-53
49window.dataLayer = window.dataLayer || []; 50 function gtag () { dataLayer.push(arguments); } 51 gtag('js', new Date()); 52 53 gtag('config', '
53G-XD4J271NBT
vendor: 4 bytes, line 53
53');
54</script>
54 55 56 57 58 59 60 61<!-- MathJax -->
62<script type="text/javascript"> 63 window.MathJax = { 64 tex: { 65 tags: 'ams' 66 } 67 }; 68</script>
vendor: 1 bytes, line 68
68
69<script defer type="text/javascript" id="MathJax-script" 70 src="https://cdn.jsdelivr.net/npm/[email protected]/es5/tex-mml-chtml.js"></script>
vendor: 1 bytes, line 70
70
71<script defer src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?features=es6"></script>
71 72 73</head> 74 75<body> 76 <!-- Header --> 77 78 79<nav class="navbar" role="navigation" aria-label="main navigation"> 80 <div class="navbar-brand"> 81 <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false"> 82 <span aria-hidden="true"></span> 83 <span aria-hidden="true"></span> 84 <span aria-hidden="true"></span> 85 </a> 86 </div> 87 <div class="navbar-menu"> 88 <div class="navbar-start" style="flex-grow: 1; justify-content: center;"> 89 <a class="navbar-item" href="https://videollamb.github.io"> 90 <span class="icon"> 91 <i class="fas fa-home"></i> 92 </span> 93 </a> 94 95 <div class="navbar-item has-dropdown is-hoverable"> 96 <a class="navbar-link"> 97 More Research 98 </a> 99 <div class="navbar-dropdown"> 100 101 <a class="navbar-item" href="https://bigai-nlco.github.io/ExoViP/"> 102 ExoViP 103 </a> 104 105 <a class="navbar-item" href="https://videohallucer.github.io/"> 106 VideoHallucer 107 </a> 108 109 <a class="navbar-item" href="https://github.com/bigai-nlco/LSTP-Chat"> 110 LSTPChat 111 </a> 112 113 <a class="navbar-item" href="https://vstar-benchmark.github.io/"> 114 VSTAR 115 </a> 116 117 </div> 118 </div> 119 </div> 120 121 </div> 122</nav> 123 124 125 <!-- Content --> 126 127 <div class="container is-fullhd mt-5"> 128 <section class="hero"> 129 <div class="hero-body"> 130 <div class="container is-max-desktop"> 131 <div class="columns is-centered"> 132 <div class="column has-text-centered"> 133 <div style="display: flex;padding-bottom: 20px;"> 134 <img src="/assets/img/video-llamb.png" 135 width="100px" style="vertical-align: middle;"> 136 <h1 class="title is-2 publication-title"><span style="display: block;">VideoLLaMB: Long-Context Video Understanding with Recurrent Memory Bridges 137</span></h1> 138 </div> 139 140 141 142 <div class="is-size-5 publication-authors"> 143 144 <span class="author-block"><a href="https://patrick-tssn.github.io/">Yuxuan Wang</a><sup>1</sup></span>, 145 146 <span class="author-block"><a href="https://cihangxie.github.io/">Cihang Xie</a><sup>2</sup></span>, 147 148 <span class="author-block"><a href="http://www.csyangliu.com/">Yang Liu</a><sup>3</sup></span>, 149 150 <span class="author-block"><a href="https://zilongzheng.github.io">Zilong Zheng</a><sup>1, <i class="fa fa-envelope"></i></sup></span> 151 152 </div> 153 154 155 156 <div class="is-size-5 publication-authors"> 157 158 <span class="author-block"><sup>1</sup>BIGAI</span>, 159 160 <span class="author-block"><sup>2</sup>UCSC</span>, 161 162 <span class="author-block"><sup>3</sup>PKU</span> 163 164 </div> 165 166 167 168 169 <div class="column has-text-centered"> 170 <div class="publication-links"> 171 172
173 <span class="link-block"> 174 <a href="https://arxiv.org/abs/2409.01071" 175 class="external-link button is-normal is-rounded is-dark"> 176 <span class="icon"> 177 <i class="ai ai-arxiv"></i> 178 </span> 179 <span>arXiv</span> 180 </a> 181 </span> 182 183 184 185 <!-- Code Link. --> 186 <span class="link-block"> 187 <a href="https://github.com/bigai-nlco/VideoLLaMB" 188 class="external-link button is-normal is-rounded is-dark"> 189 <span class="icon"> 190 <i class="fab fa-github"></i> 191 </span> 192 <span>Code</span> 193 </a> 194 </span> 195 196 197 198 199 <span class="link-block"> 200 <a href=" https://huggingface.co/ColorfulAI/VideoLLaMB" 201 class="external-link button is-normal is-rounded is-dark"> 202 <span class="icon"> 203 🤗 204 </span> 205 <span>VideoLLaMB-7B</span> 206 </a> 207 </span> 208 209 <span class="link-block"> 210 <a href=" https://github.com/bigai-nlco/NeedleInAVideoHaystack" 211 class="external-link button is-normal is-rounded is-dark"> 212 <span class="icon"> 213 <i class="fab fa-github"></i> 214 </span> 215 <span>MM-NIAVH</span> 216 </a> 217 </span> 218 219 <span class="link-block"> 220 <a href=" " 221 class="external-link button is-normal is-rounded is-dark"> 222 <span class="icon"> 223 <i class="fas fa-globe"></i> 224 </span> 225 <span>Demo (Coming Soon)</span> 226 </a> 227 </span> 228 229 230 </div> 231 </div> 232 </div> 233 </div> 234 </div> 235 </div> 236 </section> 237 238 <section class="hero teaser"> 239 <div class="container is-max-desktop"> 240 <div class="hero-body"> 241<div class="columns is-centered has-text-centered"> 242 243<div class="column"> 244<figure class="image"> 245 <figcaption><span class="dnerf">VideoLLaMB</span> </figcaption> 246 <img src="/assets/img/videollamb_niavh.png" /> 247</figure> 248 249 250<figure class="image"> 251 <figcaption><span class="dnerf">VideoLLaMB w/o Memory Retrieval</span></figcaption> 252 <img src="/assets/img/videollamb_niavh_nor.png" /> 253</figure> 254 255</div> 256 257<div class="column"> 258<figure class="image">
259 <figcaption><span class="dnerf">LongVA (<a href="https://github.com/EvolvingLMMs-Lab/LongVA">Zhang et. al., 2024</a>)</span></figcaption> 260 <img src="/assets/img/longva.png" /> 261</figure> 262 263<figure class="image"> 264 <figcaption><span class="dnerf">MA-LLM (<a href="https://github.com/boheumd/MA-LMM">He et. al., 2024</a>)</span></figcaption> 265 <img src="/assets/img/mallm.png" /> 266</figure> 267 268</div> 269 270 271</div> 272 273 274<details closed=""> 275<summary><b>More comparisons</b></summary> 276<div class="columns is-centered has-text-centered"> 277 278<div class="column"> 279<figure class="image"> 280 <figcaption><span class="dnerf"><a href="https://huggingface.co/ermu2001/pllava-7b">PLLaVA-7B</a> (Run 6/5/2024)</span> </figcaption> 281 <img src="/assets/img/pllava.png" /> 282</figure> 283 284</div> 285 286<div class="column"> 287<figure class="image"> 288 <figcaption><span class="dnerf"><a href="https://huggingface.co/lmms-lab/LLaVA-NeXT-Video-7B-DPO">LLaVA-NeXT-Video-DPO-7B</a> (Run 6/5/2024) </span></figcaption> 289 <img src="/assets/img/llavanext.png" /> 290</figure> 291 292</div> 293 294 295</div> 296</details> 297 298<figcaption style="padding-top:10px;"><span class="dnerf">Figure 1.</span> <b>Comparison of long video understanding models on <a href="#stress-test-needle-in-a-video-haystack">Needle In a Video Haystack (NIAVH)</a>.</b> We set the context length to 320 seconds due to existing models' ability and set the frame rate to 1 fps to ensure the input contains the needle. The X-axis indicates the video length, and the Y-axis is the depth of the insertion point.</figcaption> 299 300 </div> 301 </div> 302</section> 303 304<section class="section"> 305 <div class="container is-max-desktop"> 306 307 <h2 class="title has-text-centered" id="abstract">Abstract</h2> 308 309 <p>VideoLLaMB is a novel long video comprehension framework utilizing Memory Bridge Layers with recurrent memory tokens to encode 100% video content without discarding critical visual cues.</p> 310 311 <p>⨠Highlights:</p> 312 313 <ol> 314 <li> 315 <p><strong>Comprehensive long video understanding.</strong> VideoLLaMB-7B reached the state-of-the-art performance among 7B models trained on vicuna-7b and videochat2 video on <a href="https://egoschema.github.io/">EgoSchema</a>, <a href="https://github.com/doc-doc/NExT-QA">NexTQA</a> and <a href="https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md">MVBench</a>, reaching <em>8x longer video length</em> with robust performance in comparison to <a href="https://pllava.github.io/">PLLaVA</a>.</p> 316 </li> 317 <li> 318 <p><strong>Memory-based egocentric planning.</strong> VideoLLaMB achieves the best performance among all video-language models on <a href="https://github.com/ChenYi99/EgoPlan">EgoPlan</a>, with an improvement of \(2.06\) over PLLaVA.</p> 319 </li> 320 <li> 321 <p><strong>Training-free streaming captioning.</strong> With our <a href="#scene-tiling-segmentation-with-semantics">SceneTiling algorithm</a>, VideoLLaMB can capture the dynamics with in a streaming video and directly predict the streaming captions in real-time, without the need to process the entire video sequence beforehand.</p> 322 </li> 323 <li> 324 <p><strong>Enhanced frame retrieval on needle in a video haystack (NIAVH).</strong> We present the âNeedle in a Video Haystackâ (NIAVH) benchmark to evaluate long video understanding over needle of different modalities comprehensively (<a href="#stress-test-needle-in-a-video-haystack">details ð</a>). In the pressure test ranging from 1 to 300 seconds in length, VideoLLaMB consistently retrieves the correct image needles at various depths, outperforming other methods as video length increases.</p> 325 </li> 326 </ol> 327 328 </div> 329 330<div class="columns is-centered has-text-centered"> 331 332<div class="column is-four-fifths"> 333<figure class="image"> 334 <img src="/assets/img/framework_new.png" /> 335 <figcaption><span class="dnerf">Figure 2.</span> <b>An overview of VideoLLaMB.</b> <a href="#technical-details">Technical details.</a> </figcaption> 336</figure> 337 338</div> 339 340</div> 341 342 343</section> 344 345<section class="section" style="background-color:#efeff081"> 346 <div class="container"> 347 348 <h2 class="title is-3 has-text-centered" id="long-form-video-understanding">Long-form Video Understanding</h2> 349 350 <div class="table-container has-text-centered" style="font-size:15px"> 351 352 <table class="table is-fullwidth is-narrow is-hoverable"> 353 <thead> 354 <tr> 355 <th style="text-align: left"><strong>Method</strong></th> 356 <th style="text-align: center"><strong>Vision Encoder</strong></th> 357 <th style="text-align: center"><strong>LLM Size</strong></th> 358 <th><strong>AS</strong></th> 359 <th><strong>AP</strong></th> 360 <th><strong>AA</strong></th> 361 <th><strong>FA</strong></th> 362 <th><strong>UA</strong></th> 363 <th><strong>OE</strong></th> 364 <th><strong>OI</strong></th> 365 <th><strong>OS</strong></th> 366 <th><strong>MD</strong></th> 367 <th><strong>AL</strong></th> 368 <th><strong>ST</strong></th> 369 <th><strong>AC</strong></th> 370 <th><strong>MC</strong></th> 371 <th><strong>MA</strong></th> 372 <th><strong>SC</strong></th> 373 <th><strong>FP</strong></th> 374 <th><strong>
374CO</strong></th> 375 <th><strong>EN</strong></th> 376 <th><strong>ER</strong></th> 377 <th><strong>CI</strong></th> 378 <th><strong>Avg.</strong></th> 379 </tr> 380 </thead> 381 <tbody> 382 <tr> 383 <td style="text-align: left">GPT-4V</td> 384 <td style="text-align: center">GPT-4V</td> 385 <td style="text-align: center">/</td> 386 <td>55.5</td> 387 <td>63.5</td> 388 <td>72.0</td> 389 <td>46.5</td> 390 <td>73.5</td> 391 <td>18.5</td> 392 <td>59.0</td> 393 <td>29.5</td> 394 <td>12.0</td> 395 <td>40.5</td> 396 <td>83.5</td> 397 <td>39.0</td> 398 <td>12.0</td> 399 <td>22.5</td> 400 <td>45.0</td> 401 <td>47.5</td> 402 <td>52.0</td> 403 <td>31.0</td> 404 <td>59.0</td> 405 <td>11.0</td> 406 <td>43.5</td> 407 </tr> 408 <tr> 409 <td style="text-align: left"><em>Image MLLMs</em></td> 410 <td style="text-align: center">Â </td> 411 <td style="text-align: center">Â </td> 412 <td>Â </td> 413 <td>Â </td> 414 <td>Â </td> 415 <td>Â </td> 416 <td>Â </td> 417 <td>Â </td> 418 <td>Â </td> 419 <td>Â </td> 420 <td>Â </td> 421 <td>Â </td> 422 <td>Â </td> 423 <td>Â </td> 424 <td>Â </td> 425 <td>Â </td> 426 <td>Â </td> 427 <td>Â </td> 428 <td>Â </td> 429 <td>Â </td> 430 <td>Â </td> 431 <td>Â </td> 432 <td>Â </td> 433 </tr> 434 <tr> 435 <td style="text-align: left">mPLUG-Owl-I</td> 436 <td style="text-align: center">ViT-L</td> 437 <td style="text-align: center">7B</td> 438 <td>25.0</td> 439 <td>20.0</td> 440 <td>44.5</td> 441 <td>27.0</td> 442 <td>23.5</td> 443 <td>36.0</td> 444 <td>24.0</td> 445 <td>34.0</td> 446 <td>23.0</td> 447 <td>24.0</td> 448 <td>34.5</td> 449 <td>34.5</td> 450 <td>22.0</td> 451 <td>31.5</td> 452 <td>40.0</td> 453 <td>24.0</td> 454 <td>37.0</td> 455 <td>25.5</td> 456 <td>21.0</td> 457 <td>37.0</td> 458 <td>29.4</td> 459 </tr> 460 <tr> 461 <td style="text-align: left">LLaMA-Adapter</td> 462 <td style="text-align: center">ViT-B</td> 463 <td style="text-align: center">7B</td> 464 <td>23.0</td> 465 <td>28.0</td> 466 <td>51.0</td> 467 <td>30.0</td> 468 <td>33.0</td> 469 <td>53.5</td> 470 <td>32.5</td> 471 <td>33.5</td> 472 <td>25.5</td> 473 <td>21.5</td> 474 <td>30.5</td> 475 <td>29.0</td> 476 <td>22.5</td> 477 <td>41.5</td> 478 <td>39.5</td> 479 <td>25.0</td> 480 <td>31.5</td> 481 <td>22.5</td> 482 <td>28.0</td> 483 <td>32.0</td> 484 <td>31.7</td> 485 </tr> 486 <tr> 487 <td style="text-align: left">BLIP2</td> 488 <td style="text-align: center">ViT-G</td> 489 <td style="text-align: center">2.7B</td> 490 <td>24.5</td> 491 <td>29.0</td> 492 <td>33.5</td> 493 <td>17.0</td> 494 <td>42.0</td> 495 <td>51.5</td> 496 <td>26.0</td> 497 <td>31.0</td> 498 <td>25.5</td> 499 <td>26.0</td> 500 <td>32.5</td> 501 <td>25.5</td> 502 <td>30.0</td> 503 <td>40.0</td> 504 <td>42.0</td> 505 <td>27.0</td> 506 <td>30.0</td> 507 <td>26.0</td> 508 <td>37.0</td> 509 <td>31.0</td> 510 <td>31.4</td> 511 </tr> 512 <tr> 513 <td style="text-align: left">Otter-I</td> 514 <td style="text-align: center">ViT-L</td> 515 <td style="text-align: center">7B</td> 516 <td>34.5</td> 517 <td>32.0</td> 518 <td>39.5</td> 519 <td>30.5</td> 520 <td>38.5</td> 521 <td>48.5</td> 522 <td>44.0</td> 523 <td>29.5</td> 524 <td>19.0</td> 525 <td>25.5</td> 526 <td>55.0</td> 527 <td>20.0</td> 528 <td>32.5</td> 529 <td>28.5</td> 530 <td>39.0</td> 531 <td>28.0</td> 532 <td>27.0</td> 533 <td>32.0</td> 534 <td>29.0</td> 535 <td>36.5</td> 536 <td>33.5</td> 537 </tr> 538 <tr> 539 <td style="text-align: left">MiniGPT-4</td> 540 <td style="text-align: center">ViT-G</td> 541 <td style="text-align: center">7B</td> 542 <td>16.0</td> 543 <td>18.0</td> 544 <td>26.0</td> 545 <td>21.5</td> 546 <td>16.0</td> 547 <td>29.5</td> 548 <td>25.5</td> 549 <td>13.0</td> 550 <td>11.5</td> 551 <td>12.0</td> 552 <td>9.5</td> 553 <td>32.5</td> 554 <td>15.5</td> 555 <td>8.0</td> 556 <td>34.0</td> 557 <td>26.0</td> 558 <td>29.5</td> 559 <td>19.0</td> 560 <td>9.9</td> 561 <td>3.0</td> 562 <td>18.8</td> 563 </tr> 564 <tr> 565 <td style="text-align: left">InstructBLIP</td> 566 <td style="text-align: center">ViT-G</td> 567 <td style="text-align: center">7B</td> 568 <td>20.0</td> 569 <td>16.5</td> 570 <td>46.0</td> 571 <td>24.5</td> 572 <td>46.0</td> 573 <td>51.0</td> 574 <td>26.0</td> 575 <td>37.5</td> 576 <td>22.0</td> 577 <td>23.0</td> 578 <td>46.5</td> 579 <td><span class="is-1">42.5</span></td> 580 <td>26.5</td> 581 <td>40.5</td> 582 <td>32.0</td> 583 <td>25.5</td> 584 <td>30.0</td> 585 <td>25.5</td> 586 <td>30.5</td> 587 <td>38.0</td> 588 <td>32.5</td> 589 </tr> 590 <tr> 591 <td style="text-align: left">LLaVA</td> 592 <td style="text-align: center">ViT-L</td> 593 <td style="text-align: center">7B</td> 594 <td>28.0</td> 595 <td>39.5</td> 596 <td>63.0</td> 597 <td>30.5</td> 598 <td>39.0</td> 599 <td>53.0</td> 600 <td>41.0</td> 601 <td>41.5</td> 602 <td>23.0</td> 603 <td>20.5</td> 604 <td>45.0</td> 605 <td>34.0</td> 606 <td>20.5</td> 607 <td>38.5</td> 608 <td>47.0</td> 609 <td>25.0</td> 610 <td>36.0</td> 611 <td>27.0</td> 612 <td>26.5</td> 613 <td>42.0</td> 614 <td>36.0</td> 615 </tr> 616 <tr> 617 <td style="text-align: left"><em>Video MLLMs</em></td> 618 <td style="text-align: center">Â </td> 619 <td style="text-align: center">Â </td> 620 <td>Â </td> 621 <td>Â </td> 622 <td>Â </td> 623 <td>Â </td> 624 <td>Â </td> 625 <td>Â </td> 626 <td>Â </td> 627 <td>Â </td> 628 <td>Â </td> 629 <td>Â </td> 630 <td>Â </td> 631 <td>Â </td> 632 <td>Â </td> 633 <td>Â </td> 634 <td>Â </td> 635 <td>Â </td> 636 <td>Â </td> 637 <td>Â </td> 638 <td>Â </td> 639 <td>Â </td> 640 <td>Â </td> 641 </tr> 642 <tr> 643 <td style="text-align: left">Video-LLaMA</td> 644 <td style="text-align: center">CLIP-G</td> 645 <td style="text-align: center">7B</td> 646 <td>27.5</td> 647 <td>25.5</td> 648 <td>51.0</td> 649 <td>29.0</td> 650 <td>39.0</td> 651 <td>48.0</td> 652 <td>40.5</td> 653 <td>38.0</td> 654 <td>22.5</td> 655 <td>22.5</td> 656 <td>43.0</td> 657 <td>34.0</td> 658 <td>22.5</td> 659 <td>32.5</td> 660 <td><span class="is-3">45.5</span></td> 661 <td>32.5</td> 662 <td>40.0</td> 663 <td>30.0</td> 664 <td>21.0</td> 665 <td>37.0</td> 666 <td>34.1</td> 667 </tr> 668 <tr> 669 <td style="text-align: left">LLaMA-Adapter</td> 670 <td style="text-align: center">ViT-B</td> 671 <td style="text-align: center">7B</td> 672 <td>23.0</td> 673 <td>28.0</td> 674 <td>51.0</td> 675 <td>30.0</td> 676 <td>33.0</td> 677 <td>53.5</td> 678 <td>32.5</td> 679 <td>33.5</td> 680 <td>25.5</td> 681 <td>21.5</td> 682 <td>30.5</td> 683 <td>29.0</td> 684 <td>22.5</td> 685 <td>41.5</td> 686 <td>39.5</td> 687 <td>25.0</td> 688 <td>31.5</td> 689 <td>22.5</td> 690 <td>28.0</td> 691 <td>32.0</td> 692 <td>31.7</td> 693 </tr> 694 <tr> 695 <td style="text-align: left">Video-ChatGPT</td> 696 <td style="text-align: center">ViT-L</td> 697 <td style="text-align: center">7B</td> 698 <td>23.5</td> 699 <td>26.0</td> 700 <td>62.0</td> 701 <td>22.5</td> 702 <td>26.5</td> 703 <td>54.0</td> 704 <td>28.0</td> 705 <td><span class="is-2">40.0</span></td> 706 <td>23.0</td> 707 <td>20.0</td> 708 <td>31.0</td> 709 <td>30.5</td> 710 <td>25.5</td> 711 <td>39.5</td> 712 <td><span class="is-1">48.5</span></td> 713 <td>29.0</td> 714 <td>33.0</td> 715 <td>29.5</td> 716 <td>26.0</td> 717 <td>35.5</td> 718 <td>32.7</td> 719 </tr> 720 <tr> 721 <td style="text-align: left">VideoChat</td> 722 <td style="text-align: center">CLIP-G</td> 723 <td style="text-align: center">7B</td> 724 <td>33.5</td> 725 <td>26.5</td> 726 <td>56.0</td> 727 <td>33.5</td> 728 <td>40.5</td> 729 <td>53.0</td> 730 <td>40.5</td> 731 <td>30.0</td> 732 <td>25.5</td> 733 <td>27.0</td> 734 <td>48.5</td> 735 <td>35.0</td> 736 <td>20.5</td> 737 <td>42.5</td> 738 <td><span class="is-2">46.0</span></td> 739 <td>26.5</td> 740 <td>41.0</td> 741 <td>23.5</td> 742 <td>23.5</td> 743 <td>36.0</td> 744 <td>35.5</td> 745 </tr> 746 <tr> 747 <td style="text-align: left">VideoChat2\(^\beta\)</td> 748 <td style="text-align: center">UMT-L</td> 749 <td style="text-align: center">7B</td> 750 <td><span class="is-1">66.0</span></td> 751 <td>47.5</td> 752 <td><span class="is-3">83.5</span></td> 753 <td><span class="is-1">49.5</span></td> 754 <td><span class="is-2">60.0</span></td> 755 <td>58.0</td> 756 <td><span class="is-1">71.5</span></td> 757 <td><span class="is-1">42.5</span></td> 758 <td>23.0</td> 759 <td>23.0</td> 760 <td><span class="is-1">88.5</span></td> 761 <td>39.0</td> 762 <td>42.0</td> 763 <td>58.5</td> 764 <td>44.0</td> 765 <td><span class="is-1">49.0</span></td> 766 <td>36.5</td> 767 <td><span class="is-1">35.0</span></td> 768 <td>40.5</td> 769 <td><span class="is-1">65.5</span></td> 770 <td><span class="is-2">51.1</span></td> 771 </tr> 772 <tr> 773 <td style="text-align: left">PLLaVA 7B\(^\alpha\)</td> 774 <td style="text-align: center">ViT-L</td> 775 <td style="text-align: center">7B</td> 776 <td><span class="is-2">58.0</span></td> 777 <td><span class="is-2">49.0</span></td> 778 <td>55.5</td> 779 <td>41.0</td> 780 <td><span class="is-1">61.0</span></td> 781 <td>56.0</td> 782 <td><span class="is-2">61.0</span></td> 783 <td>36.0</td> 784 <td>23.5</td> 785 <td>26.0</td> 786 <td>82.0</td> 787 <td>39.5</td> 788 <td>42.0</td> 789 <td>52.0</td> 790 <td>45.0</td> 791 <td><span class="is-2">42.0</span></td> 792 <td><span class="is-1">53.5</span></td> 793 <td><span class="is-3">30.5</span></td> 794 <td><span class="is-1">48.0</span></td> 795 <td>31.0</td> 796 <td>46.6</td> 797 </tr> 798 <tr> 799 <td style="text-align: left"><span class="is-ignorable">PLLaVA 13B</span>\(^\alpha\)</td> 800 <td style="text-align: center">ViT-L</td> 801 <td style="text-align: center">13B</td> 802 <td>66.0</td> 803 <td>53.0</td> 804 <td>65.5</td> 805 <td>45.0</td> 806 <td>65.0</td> 807 <td>58.0</td> 808 <td>64.5</td> 809 <td>35.5</td> 810 <td>23.5</td> 811 <td>30.0</td> 812 <td>85.0</td> 813 <td>39.5</td> 814 <td>45.5</td> 815 <td>57.0</td> 816 <td>47.5</td> 817 <td>49.5</td> 818 <td>49.0</td> 819 <td>33.0</td> 820 <td>53.0</td> 821 <td>37.0</td> 822 <td><span class="is-ignorable">50.1</span></td> 823 </tr> 824 <tr> 825 <td style="text-align: left"><strong>VideoLLaMB</strong>\(^\alpha\)</td> 826 <td style="text-align: center">ViT-L</td> 827 <td style="text-align: center">7B</td> 828 <td>52.0</td> 829 <td><span class="is-1">50.5</span></td> 830 <td><span class="is-2">85.5</span></td> 831 <td>42.5</td> 832 <td>51.0</td> 833 <td>69.5</td> 834 <td>56.0</td> 835 <td><span class="is-3">38.5</span></td> 836 <td>41.0</td> 837 <td>24.0</td> 838 <td>69.5</td> 839 <td><span class="is-3">40.0</span></td> 840 <td><span class="is-2">48.0</span></td> 841 <td><span class="is-2">71.5</span></td> 842 <td>43.5</td> 843 <td>34.5</td> 844 <td>41.5</td> 845 <td>29.5</td> 846 <td>38.0</td> 847 <td><span class="is-2">60.0</span></td> 848 <td><span class="is-3">49.3</span></td> 849 </tr> 850 <tr> 851 <td style="text-align: left"><strong>VideoLLaMB</strong>\(^\beta\)</td> 852 <td style="text-align: center">ViT-L</td> 853 <td style="text-align: center">7B</td> 854 <td><span class="is-3">54.5</span></td> 855 <td>47.0</td> 856 <td><span class="is-1">86.5</span></td> 857 <td><span class="is-2">44.5</span></td> 858 <td><span class="is-3">52.0</span></td> 859 <td><span class="is-1">79.0</span></td> 860 <td><span class="is-3">58.5</span></td> 861 <td>32.0</td> 862 <td><span class="is-1">47.0</span></td> 863 <td><span class="is-1">33.0</span></td> 864 <td><span class="is-2">82.5</span></td> 865 <td><span class="is-2">40.5</span></td> 866 <td><span class="is-1">52.0</span></td> 867 <td><span class="is-1">82.0</span></td> 868 <td>40.5</td> 869 <td><span class="is-3">37.5</span></td> 870 <td><span class="is-2">43.0</span></td> 871 <td><span class="is-2">31.0</span></td> 872 <td><span class="is-2">42.5</span></td> 873 <td><span class="is-2">60.0</span></td> 874 <td><span class="is-1">52.5</span></td> 875 </tr> 876 </tbody> 877 </table> 878 <p><span class="dnerf">Table 1.</span> <b>Results on <a href="https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md">MVBench</a> multi-choice question answering.</b>
878 The top 3 results among 7B models are highlighted. \(\alpha\): training with data from <a href="https://pllava.github.io">PLLaVA</a>. \(\beta\): training with data from <a href="https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md">MVBench</a>.</p> 879 880 </div> 881 882 <div class="columns is-centered has-text-centered"> 883<div class="column is-centered "> 884 885 <table class=" table is-narrow is-hoverable is-fullwidth"> 886 <thead> 887 <tr> 888 <th><b>Model</b></th> 889 <th><b>LLM</b></th> 890 <th><b>Frames</b></th> 891 <th><b>Accuracy</b></th> 892 </tr> 893 </thead> 894 <tbody> 895 <tr> 896 <td style="text-align: left">GPT4-o</td> 897 <td>OpenAI API</td> 898 <td>16</td> 899 <td>72.2</td> 900 </tr> 901 <tr> 902 <td colspan="3" style="text-align: left"><small><i>Retrieval-based Video-Language Models</i></small></td> 903 </tr> 904 <tr> 905 <td style="text-align: left">LongViViT*</td> 906 <td>-</td> 907 <td>256</td> 908 <td>56.8</td> 909 </tr> 910 <tr> 911 <td style="text-align: left">MC-ViT-L*</td> 912 <td>-</td> 913 <td>128</td> 914 <td>62.5</td> 915 </tr> 916 <tr> 917 <td colspan="3" style="text-align: left"><small><i>Generative Video-Language Models</i></small></td> 918 </tr> 919 <tr> 920 <td style="text-align: left">SeViLA</td> 921 <td>Flan-T5-XL</td> 922 <td>32</td> 923 <td>25.8</td> 924 </tr> 925 <tr> 926 <td style="text-align: left">mPLUG-Owl</td> 927 <td>LLaMA-7B</td> 928 <td>5</td> 929 <td>33.8</td> 930 </tr> 931 <tr> 932 <td style="text-align: left">VideoLLaVA</td> 933 <td>Vicuna-7B</td> 934 <td>8</td> 935 <td>40.2</td> 936 </tr> 937 <tr> 938 <td style="text-align: left">LLaVA-NeXT-Video-DPO</td> 939 <td>Vicuna-7B</td> 940 <td>32</td> 941 <td>41.6</td> 942 </tr> 943 <tr> 944 <td style="text-align: left">PLLaVA</td> 945 <td>Vicuna-7B</td> 946 <td>16 (16)</td> 947 <td>45.6</td> 948 </tr> 949 <tr> 950 <td style="text-align: left">PLLaVA</td> 951 <td>Vicuna-7B</td> 952 <td>32 (16)</td> 953 <td>43.8</td> 954 </tr> 955 <tr> 956 <td style="text-align: left"><b>VideoLLaMB</b></td> 957 <td>Vicuna-7B</td> 958 <td>32 (8)</td> 959 <td><b>53.8</b></td> 960 </tr> 961 </tbody> 962</table> 963 964 <p><span class="dnerf">Table 2.</span> <b>Results on subset of <a href="https://egoschema.github.io/">EgoSchema</a> under zero-shot setting.</b> \(^*\) indicates that the model has been fine-tuned using the training data from <a href="https://egoschema.github.io/">EgoSchema</a>.</p> 965 966 </div> 967 968<div class="column is-centered"> 969 970 <table class=" table is-narrow is-hoverable is-fullwidth"> 971 <thead> 972 <tr> 973 <th><b>Model</b></th> 974 <th><b>Temporal</b></th> 975 <th><b>Causal</b></th> 976 <th><b>Description</b></th> 977 <th><b>All</b></th> 978 </tr> 979 </thead> 980 <tbody> 981 <tr> 982 <td style="text-align: left">GPT4-o</td> 983 <td>70.3</td> 984 <td>78.0</td> 985 <td>80.8</td> 986 <td>76.0</td> 987 </tr> 988 <tr> 989 <td colspan="5" style="text-align: left"><small><i>Retrieval-based Video-Language Models</i></small></td> 990 </tr> 991 <tr> 992 <td style="text-align: left">AIO*</td> 993 <td>48.0</td> 994 <td>48.6</td> 995 <td>63.2</td> 996 <td>50.6</td> 997 </tr> 998 <tr> 999 <td style="text-align: left">VQA-T*</td> 1000 <td>49.6</td> 1001 <td>51.5</td> 1002 <td>63.2</td> 1003 <td>52.3</td> 1004 </tr> 1005 <tr> 1006 <td style="text-align: left">ATP*</td> 1007 <td>50.2</td> 1008 <td>53.1</td> 1009 <td>66.8</td> 1010 <td>54.3</td> 1011 </tr> 1012 <tr> 1013 <td style="text-align: left">VGT*</td> 1014 <td>52.3</td> 1015 <td>55.1</td> 1016 <td>64.1</td> 1017 <td>55.0</td> 1018 </tr> 1019 <tr> 1020 <td style="text-align: left">MIST-CLIP*</td> 1021 <td>56.6</td> 1022 <td>54.6</td> 1023 <td>66.9</td> 1024 <td>57.1</td> 1025 </tr> 1026 <tr> 1027 <td colspan="5" style="text-align: left"><small><i>Generative Video-Language Models</i></small></td> 1028 </tr> 1029 <tr> 1030 <td style="text-align: left">SeViLA</td> 1031 <td>61.5</td> 1032 <td>61.3</td> 1033 <td>75.6</td> 1034 <td>63.6</td> 1035 </tr> 1036 <tr> 1037 <td style="text-align: left">LLaMA-VID</td> 1038 <td>53.8</td> 1039 <td>60.0</td> 1040 <td>73.0</td> 1041 <td>59.5</td> 1042 </tr> 1043 <tr> 1044 <td style="text-align: left">VideoLLaVA</td> 1045 <td>56.9</td> 1046 <td>61.0</td> 1047 <td>75.0</td> 1048 <td>61.3</td> 1049 </tr> 1050 <tr> 1051 <td style="text-align: left">LLaVA-NeXT-Video-DPO</td> 1052 <td>55.6</td> 1053 <td>61.0</td> 1054 <td>73.9</td> 1055 <td>61.3</td> 1056 </tr> 1057 <tr> 1058 <td style="text-align: left">PLLaVA*</td> 1059 <td>62.2</td> 1060 <td>68.5</td> 1061 <td><strong>79.7</strong></td> 1062 <td>68.2</td> 1063 </tr> 1064 <tr> 1065 <td style="text-align: left"><strong>VideoLLaMB*</strong></td> 1066 <td><strong>66.8</strong></td> 1067 <td><strong>71.6</strong></td> 1068 <td>78.4</td> 1069 <td><strong>71.1</strong></td> 1070 </tr> 1071 </tbody> 1072</table> 1073 1074 <p><span class="dnerf">Table 3.</span> <b>Comparison accuracy on <a href="hhttps://github.com/doc-doc/NExT
1074-QA">NExT-QA</a>.</b> \(^*\) indicates that the instruction data includes the training data from NExT-QA.</p> 1075 1076 </div> 1077 1078 1079</div> 1080 1081 <details open=""> 1082<summary><b>Streaming Caption</b></summary> 1083<div class="is-centered has-text-centered" style="background-color:rgba(117, 209, 215, 0.1)"> 1084 1085<h4 style="font-size: 20px; padding: 10px 0 10px;">Task: Describe the streaming video in real-time.</h4> 1086 <video poster="" id="streaming_caption" autoplay="" controls="" muted="" loop="" playsinline="" height="70%"> 1087 <source src="/assets/img/streaming_caption.mp4" /> 1088 </video> 1089</div> 1090</details> 1091 1092 <!-- </div> 1093</section> 1094 1095 1096<section class="section" > 1097 <div class="container" markdown="1"> --> 1098 1099 <p><br /></p> 1100 1101 <h2 class="title is-3 has-text-centered" id="egocentric-embodied-planning">Egocentric Embodied Planning</h2> 1102 1103 <div class="columns is-centered has-text-centered"> 1104<div class="column is-centered "> 1105 1106 <table class="table is-narrow is-hoverable is-fullwidth"> 1107 <thead> 1108 <tr> 1109 <th style="text-align: left; font-weight: bold; text-align: center;">Model</th> 1110 <th style="font-weight: bold; text-align: center;">LLM</th> 1111 <th style="font-weight: bold; text-align: center;">Accuracy</th> 1112 </tr> 1113 </thead> 1114 <tbody> 1115 <tr> 1116 <td style="text-align: left;">GPT-4V</td> 1117 <td style="text-align: center;">OpenAI API</td> 1118 <td style="text-align: center;">37.98</td> 1119 </tr> 1120 <tr> 1121 <td colspan="3" style="text-align: left; font-size: small; font-style: italic;">Image-Language Model</td> 1122 </tr> 1123 <tr> 1124 <td style="text-align: left;">Qwen-VL-Chat</td> 1125 <td style="text-align: center;">Qwen-7B</td> 1126 <td style="text-align: center;">26.32</td> 1127 </tr> 1128 <tr> 1129 <td style="text-align: left;">LLaVA-1.5</td> 1130 <td style="text-align: center;">Vicuna-7B</td> 1131 <td style="text-align: center;">26.80</td> 1132 </tr> 1133 <tr> 1134 <td style="text-align: left;">SEED-LLaMA</td> 1135 <td style="text-align: center;">LLaMA2-Chat-13B</td> 1136 <td style="text-align: center;">29.93</td> 1137 </tr> 1138 <tr> 1139 <td style="text-align: left;">InternLM-Xcomposer</td> 1140 <td style="text-align: center;">InternLM-7B</td> 1141 <td style="text-align: center;">34.4</td> 1142 </tr> 1143 <tr> 1144 <td colspan="3" style="text-align: left; font-size: small; font-style: italic;">Video-Language Model</td> 1145 </tr> 1146 <tr> 1147 <td style="text-align: left;">VideoChatGPT</td> 1148 <td style="text-align: center;">LLaMA-7B</td> 1149 <td style="text-align: center;">26.35</td> 1150 </tr> 1151 <tr> 1152 <td style="text-align: left;">Valley</td> 1153 <td style="text-align: center;">LLaMA-13B</td> 1154 <td style="text-align: center;">26.17</td> 1155 </tr> 1156 <tr> 1157 <td style="text-align: left;">VideoLLaMA</td> 1158 <td style="text-align: center;">LLaMA2-Chat-7B</td> 1159 <td style="text-align: center;">29.85</td> 1160 </tr> 1161 <tr> 1162 <td style="text-align: left;">LLaVA-NeXT-Video</td> 1163 <td style="text-align: center;">Vicuna-7B</td> 1164 <td style="text-align: center;">28.96</td> 1165 </tr> 1166 <tr> 1167 <td style="text-align: left;">PLLaVA</td> 1168 <td style="text-align: center;">Vicuna-7B</td> 1169 <td style="text-align: center;">30.26</td> 1170 </tr> 1171 <tr> 1172 <td style="text-align: left; font-weight: bold;">VideoLLaMB-7B</td> 1173 <td style="text-align: center;">Vicuna-7B</td> 1174 <td style="text-align: center; font-weight: bold;">32.32</td> 1175 </tr> 1176 </tbody> 1177</table> 1178 1179 <!-- <figcaption> --> 1180 <p><span class="dnerf">Table 4.</span> <b>Results on <a href="https://github.com/ChenYi99/EgoPlan">EgoPlan</a> under Zero-shot setting.</b></p> 1181 1182 </div> 1183 1184<div class="column is-centered has-text-centered"> 1185 1186 <figure class="image"> 1187 <b>Goal:</b> <em>clean and organize kitchen</em> 1188 <img src="/assets/img/case.png" /> 1189 <figcaption><span class="dnerf">Figure 2.</span> <b>Qualitative results on <a href="https://github.com/ChenYi99/EgoPlan">EgoPlan</a>.</b> </figcaption> 1190</figure> 1191 1192 </div> 1193 1194 1195</div> 1196 1197 </div> 1198</section> 1199 1200<section class="section"> 1201 <div class="container is-max-desktop"> 1202 1203 <h2 class="title is-3 has-text-centered" id="stress-test-needle-in-a-video-haystack">Stress Test: âNeedle In A Video Haystackâ</h2> 1204 1205 <div class="columns is-centered has-text-centered"> 1206<div class="column"> 1207 1208<figure class="image"> 1209 <b>Needle:</b> "<em>A young man is sitting on a piece of cloud in the sky, reading a book.</em>" 1210 <img src="/assets/img/case_needle.png" />
1211 <figcaption><span class="dnerf">Figure 3.</span> <b>An example of NIAVH.</b> </figcaption> 1212</figure> 1213 1214</div> 1215</div> 1216 1217 <p>We utilize ego-centric videos from the Ego4D dataset as the âhaystackâ. Within this haystack, we seek to locate the âneedleâ, which we provide in three distinct modalities. For the textual modality, we supply a crafted description. For the image modality, we employ DALL-E to create an image that visually represents this description. For the video modality, we use Sora to generate a short video clip based on the same description. In each case, the âneedleâ - whether text, image, or video - is set to a duration of 1 second.</p> 1218 1219 </div> 1220</section> 1221 1222<section class="section" style="background-color:#efeff081"> 1223 <div class="container is-max-desktop"> 1224 1225 <h2 class="title is-3 has-text-centered" id="technical-details">Technical Details</h2> 1226 1227 <h3 class="title is-4" id="scene-tiling-segmentation-with-semantics">Scene Tiling: Segmentation with Semantics</h3> 1228 1229 <p>We introduce SceneTilling, a <em>model-free</em> scene segmentation algorithm, to divide the entire video sequence into video segments such that each segment is semantically non-overlap with others, <em>i.e.</em>, inter-segment coherence. Formally, given a sequence of \(n\) frames \(\{v_1, v_2, \ldots, v_n\}\), the SceneTiling algorithm is as follows.</p> 1230 <ol> 1231 <li>Compute the cosine similarity \(S_C(\cdot, \cdot)\) between adjacent frame pairs using the [CLS] token from ViT, resulting in a sequence of similarity scores \(\{c_1, c_2, \ldots, c_{n-1}\}\), where \(c_i = S_C ({\rm ViT}(v_i), {\rm ViT}(v_{i+1}))\).</li> 1232 <li>Calculate the depth score for each point as \(d_i = \left(cl_i+cr_i-2c_i\right)/{2}\), where \(cl_i\) and \(cr_i\) are the highest score to the left and right of \(c_i\), respectively. A higher depth score indicates that the surrounding similarity is greater than at the point itself.</li> 1233 <li>Calculate the expectation \(\mu\) and variance \(\sigma\) of the depth scores \(\{d_1, d_2, \ldots, d_{n-1}\}\). Set the segmentation threshold as \(\mu + \alpha \cdot \sigma\), where \(\alpha\) is a hyperparameter controlling the likelihood of segmenting the video.</li> 1234 <li>Select the \(K-1\) depth scores that exceed the threshold to divide the video into \(K\) semantic segments \(\{s_1, s_2, \ldots, s_K\}\). Each segment represents a relatively independent semantic unit consisting of a sequence of frames. 1235<br /></li> 1236 </ol> 1237 1238 <h3 class="title is-4" id="recurrent-memory-bridge-layers">Recurrent Memory Bridge Layers</h3> 1239 1240 <p>We devised a novel Recurrent Memory Bridge Layer, implemented as a multi-layer Transformer block, that integrates recurrent memory tokens within bridge layers to enhance the linear layerâs memorization ability.</p> 1241 1242 <p>For each video segment \(s_i\), we prepend a fixed number of memory tokens, denoted as \([m_i; s_i]\), where \(m_i\) represents the memory tokens. Subsequently, we apply standard self-attention to this sequence, yielding \([m_{i+1}; o_{i}] = {\rm BridgeLayer}([m_i; s_i])\). Here, \(m_{i+1}\) is the updated memory token, and \(o_{i}\) is the visual representation from the bridge layers.</p> 1243 1244 <p>As such, the Memory Bridge can <strong>compress past video into memory tokens while preserving current video scenes through projection without losing detailed information by compressing</strong>.</p> 1245 1246 <!-- Formally, for each video segment $$s_i$$, we prepend a fixed number of memory tokens, denoted as $$[m_i; s_i]$$, where $$m_i$$ represents the memory tokens. Subsequently, we apply standard self-attention to this sequence, yielding $$[m_{i+1}; o_{i+1}] = {\rm BridgeLayer}([m_i; s_i])$$. Here, $$m_{i+1}$$ is the updated memory token, and $$o_{i+1}$$ is the refreshed visual representation. --> 1247 <!-- This process is carried out recursively, traversing the semantic video segments while updating the memory tokens. After a total of $$k$$ steps, the final output $$o_k$$ can be obtained as the condensed visual representation of the video sequence. This methodology allows us to encode an entire video into a concise sequence while retaining all pertinent information from previous segments. --> 1248 1249 <p><br /></p> 1250 1251 <h3 class="title is-4" id="memory-cache-with-retrieval">Memory Cache with Retrieval</h3> 1252 1253 <p>One of the primary challenges associated with recurrent memory bridge layers is the potential for gradient vanishing, which can impede the modelâs ability to learn long-range dependencies. To mitigate this issue, we propose the incorporation of a <strong>memory cache</strong> with a retrieval strategy designed to preserve previous states of memory.</p> 1254 1255 <p><strong>Memory Attention</strong> At each timestep \(i\), the system stores all previous memory tokens in a memory cache, denoted as \(M_i = [m_1, \ldots, m_i]\). We employ a self-retrieval mechanism to update the current memory token \(m_i\). Specifically, we treat \(m_i\) as a query and the concatenated memory cache \(M_i\) as key and value. The model performs a standard multi-head cross-attention operation to integrate information from previous timesteps into the current memory state, yielding the updated memory token</p> 1256 1257\[m_{i+1} = \text{Softmax}\left(\frac{W_i^Q m_i (W_i^K M_i)^\top}{\sqrt{d_k}}\right) W_i^V M_i,\] 1258 1259 <p>where \(W_i^Q, W_i^K, W_i^V\) are weight martices for query, key and value, respectively.</p> 1260 1261 </div> 1262</section> 1263 1264<section class="section"> 1265 <div class="container is-max-desktop"> 1266 1267 <h2 class="title" id="citation">Citation</h2> 1268 1269 <div class="language-bibtex highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nc">@article</span><span class="p">{</span><span class="nl">wang2024videollamb</span><span class="p">,</span> 1270 <span class="na">title</span><span class="p">=</span><span class="s">{VideoLLaMB: Long-context Video Understanding with Recurrent Memory Bridges}</span><span class="p">,</span> 1271 <span class="na">author</span><span class="p">=</span><span class="s">{Wang, Yuxuan and Xie, Cihang and Liu, Yang and Zheng, Zilong}</span><span class="p">,</span> 1272 <span class="na">
1272journal</span><span class="p">=</span><span class="s">{arXiv preprint arXiv:2409.01071}</span><span class="p">,</span> 1273 <span class="na">year</span><span class="p">=</span><span class="s">{2024}</span> 1274<span class="p">}</span> 1275</code></pre></div> </div> 1276 1277 </div> 1278</section> 1279 1280 1281 </div> 1282 1283 <!-- Footer --> 1284 1285 <footer class="footer"> 1286 <div class="container"> 1287 <div class="columns is-centered"> 1288 <div class="column is-8"> 1289 <div class="content"> 1290 <p> 1291 This website was built using a <a href="https://jekyllrb.com/">Jekyll</a> template adapted from 1292 <a href="https://github.com/nerfies/nerfies.github.io">Nerfies</a>, 1293 licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative 1294 Commons Attribution-ShareAlike 4.0 International License</a>. 1295 </p> 1296 </div> 1297 </div> 1298 </div> 1299 </div> 1300</footer> 1301 1302<!-- jQuery -->
1303<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js" 1304 integrity="sha512-bLT0Qm9VnAYZDflyKcBaQ2gg0hSYNQrJ8RilYldYQ1FxQYoCLtUjuuRuZo+fjqhx/qtq/1itJ0C2ejDxltZVFg==" crossorigin="anonymous"></script>
vendor: 1 bytes, line 1304
1304
1305<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-easing/1.4.1/jquery.easing.min.js" 1306 crossorigin="anonymous"></script>
1306 1307 1308<!-- Bulma -->
1309<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bulma-slider.min.js"></script>
vendor: 1 bytes, line 1309
1309
1310<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bulma-carousel.min.js"></script>
1310 1311
1312<script src="//cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/js/toastr.min.js"></script>
1312 1313
1314<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js"> 1315</script>
vendor: 1 bytes, line 1315
1315
1316<script src="https://cdn.jsdelivr.net/npm/[email protected]/prism-bibtex.min.js"> 1317</script>
1317 1318 1319</body> 1320 1321 1322</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.