PageSourceSearch

https://videollamb.github.io/

html videollamb.github.io collected 2026-10-03 10:08:14 UTC 50,416 bytes, 1,322 lines download raw bytes

1<!DOCTYPE html>
2<html>
3
4<head>
5    <meta charset="utf-8">
6<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
7<meta http-equiv="X-UA-Compatible" content="IE=edge">
8
9<title>VideoLLaMB</title>
10<meta name="description" content="VideoLLaMB: Long Video Understanding with Recurrent Memory Bridges">
11
12<!-- Open Graph -->
13
14
15<!-- Bulma & MDB -->
16<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/css/bulma.min.css">
17<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bulma-carousel.min.css">
18<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bulma-slider.min.css">
19
20<!-- Fonts & Icons -->
21<link rel="stylesheet"
22    href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.2/css/all.min.css"
23    integrity="sha512-z3gLpd7yknf1YoNbCzqRKc4qyor8gaKU1qmn+CShxbuBusANI9QpRohGBreCFkKxLhei6S9CQXFEbbKuqLg0DA==" crossorigin="anonymous">
24<link rel="stylesheet"
25    href="https://cdnjs.cloudflare.com/ajax/libs/academicons/1.9.4/css/academicons.min.css"
26    crossorigin="anonymous">
27<link rel="stylesheet" type="text/css"
28    href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700|Roboto+Slab:100,300,400,500,700|Lora|Google+Sans|Material+Icons">
29
30<!-- Code Syntax Highlighting -->
31<!-- <link rel="stylesheet" href="https://gitcdn.link/repo/jwarby/jekyll-pygments-themes/master/.css" />
31 -->
32
33<!-- Styles -->
34<link rel="shortcut icon" href="/assets/img/favicon.ico">
35<link rel="stylesheet" href="/assets/css/main.css">
36
37<link rel="canonical" href="/">
38
39<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism-solarizedlight.min.css"/>
40
41
42<!-- Theming-->
43
44
45
46<!-- Global site tag (gtag.js) - Google Analytics -->
vendor: 64 bytes, lines 46-47
46
47<script async src="https://www.googletagmanager.com/gtag/js?id=
47G-XD4J271NBT
vendor: 12 bytes, line 47
47"></script>
48<script>
49    
vendor: 144 bytes, lines 49-53
49window.dataLayer = window.dataLayer || [];
50    function gtag () { dataLayer.push(arguments); }
51    gtag('js', new Date());
52
53    gtag('config', '
53G-XD4J271NBT
vendor: 4 bytes, line 53
53');
54</script>
54
55
56
57
58
59
60    
61<!-- MathJax -->
62<script type="text/javascript">
63    window.MathJax = {
64        tex: {
65            tags: 'ams'
66        }
67    };
68</script>
vendor: 1 bytes, line 68
68
69<script defer type="text/javascript" id="MathJax-script"
70    src="https://cdn.jsdelivr.net/npm/[email protected]/es5/tex-mml-chtml.js"></script>
vendor: 1 bytes, line 70
70
71<script defer src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?features=es6"></script>
71
72
73</head>
74
75<body>
76    <!-- Header -->
77
78    
79<nav class="navbar" role="navigation" aria-label="main navigation">
80    <div class="navbar-brand">
81        <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false">
82            <span aria-hidden="true"></span>
83            <span aria-hidden="true"></span>
84            <span aria-hidden="true"></span>
85        </a>
86    </div>
87    <div class="navbar-menu">
88        <div class="navbar-start" style="flex-grow: 1; justify-content: center;">
89            <a class="navbar-item" href="https://videollamb.github.io">
90                <span class="icon">
91                    <i class="fas fa-home"></i>
92                </span>
93            </a>
94
95            <div class="navbar-item has-dropdown is-hoverable">
96                <a class="navbar-link">
97                    More Research
98                </a>
99                <div class="navbar-dropdown">
100                    
101                    <a class="navbar-item" href="https://bigai-nlco.github.io/ExoViP/">
102                        ExoViP
103                    </a>
104                    
105                    <a class="navbar-item" href="https://videohallucer.github.io/">
106                        VideoHallucer
107                    </a>
108                    
109                    <a class="navbar-item" href="https://github.com/bigai-nlco/LSTP-Chat">
110                        LSTPChat
111                    </a>
112                    
113                    <a class="navbar-item" href="https://vstar-benchmark.github.io/">
114                        VSTAR
115                    </a>
116                    
117                </div>
118            </div>
119        </div>
120
121    </div>
122</nav>
123
124
125    <!-- Content -->
126
127    <div class="container is-fullhd mt-5">
128        <section class="hero">
129            <div class="hero-body">
130                <div class="container is-max-desktop">
131                    <div class="columns is-centered">
132                        <div class="column has-text-centered">
133                            <div style="display: flex;padding-bottom: 20px;">
134                                <img src="/assets/img/video-llamb.png"
135                                    width="100px" style="vertical-align: middle;">
136                                <h1 class="title is-2 publication-title"><span style="display: block;">VideoLLaMB: Long-Context Video Understanding with Recurrent Memory Bridges
137</span></h1>
138                            </div>
139                            
140                            
141
142                            <div class="is-size-5 publication-authors">
143                                
144                                <span class="author-block"><a href="https://patrick-tssn.github.io/">Yuxuan Wang</a><sup>1</sup></span>, 
145                                
146                                <span class="author-block"><a href="https://cihangxie.github.io/">Cihang Xie</a><sup>2</sup></span>, 
147                                
148                                <span class="author-block"><a href="http://www.csyangliu.com/">Yang Liu</a><sup>3</sup></span>, 
149                                
150                                <span class="author-block"><a href="https://zilongzheng.github.io">Zilong Zheng</a><sup>1, <i class="fa fa-envelope"></i></sup></span>
151                                
152                            </div>
153                            
154
155                            
156                            <div class="is-size-5 publication-authors">
157                                
158                                <span class="author-block"><sup>1</sup>BIGAI</span>, 
159                                
160                                <span class="author-block"><sup>2</sup>UCSC</span>, 
161                                
162                                <span class="author-block"><sup>3</sup>PKU</span>
163                                
164                            </div>
165                            
166
167                            
168
169                            <div class="column has-text-centered">
170                                <div class="publication-links">
171                                    
172                                    
173                                    <span class="link-block">
174                                        <a href="https://arxiv.org/abs/2409.01071"
175                                            class="external-link button is-normal is-rounded is-dark">
176                                            <span class="icon">
177                                                <i class="ai ai-arxiv"></i>
178                                            </span>
179                                            <span>arXiv</span>
180                                        </a>
181                                    </span>
182                                    
183                                    
184                                    
185                                    <!-- Code Link. -->
186                                    <span class="link-block">
187                                        <a href="https://github.com/bigai-nlco/VideoLLaMB"
188                                            class="external-link button is-normal is-rounded is-dark">
189                                            <span class="icon">
190                                                <i class="fab fa-github"></i>
191                                            </span>
192                                            <span>Code</span>
193                                        </a>
194                                    </span>
195                                    
196                                    
197                                    
198                                    
199                                    <span class="link-block">
200                                        <a href=" https://huggingface.co/ColorfulAI/VideoLLaMB"
201                                            class="external-link button is-normal is-rounded is-dark">
202                                            <span class="icon">
203                                                &#129303;
204                                            </span>
205                                            <span>VideoLLaMB-7B</span>
206                                        </a>
207                                    </span>
208                                    
209                                    <span class="link-block">
210                                        <a href=" https://github.com/bigai-nlco/NeedleInAVideoHaystack"
211                                            class="external-link button is-normal is-rounded is-dark">
212                                            <span class="icon">
213                                                <i class="fab fa-github"></i>
214                                            </span>
215                                            <span>MM-NIAVH</span>
216                                        </a>
217                                    </span>
218                                    
219                                    <span class="link-block">
220                                        <a href=" "
221                                            class="external-link button is-normal is-rounded is-dark">
222                                            <span class="icon">
223                                                <i class="fas fa-globe"></i>
224                                            </span>
225                                            <span>Demo (Coming Soon)</span>
226                                        </a>
227                                    </span>
228                                    
229                                    
230                                </div>
231                            </div>
232                        </div>
233                    </div>
234                </div>
235            </div>
236        </section>
237
238        <section class="hero teaser">
239  <div class="container is-max-desktop">
240    <div class="hero-body">
241<div class="columns is-centered has-text-centered">
242
243<div class="column">
244<figure class="image">
245      <figcaption><span class="dnerf">VideoLLaMB</span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</figcaption>
246      <img src="/assets/img/videollamb_niavh.png" />
247</figure>
248
249
250<figure class="image">
251      <figcaption><span class="dnerf">VideoLLaMB w/o Memory Retrieval</span></figcaption>
252      <img src="/assets/img/videollamb_niavh_nor.png" />
253</figure>
254
255</div>
256
257<div class="column">
258<figure class="image">
259      <figcaption><span class="dnerf">LongVA (<a href="https://github.com/EvolvingLMMs-Lab/LongVA">Zhang et. al., 2024</a>)</span></figcaption>
260      <img src="/assets/img/longva.png" />
261</figure>
262
263<figure class="image">
264      <figcaption><span class="dnerf">MA-LLM (<a href="https://github.com/boheumd/MA-LMM">He et. al., 2024</a>)</span></figcaption>
265      <img src="/assets/img/mallm.png" />
266</figure>
267
268</div>
269
270
271</div>
272
273
274<details closed="">
275<summary><b>More comparisons</b></summary>
276<div class="columns is-centered has-text-centered">
277
278<div class="column">
279<figure class="image">
280      <figcaption><span class="dnerf"><a href="https://huggingface.co/ermu2001/pllava-7b">PLLaVA-7B</a>  (Run 6/5/2024)</span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</figcaption>
281      <img src="/assets/img/pllava.png" />
282</figure>
283
284</div>
285
286<div class="column">
287<figure class="image">
288      <figcaption><span class="dnerf"><a href="https://huggingface.co/lmms-lab/LLaVA-NeXT-Video-7B-DPO">LLaVA-NeXT-Video-DPO-7B</a> (Run 6/5/2024) </span></figcaption>
289      <img src="/assets/img/llavanext.png" />
290</figure>
291
292</div>
293
294
295</div>
296</details>
297
298<figcaption style="padding-top:10px;"><span class="dnerf">Figure 1.</span> <b>Comparison of long video understanding models on <a href="#stress-test-needle-in-a-video-haystack">Needle In a Video Haystack (NIAVH)</a>.</b> We set the context length to 320 seconds due to existing models' ability and set the frame rate to 1 fps to ensure the input contains the needle. The X-axis indicates the video length, and the Y-axis is the depth of the insertion point.</figcaption>
299
300    </div>
301  </div>
302</section>
303
304<section class="section">
305    <div class="container is-max-desktop">
306
307    <h2 class="title has-text-centered" id="abstract">Abstract</h2>
308
309    <p>VideoLLaMB is a novel long video comprehension framework utilizing Memory Bridge Layers with recurrent memory tokens to encode 100% video content without discarding critical visual cues.</p>
310
311    <p>✨ Highlights:</p>
312
313    <ol>
314      <li>
315        <p><strong>Comprehensive long video understanding.</strong> VideoLLaMB-7B reached the state-of-the-art performance among 7B models trained on vicuna-7b and videochat2 video on <a href="https://egoschema.github.io/">EgoSchema</a>, <a href="https://github.com/doc-doc/NExT-QA">NexTQA</a> and <a href="https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md">MVBench</a>, reaching <em>8x longer video length</em> with robust performance in comparison to <a href="https://pllava.github.io/">PLLaVA</a>.</p>
316      </li>
317      <li>
318        <p><strong>Memory-based egocentric planning.</strong> VideoLLaMB achieves the best performance among all video-language models on <a href="https://github.com/ChenYi99/EgoPlan">EgoPlan</a>, with an improvement of \(2.06\) over PLLaVA.</p>
319      </li>
320      <li>
321        <p><strong>Training-free streaming captioning.</strong> With our <a href="#scene-tiling-segmentation-with-semantics">SceneTiling algorithm</a>, VideoLLaMB can capture the dynamics with in a streaming video and directly predict the streaming captions in real-time, without the need to process the entire video sequence beforehand.</p>
322      </li>
323      <li>
324        <p><strong>Enhanced frame retrieval on needle in a video haystack (NIAVH).</strong> We present the “Needle in a Video Haystack” (NIAVH) benchmark to evaluate long video understanding over needle of different modalities comprehensively (<a href="#stress-test-needle-in-a-video-haystack">details 👉</a>). In the pressure test ranging from 1 to 300 seconds in length, VideoLLaMB consistently retrieves the correct image needles at various depths, outperforming other methods as video length increases.</p>
325      </li>
326    </ol>
327
328  </div>
329
330<div class="columns is-centered has-text-centered">
331
332<div class="column is-four-fifths">
333<figure class="image">
334      <img src="/assets/img/framework_new.png" />
335      <figcaption><span class="dnerf">Figure 2.</span> <b>An overview of VideoLLaMB.</b> <a href="#technical-details">Technical details.</a> </figcaption>
336</figure>
337
338</div>
339
340</div>
341
342
343</section>
344
345<section class="section" style="background-color:#efeff081">
346    <div class="container">
347
348    <h2 class="title is-3 has-text-centered" id="long-form-video-understanding">Long-form Video Understanding</h2>
349
350    <div class="table-container has-text-centered" style="font-size:15px">
351
352      <table class="table is-fullwidth is-narrow is-hoverable">
353        <thead>
354          <tr>
355            <th style="text-align: left"><strong>Method</strong></th>
356            <th style="text-align: center"><strong>Vision Encoder</strong></th>
357            <th style="text-align: center"><strong>LLM Size</strong></th>
358            <th><strong>AS</strong></th>
359            <th><strong>AP</strong></th>
360            <th><strong>AA</strong></th>
361            <th><strong>FA</strong></th>
362            <th><strong>UA</strong></th>
363            <th><strong>OE</strong></th>
364            <th><strong>OI</strong></th>
365            <th><strong>OS</strong></th>
366            <th><strong>MD</strong></th>
367            <th><strong>AL</strong></th>
368            <th><strong>ST</strong></th>
369            <th><strong>AC</strong></th>
370            <th><strong>MC</strong></th>
371            <th><strong>MA</strong></th>
372            <th><strong>SC</strong></th>
373            <th><strong>FP</strong></th>
374            <th><strong>
374CO</strong></th>
375            <th><strong>EN</strong></th>
376            <th><strong>ER</strong></th>
377            <th><strong>CI</strong></th>
378            <th><strong>Avg.</strong></th>
379          </tr>
380        </thead>
381        <tbody>
382          <tr>
383            <td style="text-align: left">GPT-4V</td>
384            <td style="text-align: center">GPT-4V</td>
385            <td style="text-align: center">/</td>
386            <td>55.5</td>
387            <td>63.5</td>
388            <td>72.0</td>
389            <td>46.5</td>
390            <td>73.5</td>
391            <td>18.5</td>
392            <td>59.0</td>
393            <td>29.5</td>
394            <td>12.0</td>
395            <td>40.5</td>
396            <td>83.5</td>
397            <td>39.0</td>
398            <td>12.0</td>
399            <td>22.5</td>
400            <td>45.0</td>
401            <td>47.5</td>
402            <td>52.0</td>
403            <td>31.0</td>
404            <td>59.0</td>
405            <td>11.0</td>
406            <td>43.5</td>
407          </tr>
408          <tr>
409            <td style="text-align: left"><em>Image MLLMs</em></td>
410            <td style="text-align: center"> </td>
411            <td style="text-align: center"> </td>
412            <td> </td>
413            <td> </td>
414            <td> </td>
415            <td> </td>
416            <td> </td>
417            <td> </td>
418            <td> </td>
419            <td> </td>
420            <td> </td>
421            <td> </td>
422            <td> </td>
423            <td> </td>
424            <td> </td>
425            <td> </td>
426            <td> </td>
427            <td> </td>
428            <td> </td>
429            <td> </td>
430            <td> </td>
431            <td> </td>
432            <td> </td>
433          </tr>
434          <tr>
435            <td style="text-align: left">mPLUG-Owl-I</td>
436            <td style="text-align: center">ViT-L</td>
437            <td style="text-align: center">7B</td>
438            <td>25.0</td>
439            <td>20.0</td>
440            <td>44.5</td>
441            <td>27.0</td>
442            <td>23.5</td>
443            <td>36.0</td>
444            <td>24.0</td>
445            <td>34.0</td>
446            <td>23.0</td>
447            <td>24.0</td>
448            <td>34.5</td>
449            <td>34.5</td>
450            <td>22.0</td>
451            <td>31.5</td>
452            <td>40.0</td>
453            <td>24.0</td>
454            <td>37.0</td>
455            <td>25.5</td>
456            <td>21.0</td>
457            <td>37.0</td>
458            <td>29.4</td>
459          </tr>
460          <tr>
461            <td style="text-align: left">LLaMA-Adapter</td>
462            <td style="text-align: center">ViT-B</td>
463            <td style="text-align: center">7B</td>
464            <td>23.0</td>
465            <td>28.0</td>
466            <td>51.0</td>
467            <td>30.0</td>
468            <td>33.0</td>
469            <td>53.5</td>
470            <td>32.5</td>
471            <td>33.5</td>
472            <td>25.5</td>
473            <td>21.5</td>
474            <td>30.5</td>
475            <td>29.0</td>
476            <td>22.5</td>
477            <td>41.5</td>
478            <td>39.5</td>
479            <td>25.0</td>
480            <td>31.5</td>
481            <td>22.5</td>
482            <td>28.0</td>
483            <td>32.0</td>
484            <td>31.7</td>
485          </tr>
486          <tr>
487            <td style="text-align: left">BLIP2</td>
488            <td style="text-align: center">ViT-G</td>
489            <td style="text-align: center">2.7B</td>
490            <td>24.5</td>
491            <td>29.0</td>
492            <td>33.5</td>
493            <td>17.0</td>
494            <td>42.0</td>
495            <td>51.5</td>
496            <td>26.0</td>
497            <td>31.0</td>
498            <td>25.5</td>
499            <td>26.0</td>
500            <td>32.5</td>
501            <td>25.5</td>
502            <td>30.0</td>
503            <td>40.0</td>
504            <td>42.0</td>
505            <td>27.0</td>
506            <td>30.0</td>
507            <td>26.0</td>
508            <td>37.0</td>
509            <td>31.0</td>
510            <td>31.4</td>
511          </tr>
512          <tr>
513            <td style="text-align: left">Otter-I</td>
514            <td style="text-align: center">ViT-L</td>
515            <td style="text-align: center">7B</td>
516            <td>34.5</td>
517            <td>32.0</td>
518            <td>39.5</td>
519            <td>30.5</td>
520            <td>38.5</td>
521            <td>48.5</td>
522            <td>44.0</td>
523            <td>29.5</td>
524            <td>19.0</td>
525            <td>25.5</td>
526            <td>55.0</td>
527            <td>20.0</td>
528            <td>32.5</td>
529            <td>28.5</td>
530            <td>39.0</td>
531            <td>28.0</td>
532            <td>27.0</td>
533            <td>32.0</td>
534            <td>29.0</td>
535            <td>36.5</td>
536            <td>33.5</td>
537          </tr>
538          <tr>
539            <td style="text-align: left">MiniGPT-4</td>
540            <td style="text-align: center">ViT-G</td>
541            <td style="text-align: center">7B</td>
542            <td>16.0</td>
543            <td>18.0</td>
544            <td>26.0</td>
545            <td>21.5</td>
546            <td>16.0</td>
547            <td>29.5</td>
548            <td>25.5</td>
549            <td>13.0</td>
550            <td>11.5</td>
551            <td>12.0</td>
552            <td>9.5</td>
553            <td>32.5</td>
554            <td>15.5</td>
555            <td>8.0</td>
556            <td>34.0</td>
557            <td>26.0</td>
558            <td>29.5</td>
559            <td>19.0</td>
560            <td>9.9</td>
561            <td>3.0</td>
562            <td>18.8</td>
563          </tr>
564          <tr>
565            <td style="text-align: left">InstructBLIP</td>
566            <td style="text-align: center">ViT-G</td>
567            <td style="text-align: center">7B</td>
568            <td>20.0</td>
569            <td>16.5</td>
570            <td>46.0</td>
571            <td>24.5</td>
572            <td>46.0</td>
573            <td>51.0</td>
574            <td>26.0</td>
575            <td>37.5</td>
576            <td>22.0</td>
577            <td>23.0</td>
578            <td>46.5</td>
579            <td><span class="is-1">42.5</span></td>
580            <td>26.5</td>
581            <td>40.5</td>
582            <td>32.0</td>
583            <td>25.5</td>
584            <td>30.0</td>
585            <td>25.5</td>
586            <td>30.5</td>
587            <td>38.0</td>
588            <td>32.5</td>
589          </tr>
590          <tr>
591            <td style="text-align: left">LLaVA</td>
592            <td style="text-align: center">ViT-L</td>
593            <td style="text-align: center">7B</td>
594            <td>28.0</td>
595            <td>39.5</td>
596            <td>63.0</td>
597            <td>30.5</td>
598            <td>39.0</td>
599            <td>53.0</td>
600            <td>41.0</td>
601            <td>41.5</td>
602            <td>23.0</td>
603            <td>20.5</td>
604            <td>45.0</td>
605            <td>34.0</td>
606            <td>20.5</td>
607            <td>38.5</td>
608            <td>47.0</td>
609            <td>25.0</td>
610            <td>36.0</td>
611            <td>27.0</td>
612            <td>26.5</td>
613            <td>42.0</td>
614            <td>36.0</td>
615          </tr>
616          <tr>
617            <td style="text-align: left"><em>Video MLLMs</em></td>
618            <td style="text-align: center"> </td>
619            <td style="text-align: center"> </td>
620            <td> </td>
621            <td> </td>
622            <td> </td>
623            <td> </td>
624            <td> </td>
625            <td> </td>
626            <td> </td>
627            <td> </td>
628            <td> </td>
629            <td> </td>
630            <td> </td>
631            <td> </td>
632            <td> </td>
633            <td> </td>
634            <td> </td>
635            <td> </td>
636            <td> </td>
637            <td> </td>
638            <td> </td>
639            <td> </td>
640            <td> </td>
641          </tr>
642          <tr>
643            <td style="text-align: left">Video-LLaMA</td>
644            <td style="text-align: center">CLIP-G</td>
645            <td style="text-align: center">7B</td>
646            <td>27.5</td>
647            <td>25.5</td>
648            <td>51.0</td>
649            <td>29.0</td>
650            <td>39.0</td>
651            <td>48.0</td>
652            <td>40.5</td>
653            <td>38.0</td>
654            <td>22.5</td>
655            <td>22.5</td>
656            <td>43.0</td>
657            <td>34.0</td>
658            <td>22.5</td>
659            <td>32.5</td>
660            <td><span class="is-3">45.5</span></td>
661            <td>32.5</td>
662            <td>40.0</td>
663            <td>30.0</td>
664            <td>21.0</td>
665            <td>37.0</td>
666            <td>34.1</td>
667          </tr>
668          <tr>
669            <td style="text-align: left">LLaMA-Adapter</td>
670            <td style="text-align: center">ViT-B</td>
671            <td style="text-align: center">7B</td>
672            <td>23.0</td>
673            <td>28.0</td>
674            <td>51.0</td>
675            <td>30.0</td>
676            <td>33.0</td>
677            <td>53.5</td>
678            <td>32.5</td>
679            <td>33.5</td>
680            <td>25.5</td>
681            <td>21.5</td>
682            <td>30.5</td>
683            <td>29.0</td>
684            <td>22.5</td>
685            <td>41.5</td>
686            <td>39.5</td>
687            <td>25.0</td>
688            <td>31.5</td>
689            <td>22.5</td>
690            <td>28.0</td>
691            <td>32.0</td>
692            <td>31.7</td>
693          </tr>
694          <tr>
695            <td style="text-align: left">Video-ChatGPT</td>
696            <td style="text-align: center">ViT-L</td>
697            <td style="text-align: center">7B</td>
698            <td>23.5</td>
699            <td>26.0</td>
700            <td>62.0</td>
701            <td>22.5</td>
702            <td>26.5</td>
703            <td>54.0</td>
704            <td>28.0</td>
705            <td><span class="is-2">40.0</span></td>
706            <td>23.0</td>
707            <td>20.0</td>
708            <td>31.0</td>
709            <td>30.5</td>
710            <td>25.5</td>
711            <td>39.5</td>
712            <td><span class="is-1">48.5</span></td>
713            <td>29.0</td>
714            <td>33.0</td>
715            <td>29.5</td>
716            <td>26.0</td>
717            <td>35.5</td>
718            <td>32.7</td>
719          </tr>
720          <tr>
721            <td style="text-align: left">VideoChat</td>
722            <td style="text-align: center">CLIP-G</td>
723            <td style="text-align: center">7B</td>
724            <td>33.5</td>
725            <td>26.5</td>
726            <td>56.0</td>
727            <td>33.5</td>
728            <td>40.5</td>
729            <td>53.0</td>
730            <td>40.5</td>
731            <td>30.0</td>
732            <td>25.5</td>
733            <td>27.0</td>
734            <td>48.5</td>
735            <td>35.0</td>
736            <td>20.5</td>
737            <td>42.5</td>
738            <td><span class="is-2">46.0</span></td>
739            <td>26.5</td>
740            <td>41.0</td>
741            <td>23.5</td>
742            <td>23.5</td>
743            <td>36.0</td>
744            <td>35.5</td>
745          </tr>
746          <tr>
747            <td style="text-align: left">VideoChat2\(^\beta\)</td>
748            <td style="text-align: center">UMT-L</td>
749            <td style="text-align: center">7B</td>
750            <td><span class="is-1">66.0</span></td>
751            <td>47.5</td>
752            <td><span class="is-3">83.5</span></td>
753            <td><span class="is-1">49.5</span></td>
754            <td><span class="is-2">60.0</span></td>
755            <td>58.0</td>
756            <td><span class="is-1">71.5</span></td>
757            <td><span class="is-1">42.5</span></td>
758            <td>23.0</td>
759            <td>23.0</td>
760            <td><span class="is-1">88.5</span></td>
761            <td>39.0</td>
762            <td>42.0</td>
763            <td>58.5</td>
764            <td>44.0</td>
765            <td><span class="is-1">49.0</span></td>
766            <td>36.5</td>
767            <td><span class="is-1">35.0</span></td>
768            <td>40.5</td>
769            <td><span class="is-1">65.5</span></td>
770            <td><span class="is-2">51.1</span></td>
771          </tr>
772          <tr>
773            <td style="text-align: left">PLLaVA 7B\(^\alpha\)</td>
774            <td style="text-align: center">ViT-L</td>
775            <td style="text-align: center">7B</td>
776            <td><span class="is-2">58.0</span></td>
777            <td><span class="is-2">49.0</span></td>
778            <td>55.5</td>
779            <td>41.0</td>
780            <td><span class="is-1">61.0</span></td>
781            <td>56.0</td>
782            <td><span class="is-2">61.0</span></td>
783            <td>36.0</td>
784            <td>23.5</td>
785            <td>26.0</td>
786            <td>82.0</td>
787            <td>39.5</td>
788            <td>42.0</td>
789            <td>52.0</td>
790            <td>45.0</td>
791            <td><span class="is-2">42.0</span></td>
792            <td><span class="is-1">53.5</span></td>
793            <td><span class="is-3">30.5</span></td>
794            <td><span class="is-1">48.0</span></td>
795            <td>31.0</td>
796            <td>46.6</td>
797          </tr>
798          <tr>
799            <td style="text-align: left"><span class="is-ignorable">PLLaVA 13B</span>\(^\alpha\)</td>
800            <td style="text-align: center">ViT-L</td>
801            <td style="text-align: center">13B</td>
802            <td>66.0</td>
803            <td>53.0</td>
804            <td>65.5</td>
805            <td>45.0</td>
806            <td>65.0</td>
807            <td>58.0</td>
808            <td>64.5</td>
809            <td>35.5</td>
810            <td>23.5</td>
811            <td>30.0</td>
812            <td>85.0</td>
813            <td>39.5</td>
814            <td>45.5</td>
815            <td>57.0</td>
816            <td>47.5</td>
817            <td>49.5</td>
818            <td>49.0</td>
819            <td>33.0</td>
820            <td>53.0</td>
821            <td>37.0</td>
822            <td><span class="is-ignorable">50.1</span></td>
823          </tr>
824          <tr>
825            <td style="text-align: left"><strong>VideoLLaMB</strong>\(^\alpha\)</td>
826            <td style="text-align: center">ViT-L</td>
827            <td style="text-align: center">7B</td>
828            <td>52.0</td>
829            <td><span class="is-1">50.5</span></td>
830            <td><span class="is-2">85.5</span></td>
831            <td>42.5</td>
832            <td>51.0</td>
833            <td>69.5</td>
834            <td>56.0</td>
835            <td><span class="is-3">38.5</span></td>
836            <td>41.0</td>
837            <td>24.0</td>
838            <td>69.5</td>
839            <td><span class="is-3">40.0</span></td>
840            <td><span class="is-2">48.0</span></td>
841            <td><span class="is-2">71.5</span></td>
842            <td>43.5</td>
843            <td>34.5</td>
844            <td>41.5</td>
845            <td>29.5</td>
846            <td>38.0</td>
847            <td><span class="is-2">60.0</span></td>
848            <td><span class="is-3">49.3</span></td>
849          </tr>
850          <tr>
851            <td style="text-align: left"><strong>VideoLLaMB</strong>\(^\beta\)</td>
852            <td style="text-align: center">ViT-L</td>
853            <td style="text-align: center">7B</td>
854            <td><span class="is-3">54.5</span></td>
855            <td>47.0</td>
856            <td><span class="is-1">86.5</span></td>
857            <td><span class="is-2">44.5</span></td>
858            <td><span class="is-3">52.0</span></td>
859            <td><span class="is-1">79.0</span></td>
860            <td><span class="is-3">58.5</span></td>
861            <td>32.0</td>
862            <td><span class="is-1">47.0</span></td>
863            <td><span class="is-1">33.0</span></td>
864            <td><span class="is-2">82.5</span></td>
865            <td><span class="is-2">40.5</span></td>
866            <td><span class="is-1">52.0</span></td>
867            <td><span class="is-1">82.0</span></td>
868            <td>40.5</td>
869            <td><span class="is-3">37.5</span></td>
870            <td><span class="is-2">43.0</span></td>
871            <td><span class="is-2">31.0</span></td>
872            <td><span class="is-2">42.5</span></td>
873            <td><span class="is-2">60.0</span></td>
874            <td><span class="is-1">52.5</span></td>
875          </tr>
876        </tbody>
877      </table>
878      <p><span class="dnerf">Table 1.</span> <b>Results on <a href="https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md">MVBench</a> multi-choice question answering.</b>
878 The top 3 results among 7B models are highlighted. \(\alpha\): training with data from <a href="https://pllava.github.io">PLLaVA</a>. \(\beta\): training with data from <a href="https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md">MVBench</a>.</p>
879
880    </div>
881
882    <div class="columns is-centered has-text-centered">
883<div class="column is-centered ">
884
885        <table class=" table is-narrow is-hoverable is-fullwidth">
886  <thead>
887    <tr>
888      <th><b>Model</b></th>
889      <th><b>LLM</b></th>
890      <th><b>Frames</b></th>
891      <th><b>Accuracy</b></th>
892    </tr>
893  </thead>
894  <tbody>
895    <tr>
896      <td style="text-align: left">GPT4-o</td>
897      <td>OpenAI API</td>
898      <td>16</td>
899      <td>72.2</td>
900    </tr>
901    <tr>
902      <td colspan="3" style="text-align: left"><small><i>Retrieval-based Video-Language Models</i></small></td>
903    </tr>
904    <tr>
905      <td style="text-align: left">LongViViT*</td>
906      <td>-</td>
907      <td>256</td>
908      <td>56.8</td>
909    </tr>
910    <tr>
911      <td style="text-align: left">MC-ViT-L*</td>
912      <td>-</td>
913      <td>128</td>
914      <td>62.5</td>
915    </tr>
916    <tr>
917      <td colspan="3" style="text-align: left"><small><i>Generative Video-Language Models</i></small></td>
918    </tr>
919    <tr>
920      <td style="text-align: left">SeViLA</td>
921      <td>Flan-T5-XL</td>
922      <td>32</td>
923      <td>25.8</td>
924    </tr>
925    <tr>
926      <td style="text-align: left">mPLUG-Owl</td>
927      <td>LLaMA-7B</td>
928      <td>5</td>
929      <td>33.8</td>
930    </tr>
931    <tr>
932      <td style="text-align: left">VideoLLaVA</td>
933      <td>Vicuna-7B</td>
934      <td>8</td>
935      <td>40.2</td>
936    </tr>
937    <tr>
938      <td style="text-align: left">LLaVA-NeXT-Video-DPO</td>
939      <td>Vicuna-7B</td>
940      <td>32</td>
941      <td>41.6</td>
942    </tr>
943    <tr>
944      <td style="text-align: left">PLLaVA</td>
945      <td>Vicuna-7B</td>
946      <td>16 (16)</td>
947      <td>45.6</td>
948    </tr>
949    <tr>
950      <td style="text-align: left">PLLaVA</td>
951      <td>Vicuna-7B</td>
952      <td>32 (16)</td>
953      <td>43.8</td>
954    </tr>
955    <tr>
956      <td style="text-align: left"><b>VideoLLaMB</b></td>
957      <td>Vicuna-7B</td>
958      <td>32 (8)</td>
959      <td><b>53.8</b></td>
960    </tr>
961  </tbody>
962</table>
963
964        <p><span class="dnerf">Table 2.</span> <b>Results on subset of <a href="https://egoschema.github.io/">EgoSchema</a> under zero-shot setting.</b>  \(^*\) indicates that the model has been fine-tuned using the training data from <a href="https://egoschema.github.io/">EgoSchema</a>.</p>
965
966      </div>
967
968<div class="column  is-centered">
969
970        <table class=" table is-narrow is-hoverable is-fullwidth">
971    <thead>
972        <tr>
973            <th><b>Model</b></th>
974            <th><b>Temporal</b></th>
975            <th><b>Causal</b></th>
976            <th><b>Description</b></th>
977            <th><b>All</b></th>
978        </tr>
979    </thead>
980    <tbody>
981        <tr>
982            <td style="text-align: left">GPT4-o</td>
983            <td>70.3</td>
984            <td>78.0</td>
985            <td>80.8</td>
986            <td>76.0</td>
987        </tr>
988        <tr>
989            <td colspan="5" style="text-align: left"><small><i>Retrieval-based Video-Language Models</i></small></td>
990        </tr>
991        <tr>
992            <td style="text-align: left">AIO*</td>
993            <td>48.0</td>
994            <td>48.6</td>
995            <td>63.2</td>
996            <td>50.6</td>
997        </tr>
998        <tr>
999            <td style="text-align: left">VQA-T*</td>
1000            <td>49.6</td>
1001            <td>51.5</td>
1002            <td>63.2</td>
1003            <td>52.3</td>
1004        </tr>
1005        <tr>
1006            <td style="text-align: left">ATP*</td>
1007            <td>50.2</td>
1008            <td>53.1</td>
1009            <td>66.8</td>
1010            <td>54.3</td>
1011        </tr>
1012        <tr>
1013            <td style="text-align: left">VGT*</td>
1014            <td>52.3</td>
1015            <td>55.1</td>
1016            <td>64.1</td>
1017            <td>55.0</td>
1018        </tr>
1019        <tr>
1020            <td style="text-align: left">MIST-CLIP*</td>
1021            <td>56.6</td>
1022            <td>54.6</td>
1023            <td>66.9</td>
1024            <td>57.1</td>
1025        </tr>
1026        <tr>
1027            <td colspan="5" style="text-align: left"><small><i>Generative Video-Language Models</i></small></td>
1028        </tr>
1029        <tr>
1030            <td style="text-align: left">SeViLA</td>
1031            <td>61.5</td>
1032            <td>61.3</td>
1033            <td>75.6</td>
1034            <td>63.6</td>
1035        </tr>
1036        <tr>
1037            <td style="text-align: left">LLaMA-VID</td>
1038            <td>53.8</td>
1039            <td>60.0</td>
1040            <td>73.0</td>
1041            <td>59.5</td>
1042        </tr>
1043        <tr>
1044            <td style="text-align: left">VideoLLaVA</td>
1045            <td>56.9</td>
1046            <td>61.0</td>
1047            <td>75.0</td>
1048            <td>61.3</td>
1049        </tr>
1050        <tr>
1051            <td style="text-align: left">LLaVA-NeXT-Video-DPO</td>
1052            <td>55.6</td>
1053            <td>61.0</td>
1054            <td>73.9</td>
1055            <td>61.3</td>
1056        </tr>
1057        <tr>
1058            <td style="text-align: left">PLLaVA*</td>
1059            <td>62.2</td>
1060            <td>68.5</td>
1061            <td><strong>79.7</strong></td>
1062            <td>68.2</td>
1063        </tr>
1064        <tr>
1065            <td style="text-align: left"><strong>VideoLLaMB*</strong></td>
1066            <td><strong>66.8</strong></td>
1067            <td><strong>71.6</strong></td>
1068            <td>78.4</td>
1069            <td><strong>71.1</strong></td>
1070        </tr>
1071    </tbody>
1072</table>
1073
1074        <p><span class="dnerf">Table 3.</span> <b>Comparison accuracy on <a href="hhttps://github.com/doc-doc/NExT
1074-QA">NExT-QA</a>.</b>  \(^*\) indicates that the instruction data includes the training data from NExT-QA.</p>
1075
1076      </div>
1077
1078
1079</div>
1080
1081    <details open="">
1082<summary><b>Streaming Caption</b></summary>
1083<div class="is-centered has-text-centered" style="background-color:rgba(117, 209, 215, 0.1)">
1084
1085<h4 style="font-size: 20px; padding: 10px 0 10px;">Task: Describe the streaming video in real-time.</h4>
1086  <video poster="" id="streaming_caption" autoplay="" controls="" muted="" loop="" playsinline="" height="70%">
1087    <source src="/assets/img/streaming_caption.mp4" />
1088  </video>
1089</div>
1090</details>
1091
1092    <!-- </div>
1093</section>
1094
1095
1096<section class="section" >
1097    <div class="container" markdown="1"> -->
1098
1099    <p><br /></p>
1100
1101    <h2 class="title is-3 has-text-centered" id="egocentric-embodied-planning">Egocentric Embodied Planning</h2>
1102
1103    <div class="columns is-centered has-text-centered">
1104<div class="column is-centered ">
1105
1106        <table class="table is-narrow is-hoverable is-fullwidth">
1107  <thead>
1108    <tr>
1109      <th style="text-align: left; font-weight: bold; text-align: center;">Model</th>
1110      <th style="font-weight: bold; text-align: center;">LLM</th>
1111      <th style="font-weight: bold; text-align: center;">Accuracy</th>
1112    </tr>
1113  </thead>
1114  <tbody>
1115    <tr>
1116      <td style="text-align: left;">GPT-4V</td>
1117      <td style="text-align: center;">OpenAI API</td>
1118      <td style="text-align: center;">37.98</td>
1119    </tr>
1120    <tr>
1121      <td colspan="3" style="text-align: left; font-size: small; font-style: italic;">Image-Language Model</td>
1122    </tr>
1123    <tr>
1124      <td style="text-align: left;">Qwen-VL-Chat</td>
1125      <td style="text-align: center;">Qwen-7B</td>
1126      <td style="text-align: center;">26.32</td>
1127    </tr>
1128    <tr>
1129      <td style="text-align: left;">LLaVA-1.5</td>
1130      <td style="text-align: center;">Vicuna-7B</td>
1131      <td style="text-align: center;">26.80</td>
1132    </tr>
1133    <tr>
1134      <td style="text-align: left;">SEED-LLaMA</td>
1135      <td style="text-align: center;">LLaMA2-Chat-13B</td>
1136      <td style="text-align: center;">29.93</td>
1137    </tr>
1138    <tr>
1139      <td style="text-align: left;">InternLM-Xcomposer</td>
1140      <td style="text-align: center;">InternLM-7B</td>
1141      <td style="text-align: center;">34.4</td>
1142    </tr>
1143    <tr>
1144      <td colspan="3" style="text-align: left; font-size: small; font-style: italic;">Video-Language Model</td>
1145    </tr>
1146    <tr>
1147      <td style="text-align: left;">VideoChatGPT</td>
1148      <td style="text-align: center;">LLaMA-7B</td>
1149      <td style="text-align: center;">26.35</td>
1150    </tr>
1151    <tr>
1152      <td style="text-align: left;">Valley</td>
1153      <td style="text-align: center;">LLaMA-13B</td>
1154      <td style="text-align: center;">26.17</td>
1155    </tr>
1156    <tr>
1157      <td style="text-align: left;">VideoLLaMA</td>
1158      <td style="text-align: center;">LLaMA2-Chat-7B</td>
1159      <td style="text-align: center;">29.85</td>
1160    </tr>
1161    <tr>
1162      <td style="text-align: left;">LLaVA-NeXT-Video</td>
1163      <td style="text-align: center;">Vicuna-7B</td>
1164      <td style="text-align: center;">28.96</td>
1165    </tr>
1166    <tr>
1167      <td style="text-align: left;">PLLaVA</td>
1168      <td style="text-align: center;">Vicuna-7B</td>
1169      <td style="text-align: center;">30.26</td>
1170    </tr>
1171    <tr>
1172      <td style="text-align: left; font-weight: bold;">VideoLLaMB-7B</td>
1173      <td style="text-align: center;">Vicuna-7B</td>
1174      <td style="text-align: center; font-weight: bold;">32.32</td>
1175    </tr>
1176  </tbody>
1177</table>
1178
1179        <!-- <figcaption> -->
1180        <p><span class="dnerf">Table 4.</span> <b>Results on <a href="https://github.com/ChenYi99/EgoPlan">EgoPlan</a> under Zero-shot setting.</b></p>
1181
1182      </div>
1183
1184<div class="column  is-centered has-text-centered">
1185
1186        <figure class="image">
1187      <b>Goal:</b> <em>clean and organize kitchen</em>
1188      <img src="/assets/img/case.png" />
1189      <figcaption><span class="dnerf">Figure 2.</span> <b>Qualitative results on <a href="https://github.com/ChenYi99/EgoPlan">EgoPlan</a>.</b> </figcaption>
1190</figure>
1191
1192      </div>
1193
1194
1195</div>
1196
1197  </div>
1198</section>
1199
1200<section class="section">
1201    <div class="container is-max-desktop">
1202
1203    <h2 class="title is-3 has-text-centered" id="stress-test-needle-in-a-video-haystack">Stress Test: “Needle In A Video Haystack”</h2>
1204
1205    <div class="columns is-centered has-text-centered">
1206<div class="column">
1207
1208<figure class="image">
1209      <b>Needle:</b> "<em>A young man is sitting on a piece of cloud in the sky, reading a book.</em>"
1210      <img src="/assets/img/case_needle.png" />
1211      <figcaption><span class="dnerf">Figure 3.</span> <b>An example of NIAVH.</b> </figcaption>
1212</figure>
1213
1214</div>
1215</div>
1216
1217    <p>We utilize ego-centric videos from the Ego4D dataset as the “haystack”. Within this haystack, we seek to locate the “needle”, which we provide in three distinct modalities. For the textual modality, we supply a crafted description. For the image modality, we employ DALL-E to create an image that visually represents this description. For the video modality, we use Sora to generate a short video clip based on the same description. In each case, the “needle” - whether text, image, or video - is set to a duration of 1 second.</p>
1218
1219  </div>
1220</section>
1221
1222<section class="section" style="background-color:#efeff081">
1223    <div class="container is-max-desktop">
1224
1225    <h2 class="title is-3 has-text-centered" id="technical-details">Technical Details</h2>
1226
1227    <h3 class="title is-4" id="scene-tiling-segmentation-with-semantics">Scene Tiling: Segmentation with Semantics</h3>
1228
1229    <p>We introduce SceneTilling, a <em>model-free</em> scene segmentation algorithm, to divide the entire video sequence into video segments such that each segment is semantically non-overlap with others, <em>i.e.</em>, inter-segment coherence. Formally, given a sequence of \(n\) frames \(\{v_1, v_2, \ldots, v_n\}\), the SceneTiling algorithm is as follows.</p>
1230    <ol>
1231      <li>Compute the cosine similarity \(S_C(\cdot, \cdot)\) between adjacent frame pairs using the [CLS] token from ViT, resulting in a sequence of similarity scores \(\{c_1, c_2, \ldots, c_{n-1}\}\), where \(c_i = S_C ({\rm ViT}(v_i), {\rm ViT}(v_{i+1}))\).</li>
1232      <li>Calculate the depth score for each point as \(d_i = \left(cl_i+cr_i-2c_i\right)/{2}\), where \(cl_i\) and \(cr_i\) are the highest score to the left and right of \(c_i\), respectively. A higher depth score indicates that the surrounding similarity is greater than at the point itself.</li>
1233      <li>Calculate the expectation \(\mu\) and variance \(\sigma\) of the depth scores \(\{d_1, d_2, \ldots, d_{n-1}\}\). Set the segmentation threshold as \(\mu + \alpha \cdot \sigma\), where \(\alpha\) is a hyperparameter controlling the likelihood of segmenting the video.</li>
1234      <li>Select the \(K-1\) depth scores that exceed the threshold to divide the video into \(K\) semantic segments \(\{s_1, s_2, \ldots, s_K\}\). Each segment represents a relatively independent semantic unit consisting of a sequence of frames.
1235<br /></li>
1236    </ol>
1237
1238    <h3 class="title is-4" id="recurrent-memory-bridge-layers">Recurrent Memory Bridge Layers</h3>
1239
1240    <p>We devised a novel Recurrent Memory Bridge Layer, implemented as a multi-layer Transformer block, that integrates recurrent memory tokens within bridge layers to enhance the linear layer’s memorization ability.</p>
1241
1242    <p>For each video segment \(s_i\), we prepend a fixed number of memory tokens, denoted as \([m_i; s_i]\), where \(m_i\) represents the memory tokens. Subsequently, we apply standard self-attention to this sequence, yielding \([m_{i+1}; o_{i}] = {\rm BridgeLayer}([m_i; s_i])\). Here, \(m_{i+1}\) is the updated memory token, and \(o_{i}\) is the visual representation from the bridge layers.</p>
1243
1244    <p>As such, the Memory Bridge can <strong>compress past video into memory tokens while preserving current video scenes through projection without losing detailed information by compressing</strong>.</p>
1245
1246    <!-- Formally, for each video segment $$s_i$$, we prepend a fixed number of memory tokens, denoted as $$[m_i; s_i]$$, where $$m_i$$ represents the memory tokens. Subsequently, we apply standard self-attention to this sequence, yielding $$[m_{i+1}; o_{i+1}] = {\rm BridgeLayer}([m_i; s_i])$$. Here, $$m_{i+1}$$ is the updated memory token, and $$o_{i+1}$$ is the refreshed visual representation.  -->
1247    <!-- This process is carried out recursively, traversing the semantic video segments while updating the memory tokens. After a total of $$k$$ steps, the final output $$o_k$$ can be obtained as the condensed visual representation of the video sequence. This methodology allows us to encode an entire video into a concise sequence while retaining all pertinent information from previous segments. -->
1248
1249    <p><br /></p>
1250
1251    <h3 class="title is-4" id="memory-cache-with-retrieval">Memory Cache with Retrieval</h3>
1252
1253    <p>One of the primary challenges associated with recurrent memory bridge layers is the potential for gradient vanishing, which can impede the model’s ability to learn long-range dependencies. To mitigate this issue, we propose the incorporation of a <strong>memory cache</strong> with a retrieval strategy designed to preserve previous states of memory.</p>
1254
1255    <p><strong>Memory Attention</strong> At each timestep \(i\), the system stores all previous memory tokens in a memory cache, denoted as \(M_i = [m_1, \ldots, m_i]\). We employ a self-retrieval mechanism to update the current memory token \(m_i\). Specifically, we treat \(m_i\) as a query and the concatenated memory cache \(M_i\) as key and value. The model performs a standard multi-head cross-attention operation to integrate information from previous timesteps into the current memory state, yielding the updated memory token</p>
1256
1257\[m_{i+1} = \text{Softmax}\left(\frac{W_i^Q m_i (W_i^K M_i)^\top}{\sqrt{d_k}}\right) W_i^V M_i,\]
1258
1259    <p>where \(W_i^Q, W_i^K, W_i^V\) are weight martices for query, key and value, respectively.</p>
1260
1261  </div>
1262</section>
1263
1264<section class="section">
1265    <div class="container is-max-desktop">
1266
1267    <h2 class="title" id="citation">Citation</h2>
1268
1269    <div class="language-bibtex highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nc">@article</span><span class="p">{</span><span class="nl">wang2024videollamb</span><span class="p">,</span>
1270    <span class="na">title</span><span class="p">=</span><span class="s">{VideoLLaMB: Long-context Video Understanding with Recurrent Memory Bridges}</span><span class="p">,</span>
1271    <span class="na">author</span><span class="p">=</span><span class="s">{Wang, Yuxuan and Xie, Cihang and Liu, Yang and Zheng, Zilong}</span><span class="p">,</span>
1272    <span class="na">
1272journal</span><span class="p">=</span><span class="s">{arXiv preprint arXiv:2409.01071}</span><span class="p">,</span>
1273    <span class="na">year</span><span class="p">=</span><span class="s">{2024}</span>
1274<span class="p">}</span>
1275</code></pre></div>    </div>
1276
1277  </div>
1278</section>
1279
1280
1281    </div>
1282
1283    <!-- Footer -->
1284
1285    <footer class="footer">
1286    <div class="container">
1287        <div class="columns is-centered">
1288            <div class="column is-8">
1289                <div class="content">
1290                    <p>
1291                        This website was built using a <a href="https://jekyllrb.com/">Jekyll</a> template adapted from
1292                        <a href="https://github.com/nerfies/nerfies.github.io">Nerfies</a>,
1293                        licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
1294                            Commons Attribution-ShareAlike 4.0 International License</a>.
1295                    </p>
1296                </div>
1297            </div>
1298        </div>
1299    </div>
1300</footer>
1301
1302<!-- jQuery -->
1303<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js"
1304    integrity="sha512-bLT0Qm9VnAYZDflyKcBaQ2gg0hSYNQrJ8RilYldYQ1FxQYoCLtUjuuRuZo+fjqhx/qtq/1itJ0C2ejDxltZVFg==" crossorigin="anonymous"></script>
vendor: 1 bytes, line 1304
1304
1305<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-easing/1.4.1/jquery.easing.min.js"
1306    crossorigin="anonymous"></script>
1306
1307
1308<!-- Bulma -->
1309<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bulma-slider.min.js"></script>
vendor: 1 bytes, line 1309
1309
1310<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bulma-carousel.min.js"></script>
1310
1311
1312<script src="//cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/js/toastr.min.js"></script>
1312
1313
1314<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/prism.min.js">
1315</script>
vendor: 1 bytes, line 1315
1315
1316<script src="https://cdn.jsdelivr.net/npm/[email protected]/prism-bibtex.min.js">
1317</script>
1317
1318
1319</body>
1320
1321
1322</html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.