PageSourceSearch

https://minahuh.com/ShortScribe/

html minahuh.com collected 2026-09-25 20:32:26 UTC 23,511 bytes, 321 lines download raw bytes

1<!DOCTYPE html>
2<html>
3<head>
4  <meta charset="utf-8">
5  <!-- Meta tags for social media banners, these should be filled in appropriatly as they are your "business card" -->
6  <!-- Replace the content tag with appropriate information -->
7  <meta name="description" content="ShortScribe: Making Short-Form Videos Accessible with Hierarchical Video Summaries">
8  <meta property="og:title" content=""/>
9  <meta property="og:description" content=""/>
10  <meta property="og:url" content="URL OF THE WEBSITE"/>
11  <!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X630-->
12  <meta property="og:image" content="static/image/your_banner_image.png" />
13  <meta property="og:image:width" content="1200"/>
14  <meta property="og:image:height" content="630"/>
15
16
17  <!-- <meta name="twitter:title" content="TWITTER BANNER TITLE META TAG">
18  <meta name="twitter:description" content="TWITTER BANNER DESCRIPTION META TAG"> -->
19  <!-- Path to banner image, should be in the path listed below. Optimal dimenssions are 1200X600-->
20  <meta name="twitter:image" content="static/images/your_twitter_banner_image.png">
21  <meta name="twitter:card" content="summary_large_image">
22  <!-- Keywords for your paper to be indexed by-->
23  <meta name="keywords" content="KEYWORDS SHOULD BE PLACED HERE">
24  <meta name="viewport" content="width=device-width, initial-scale=1">
25
26
27  <title>Making Short-Form Videos Accessible with Hierarchical Video Summaries</title>
28  <link rel="icon" type="image/x-icon" href="static/images/favicon.png">
29  <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
30  rel="stylesheet">
31
32  <link rel="stylesheet" href="static/css/bulma.min.css">
33  <link rel="stylesheet" href="static/css/bulma-carousel.min.css">
34  <link rel="stylesheet" href="static/css/bulma-slider.min.css">
35  <link rel="stylesheet" href="static/css/fontawesome.all.min.css">
36  <link rel="stylesheet"
37  href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
38  <link rel="stylesheet" href="static/css/index.css">
39
40  
40<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
40
41  
41<script src="https://documentcloud.adobe.com/view-sdk/main.js"></script>
41
42  
42<script defer src="static/js/fontawesome.all.min.js"></script>
42
43  
43<script src="static/js/bulma-carousel.min.js"></script>
43
44  
44<script src="static/js/bulma-slider.min.js"></script>
44
45  
45<script src="static/js/index.js"></script>
45
46</head>
47<body>
48
49
50  <section class="hero">
51    <div class="hero-body">
52      <div class="container is-max-desktop">
53        <div class="columns is-centered">
54          <div class="column has-text-centered">
55            <h1 class="title publication-title">Making Short-Form Videos Accessible <br>with Hierarchical Video Summaries</h1>
56            <div class="is-size-5 publication-authors">
57              <!-- Paper authors -->
58                <span class="author-block">
59                  <a href="https://www.linkedin.com/in/tess-van-daele" target="_blank">Tess Van Daele</a><sup>1</sup>,</span>
60                  <span class="author-block">
61                    <a href="" target="_blank"> Akhil Iyer</a><sup>1</sup>,</span>
62                    <span class="author-block">
63                      <a href="" target="_blank">Yuning Zhang</a><sup>2</sup>,</span>
64                      <span class="author-block">
65                        <a href="https://www.linkedin.com/in/jalyn-derry/" target="_blank">Jalyn Derry</a><sup>1</sup>,</span>
66                      <span class="author-block">
67                        <a href="http://www.minahuh.com" target="_blank">Mina Huh</a><sup>1</sup>,</span>
68                <span class="author-block">
69                  <a href="http://www.amypavel.com" target="_blank">Amy Pavel</a><sup>1</sup>,</span>
70                  </div>
71
72                  <div class="is-size-5 publication-authors">
73                    <span class="author-block">UT Austin<sup>1</sup>, Cornell University<sup>2</sup><br>ACM CHI 2024</span>
74                  </div>
75
76                  <div class="column has-text-centered">
77                    <div class="publication-links">
78                         <!-- Arxiv PDF link -->
79                      <span class="link-block">
80                        <a href="https://dl.acm.org/doi/10.1145/3613904.3642839" target="_blank"
81                        class="external-link button is-normal is-rounded is-dark">
82                        <span class="icon">
83                          <i class="fas fa-file-pdf"></i>
84                        </span>
85                        <span>Paper</span>
86                      </a>
87                    </span>
88
89                    <!-- Supplementary PDF link -->
90                    <!-- <span class="link-block">
91                      <a href="static/pdfs/supplementary_material.pdf" target="_blank"
92                      class="external-link button is-normal is-rounded is-dark">
93                      <span class="icon">
94                        <i class="fas fa-file-pdf"></i>
95                      </span>
96                      <span>Supplementary</span>
97                    </a>
98                  </span> -->
99
100                  <!-- Github link -->
101                <span class="link-block">
102                    <a href="https://github.com/Akl35/shortscribe?tab=readme-ov-file" target="_blank"
103                    class="external-link button is-normal is-rounded is-dark">
104                    <span class="icon">
105                      <i class="fab fa-github"></i>
106                    </span>
107                    <span>Code</span>
108                  </a>
109                </span>
110
111                <span class="link-block">
112                    <a href="https://master--sparkly-alfajores-a5ba27.netlify.app/" target="_blank"
113                    class="external-link button is-normal is-rounded is-dark">
114                    <span class="icon">
115                      <i class="fa fa-mobile"></i>
116                    </span>
117                    <span>Demo</span>
118                  </a>
119                </span>
120
121                <!-- ArXiv abstract Link -->
122<!--                 <span class="link-block">
123                  <a href="https://arxiv.org/abs/2402.10382" target="_blank"
124                  class="external-link button is-normal is-rounded is-dark">
125                  <span class="icon">
126                    <i class="ai ai-arxiv"></i>
127                  </span>
128                  <span>arXiv</span>
129                </a>
130              </span> -->
131              <!-- Demo link -->
132<!--               <span class="link-block">
133                <a 
134                class="external-link button is-normal is-rounded ">
135                <span class="icon">
136                  <i class="fa fa-image"></i>
137                </span>
138                <span>Demo coming soon...</span>
139              </a>
140            </span> -->
141            </div>
142          </div>
143        </div>
144      </div>
145    </div>
146  </div>
147</section>
148
149
150<!-- Teaser video-->
151<section class="hero teaser">
152  <div class="container is-max-desktop">
153    <div class="hero-body">
154      <img src="static/images/teaser-revised.jpg" alt="The figure shows information from left to right. The most left is the original video represented as a single frame of a woman holding a bowl of salad with the text “Let's make the salad Jennifer Aniston ate every day on the set of Friends.” Two arrows point to the right from this picture. One points to the key frames which is a stack of single frames showing ingredients being added to a bowl. The other arrow points to the transcript of the video (transcribed with automatic speech recognition). 
155      Below the keyframes are two arrows (one labeled with the vision to language model  BLIP-2) and one labelled OCR both pointing to Keyframe Text \& Descriptions. To the right of the key frames, key frame text \& description and transcript is an arrow (labelled LLM GPT-4) pointing right to two descriptions which includes on-screen text description (“Let's make the salad Jennifer Aniston ate … 3 cups cooked quinoa 1 cup cucumber 1/3 cup red onion 1/2 cup roasted, salted pistachios 1/2 cup mint … Salt & pepper”) and the shot-by-shot-description (“Shot 1: A 3-second shot showing a lady holding a large salad bowl, mentioning she is making the salad Jennifer Aniston ate…Shot 10: The last shot ends with the woman eating the salad”). 
156      To the right of these two descriptions an arrow labelled LLM and GPT-4 points to a long description (“The video showcases a woman preparing and tasting a quinoa salad, attributed to Jennifer Aniston's diet on the 'Friends' set. The salad is made from quinoa, cucumber, …and pepper.”). To the right of the descriptions is an arrow also labelled LLM and GPT-4 pointing right to a short description (“Video shows woman making Jennifer Aniston's rumored 'Friends' quinoa salad.”)."/>
157      <h2 class="subtitle has-text-centered">
158        <b>ShortScribe</b> makes short-form videos accessible with hierarchical video descriptions. ~\sysname{} extracts video data by identifying key frames then applying automatic speech recognition (ASR), automated description (BLIP-2), and optical character recognition (OCR). A large language model (GPT-4) then generates multiple descriptions.
159      </h2>
160    </div>
161  </div>
162</section>
163<!-- End teaser video -->
164
165<!-- Paper abstract -->
166<section class="section hero is-light">
167  <div class="container is-max-desktop">
168    <div class="columns is-centered has-text-centered">
169      <div class="column is-four-fifths">
170        <h2 class="title is-3">Abstract</h2>
171        <div class="content has-text-justified">
172          <p>
173            Short videos on platforms such as TikTok, Instagram Reels, and YouTube Shorts (i.e. short-form videos) have become a primary source of information and entertainment. Many short-form videos are inaccessible to blind and low vision (BLV) viewers due to their rapid visual changes, on-screen text, and music or meme-audio overlays. In our formative study, 7 BLV viewers who regularly watched short-form videos reported frequently skipping such inaccessible content. We present ShortScribe, a system that provides hierarchical visual summaries of short-form videos at three levels of detail to support BLV viewers in selecting and understanding short-form videos. ShortScribe allows BLV users to navigate between video descriptions based on their level of interest. To evaluate ShortScribe, we assessed description accuracy and conducted a user study with 10 BLV participants comparing ShortScribe to a baseline interface. When using ShortScribe, participants reported higher comprehension and provided more accurate summaries of video content.
174          </p>
175        </div>
176      </div>
177    </div>
178  </div>
179</section>
180<!-- End paper abstract -->
181
182
183
184<section class="hero is-small">
185  <div class="hero-body">
186    <div class="container">
187      <h2 class="title is-3">System</h2>
188      <div class="columns is-centered has-text-centered">
189        <div class="column is-four-fifths">
190          <img src="static/images/Interface.jpg" width="50%" alt="The figure shows two screens of a mobile interface, one on the left and one on the right. Left: A still frame of a woman holding a bowl of salad. Within the video, there is text that reads “Let's make the salad Jennifer Aniston ate every day on the set of Friends.” Elements on top of the video are grouped into four groups A, B, C, and D. In group A, in the bottom left, video text information is displayed (marked with an A) including a short description (“Video shows woman making Jennifer Aniston's rumored 'Friends' quinoa s
190alad.”), username (“@nourished.by.mads”), video caption (“This salad is actually AMAZING - no joke. Ingredients and measuring portions are all listed in the video #salad #saladrecipe #jenniferanistonsalad #friendstvshow #jeniferaniston”), and audio title (“Just a Cloud Away - @Pharell Williams”). The screen also includes button icons aligned vertically along the right of the screen. Group B is the top three buttons which includes previous (upward arrow), play/pause (play icon), next (downward arrow). Group C includes the video description button (information icon). Group D includes like (heart), comment (speech bubble), bookmark (save icon), and share (share icon). Right: A popup is shown that presents three descriptions and a close button in the top right corner. The text includes a long description (“The video showcases a woman preparing and tasting a quinoa salad, attributed to Jennifer Aniston's diet on the 'Friends' set. The salad is made from quinoa, cucumber, red onion, pistachios, mint, parsley, feta, chickpeas, lemon juice, olive oil, salt, and pepper.”), on-screen text (“Let's make the salad Jennifer Aniston ate every day on the set of Friends 3 cups cooked quinoa 1 cup cucumber 1/3 cup red onion 1/2 cup roasted, salted pistachios … 1/4 cup olive oil Salt & pepper”), and shot-by-shot-descriptions (“Shot 1: A 3-second shot showing a lady holding a large salad bowl, mentioning she is making the salad Jennifer Aniston ate every day on the set of Friends.Shot 2: In a 3-second span, we see a visual guide to preparing 3 cups of cooked quinoa. …”)."/>
191      <h2 class="subtitle has-text-centered">
192       The ShortScribe interface consists of (a) front screen video information including the short description, username, caption, and audio title, (b) video controls, (c) a button to open the description pane which includes the long description, on-screen text, and shot-by-shot descriptions, and (d) video statistics.
193      </h2>
194        </div>
195      </div>
196    </div>
197  </div>
198</section>
199
200<hr style="height: 1.2px; background-color: lightgray;">
201
202<section class="hero is-small">
203  <div class="hero-body">
204    <div class="container">
205      <h2 class="title is-3">Pipeline</h2>
206      <div class="columns is-centered has-text-centered">
207        <div class="column is-four-fifths">
208          <img src="static/images/pipeline.jpg" width="70%" alt="The figure shows a diagram of how the system works. From left to right, the diagram shows 4 groupings or columns, all connected by arrows pointing right. The first group includes Video and Keyframes with an arrow pointing downward from Video to Keyframes. From Video, there is an arrow that says ASR pointing right to Transcript which is in the second group. From Keyframes, there is an arrow that says VLM pointing right to Image Captions and another arrow that says OCR pointing right to on-screen text. Transcript, Image Captions, and on-screen text make up group 2 and are labeled at shot-by-shot details. From group 2, there are two arrows both saying LLM pointing right to long description and shot-by-short descriptions. From group 3, there is one arrow saying LLM pointing right to short description."/>
209      <h2 class="subtitle has-text-centered">
210       ShortScribe takes a video as input, transcribes the audio using automatic speech recognition (ASR), segments the video into shots, and selects the middle frame of each shot as a keyframe. It then processes the transcript, generated image captions (BLIP-2), and on-screen text (OCR) to produce video data for each keyframe. We use a large language model (GPT-4) to summarize this data into a short, long, and shot-by-shot description.
211      </h2>
212        </div>
213      </div>
214    </div>
215  </div>
216</section>
217<hr style="height: 1.2px; background-color: lightgray;">
218
219<section class="hero is-small">
220  <div class="hero-body">
221    <div class="container">
222      <h2 class="title is-3">Pipeline Evaluation</h2>
223      <div class="columns is-centered has-text-centered">
224        <div class="column is-four-fifths">
225          <img src="static/images/hallucinations.jpg" width="70%" alt="A horizontal bar graph of hallucination made by the pipeline. The x-axis is the percentage of videos. The y-axis on the left is description type, which lists (from top to bottom) long, short, 50-word, and per shot. The y-axis on the right is how many videos were analyzed for each description type, which lists (from top to bottom) 58, 58, 58, 18. Short descriptions had the highest percentage of videos with 
225no hallucinations, having 40 out of 58 with no hallucination. The per-shot descriptions had the lowest percentage of videos with no hallucinations, having 7 out of 18 with no hallucination."/>
226          <h2 class="subtitle has-text-centered">
227            We analyzed hallucinations in descriptions for 58 videos (long, short, 50-word descriptions) and for a subsample of 18 videos (per shot descriptions). Descriptions for each video contained 0-7 hallucinations. Short descriptions had the lowest percentage of videos with hallucinations, while shot-by-shot descriptions had the highest percentage of videos with hallucinations.
228        </h2>
229        <img src="static/images/error-analysis.jpg" width="50%" style="margin-top: 5%;" alt="A horizontal bar graph of hallucination made by the pipeline. The x-axis is the percentage of videos. The y-axis on the left is description type, which lists (from top to bottom) long, short, 50-word, and per shot. The y-axis on the right is how many videos were analyzed for each description type, which lists (from top to bottom) 58, 58, 58, 18. Short descriptions had the highest percentage of videos with no hallucinations, having 40 out of 58 with no hallucination. The per-shot descriptions had the lowest percentage of videos with no hallucinations, having 7 out of 18 with no hallucination."/>
230        <h2 class="subtitle has-text-centered">
231          An analysis of the errors in one of the 2 of 58 videos that had more than three errors in the short description. The video depicts a lighthearted singalong. BLIP-2 mistakenly recognizes a toddler concentrating on singing as angry, and the on-screen text shows a quiz with the lyrics to to a sad song (All To Well by Taylor Swift). The long description and then short description incorrectly infer that the video is sad.
232      </h2>
233        </div>
234      </div>
235    </div>
236  </div>
237</section>
238<hr style="height: 1.2px; background-color: lightgray;">
239
240
241<section class="hero is-small">
242  <div class="hero-body">
243    <div class="container">
244      <h2 class="title is-3">User Evaluation</h2>
245      <div class="columns is-centered has-text-centered">
246        <div class="column is-four-fifths">
247          <img src="static/images/short-video-understanding.jpg"  alt="The figure shows two figures. 
248          Left: A vertical bar graph titled “Video Understanding Summary Scores.” The x-axis is video title, listing (from left to right) V1, V4, V6, V2, V7, V5, V3, V8. The y-axis is the average score participants got on their summaries of the videos, listing (from top to bottom) 100\%, 80\%, 60\%, 40\%, 20\%, 0. Each video has two bars, the one on the left being blue (representing participants who used our system) and the one on the right being red (representing participants who used the baseline). V1 has the most significant difference between the system which scored 92\% and the baseline which scored 6\%. V4 had the least difference where the system and baseline both scored 75%.
249          Right: A vertical bar graph titled “Video Understanding Ratings.” The x-axis is video title, listing (from left to right) V5, V1, V4, V3, V8, V7, V6, V2. The y-axis is the average accessibility rating participants gave each on a scale from 1 (not accessible) to 7 (accessible), listing (from top to bottom) 7, 5, 3, 1. Each video has two bars, the one on the left being blue (representing participants who used our system) and the one on the right being red (representing participants who used the baseline). V5 has the most significant difference between the system which scored 6.3 and the baseline which scored 1.5. V4 had the least difference between the system which scored 6 and the baseline which scored 6.5."/>
250          <h2 class="subtitle has-text-centered">
251            Video comprehension for videos V1-V8 using our system (left, blue) and a baseline interface (right, orange) measured by scoring participant written video summaries (Video Summary Scores) and participant's ratings of their video understanding (Video Understanding Ratings). Ratings of the video understanding ranged from 1, did n
251ot understand, to 7, completely understood. Error bars depict the 95% confidence interval.
252        </h2>
253        <img src="static/images/feature_ranking.jpg" width="50%" style="margin-top: 5%;" alt="A horizontal bar graph titled Feature Usefulness Ratings. The x-axis is the number of participants (1 to 10) and the y-axis shows features which read (from top to bottom), “Long description, short description, per shot description, ocr description, original caption, engagement numbers, audio source, username.” The horizontal bars are filled with colors correlating to how the participants ranked the usefulness of each feature, red being definitely not useful and blue being definitely useful. The first four descriptions had overall high ratings while the last four had much lower ratings. "/>
254        <h2 class="subtitle has-text-centered">
255          Participants rated the usefulness of each feature for understanding the video. The description features (first four features) are provided by ShortScribe only, and the remaining features (last four features)  were originally available on the short-form video platform. Engagement numbers refers to the number of likes, comments, and bookmarks.
256      </h2>
257        </div>
258      </div>
259    </div>
260  </div>
261</section>
262
263  <!-- Youtube video -->
264<section class="hero is-small is-light">
265  <div class="hero-body">
266    <div class="container">
267      <h2 class="title is-3">Presentation Video</h2>
268      <div class="columns is-centered has-text-centered">
269        <div class="column is-four-fifths">
270          <div class="publication-video">
271            <iframe src="https://www.youtube.com/embed/s-d2aAdIYpk" frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
272          </div>
273        </div>
274      </div>
275    </div>
276  </div>
277</section>
278<!-- End youtube video -->
279
280
281<!--BibTex citation -->
282  <section class="section" id="BibTeX">
283    <div class="container is-max-desktop content">
284      <h2 class="title">BibTeX</h2>
285      <pre><code>@article{van2024making,
286        title={Making Short-Form Videos Accessible with Hierarchical Video Summaries},
287        author={Van Daele, Tess and Iyer, Akhil and Zhang, Yuning and Derry, Jalyn C and Huh, Mina and Pavel, Amy},
288        journal={arXiv preprint arXiv:2402.10382},
289        year={2024}
290      }</code></pre>
291    </div>
292</section>
293<!--End BibTex citation -->
294
295
296  <footer class="footer">
297  <div class="container">
298    <div class="columns is-centered">
299      <div class="column is-8">
300        <div class="content">
301
302          <p>
303            This page was built using the <a href="https://github.com/eliahuhorwitz/Academic-project-page-template" target="_blank">Academic Project Page Template</a>.
304            <br> This website is licensed under a <a rel="license"  href="http://creativecommons.org/licenses/by-sa/4.0/" target="_blank">Creative
305            Commons Attribution-ShareAlike 4.0 International License</a>.
306          </p>
307
308        </div>
309      </div>
310    </div>
311  </div>
312</footer>
313
314<!-- Statcounter tracking code -->
315  
316<!-- You can add a tracker to track page visits by creating an account at statcounter.com -->
317
318    <!-- End of Statcounter Code -->
319
320  </body>
321  </html>

Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.