1<!DOCTYPE HTML> 2<html> 3 4<!-- ======================================================================= -->
5<script src="http://www.google.com/jsapi" type="text/javascript"></script>
vendor: 1 bytes, line 5
5
6<script type="text/javascript">google.load("jquery", "1.3.2");</script>
6 7 8<link href='https://fonts.googleapis.com/css?family=Titillium Web' rel='stylesheet'> 9<link rel="stylesheet" type="text/css" href="https://cdn.rawgit.com/dreampulse/computer-modern-web-font/master/fonts.css"> 10 11<style type="text/css"> 12 /* body { 13 font-family: "Titillium Web","HelveticaNeue-Light", "Helvetica Neue Light", "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif; 14 font-weight:300; 15 font-size:18px; 16 margin-left: auto; 17 margin-right: auto; 18 width: 1100px; 19 } 20 21 h1 { 22 font-weight:300; 23 } */ 24 body { 25 font-family: "Computer Modern Sans", "Titillium Web", sans-serif; 26 font-weight:300; 27 font-size:18px; 28 margin-left: auto; 29 margin-right: auto; 30 width: 1100px; 31 } 32 33 h1 { 34 font-family: "Computer Modern Sans", "Titillium Web", sans-serif; 35 font-size:32px; 36 font-weight:300; 37 } 38 39 .disclaimerbox { 40 background-color: #eee; 41 border: 1px solid #eeeeee; 42 border-radius: 10px ; 43 -moz-border-radius: 10px ; 44 -webkit-border-radius: 10px ; 45 padding: 20px; 46 } 47 48 video.header-vid { 49 height: 140px; 50 border: 1px solid black; 51 border-radius: 10px ; 52 -moz-border-radius: 10px ; 53 -webkit-border-radius: 10px ; 54 } 55 56 img.header-img { 57 height: 140px; 58 border: 1px solid black; 59 border-radius: 10px ; 60 -moz-border-radius: 10px ; 61 -webkit-border-radius: 10px ; 62 } 63 64 img.rounded { 65 border: 1px solid #eeeeee; 66 border-radius: 10px ; 67 -moz-border-radius: 10px ; 68 -webkit-border-radius: 10px ; 69 } 70 71 a:link,a:visited 72 { 73 color: #1367a7; 74 text-decoration: none; 75 } 76 a:hover { 77 color: #208799; 78 } 79 80 td.dl-link { 81 height: 160px; 82 text-align: center; 83 font-size: 22px; 84 } 85 86 .layered-paper-big { /* modified from: http://css-tricks.com/snippets/css/layered-paper/ */ 87 box-shadow: 88 0px 0px 1px 1px rgba(0,0,0,0.35), /* The top layer shadow */ 89 5px 5px 0 0px #fff, /* The second layer */ 90 5px 5px 1px 1px rgba(0,0,0,0.35), /* The second layer shadow */ 91 10px 10px 0 0px #fff, /* The third layer */ 92 10px 10px 1px 1px rgba(0,0,0,0.35), /* The third layer shadow */ 93 15px 15px 0 0px #fff, /* The fourth layer */ 94 15px 15px 1px 1px rgba(0,0,0,0.35), /* The fourth layer shadow */ 95 20px 20px 0 0px #fff, /* The fifth layer */ 96 20px 20px 1px 1px rgba(0,0,0,0.35), /* The fifth layer shadow */ 97 25px 25px 0 0px #fff, /* The fifth layer */ 98 25px 25px 1px 1px rgba(0,0,0,0.35); /* The fifth layer shadow */ 99 margin-left: 10px; 100 margin-right: 45px; 101 } 102 103 104 .layered-paper { /* modified from: http://css-tricks.com/snippets/css/layered-paper/ */ 105 box-shadow: 106 0px 0px 1px 1px rgba(0,0,0,0.35), /* The top layer shadow */ 107 5px 5px 0 0px #fff, /* The second layer */ 108 5px 5px 1px 1px rgba(0,0,0,0.35), /* The second layer shadow */ 109 10px 10px 0 0px #fff, /* The third layer */ 110 10px 10px 1px 1px rgba(0,0,0,0.35); /* The third layer shadow */ 111 margin-top: 5px; 112 margin-left: 10px; 113 margin-right: 30px; 114 margin-bottom: 5px; 115 } 116 117 .vert-cent { 118 position: relative; 119 top: 50%; 120 transform: translateY(-50%); 121 } 122 123 hr 124 { 125 border: 0; 126 height: 1px; 127 background-image: linear-gradient(to right, rgba(0, 0, 0, 0), rgba(0, 0, 0, 0.75), rgba(0, 0, 0, 0)); 128 } 129 130 #authors td { 131 padding-bottom:5px; 132 padding-top:30px; 133 } 134</style> 135<!-- ======================================================================= --> 136 137<!-- Start : Google Analytics Code --> 138<!--
vendor: 64 bytes, lines 138-139
138 139<script async src="https://www.googletagmanager.com/gtag/js?id=
139UA-64069893-4
vendor: 12 bytes, line 139
139"></script>
140<script> 141
vendor: 134 bytes, lines 141-145
141window.dataLayer = window.dataLayer || []; 142 function gtag(){dataLayer.push(arguments);} 143 gtag('js', new Date()); 144 145 gtag('config', '
145UA-64069893-4
vendor: 4 bytes, line 145
145');
146</script>
146 147--> 148<!-- End : Google Analytics Code --> 149
150<script type="text/javascript" src="resources/hidebib.js"></script>
150 151<link href='https://fonts.googleapis.com/css?family=Titillium+Web:400,600,400italic,600italic,300,300italic' rel='stylesheet' type='text/css'> 152<head> 153 <meta charset="utf-8" /> 154 <meta http-equiv="X-UA-Compatible" content="IE=edge" /> 155 <!-- 156 <link rel="icon" type="image/png" href="resources/seal_icon.png"> 157 --> 158 <title>MultiON: Benchmarking Semantic Map Memory using Multi-Object Navigation</title> 159 <meta name="HandheldFriendly" content="True" /> 160 <meta name="viewport" content="width=device-width, initial-scale=1.0" /> 161</head> 162 163<body> 164 <br> 165 <!--<center><span style="font-size:44px;font-weight:bold;">Learning to Control Self-Assembling Morphologies:<br/>A Study of Generalization via Modularity</span></center><br/>--> 166 <!-- <center><span style="font-size:44px;font-weight:bold;">MultiON: Benchmarking Semantic Map Memory using Multi-Object Navigation</span></center><br/> --> 167 <center><span style="font-size:44px;">MultiON: Benchmarking Semantic Map Memory using Multi-Object Navigation</span></center><br/> 168 <table align=center width=950px> 169 <tr> 170 <td align=center width=150px> 171 <!-- <center><span style="font-size:22px"><a href="" target="_blank">Saim Wani*<sup>1</sup></a></span></center></td> --> 172 <center><span style="font-size:24px"><a href="https://github.com/saimwani">Saim Wani</a>*</span></center></td> 173 <td align=center width=210px> 174 <center><span style="font-size:24px"><a href="https://shivanshpatel35.github.io" target="_blank">Shivansh Patel</a>*</span></center></td> 175 <td align=center width=150px> 176 <center><span style="font-size:24px"><a href="https://unnat.github.io" target="_blank">Unnat Jain</a>*</span></center></td> 177 <td align=center width=180px> 178 <center><span style="font-size:24px"><a href="https://angelxuanchang.github.io" target="_blank">Angel X. Chang</a></span></center></td> 179 <td align=center width=180px> 180 <center><span style="font-size:24px"><a href="https://msavva.github.io" target="_blank">Manolis Savva</a></span></center></td> 181 </tr> 182 </table> 183 <table align=center width=600px style="padding-top:0px;padding-bottom:0px"> 184 <tr> 185 <td align=center width=600px><center><span style="font-size:23px">IIT Kanpur, UIUC, SFU</span></span></center></td> 186 </tr> 187 </table> 188 <table align=center width=600px style="padding-top:0px;padding-bottom:20px"> 189 <tr> 190 <td align=center width=600px><center><span style="font-size:18px">* equal contribution by SW, SP and UJ</span></center></td> 191 </tr> 192 </table> 193 <table align=center width=700px> 194 <tr> 195 <td align=center width=700px><center><span style="font-size:22px">Published at <a href="https://nips.cc/Conferences/2020">NeurIPS 2020</a></span></center></td> 196 </tr> 197 </table><br/> 198 <!-- <table align=center width=700px> 199 <tr> 200 <td align=center width=700px><center><span style="font-size:22px"><a href="">Arxiv Link</a></span></center></td> 201 </tr> 202 </table><br/> --> 203 <table align=center width=700px> 204 <tr> 205 <td align=center width=100px><center><span style="font-size:21px"><a href="https://arxiv.org/abs/2012.03912">[Paper]</a></span></center></td> 206 <td align=center width=100px><center><span style="font-size:21px"><a href="https://youtu.be/yqTlHNIcgnY">[Video]</a></span></center></td> 207 <td align=center width=100px><center><span style="font-size:21px"><a href="resources/Poster.pdf">[Poster]</a></span></center></td> 208 <td align=center width=130px><center><span style="font-size:21px"><a href="https://github.com/saimwani/multiON">[Code]</a></span></center></td> 209 <!-- <td align=center width=100px><center><span style="font-size:21px">[Slides coming soon]</span></center></td> 210 <td align=center width=130px><center><span style="font-size:21px">[Code coming soon]</span></center></td> --> 211 </tr> 212 </table><br/> 213 214 <!-- <center><h1>Project Video</h1></center> --> 215 <table align=center width=300px> 216 <tr><td align=center width=300px> 217 <iframe width="768" height="432" src="https://www.youtube.com/embed/yqTlHNIcgnY" frameborder="0" allowfullscreen></iframe> 218 </td></tr> 219 </table> 220 <br> 221 222 223 <div style="width:800px; margin:0 auto; text-align=center"> 224 Navigation tasks in photorealistic 3D environments are challenging because they require perception and effective planning under partial observability.
225 Recent work shows that map-like memory is useful for long-horizon navigation tasks. 226 However, a focused investigation of the impact of maps on navigation tasks of varying complexity has not yet been performed.<br> 227 We propose the multiON task, which requires navigation to an episode-specific sequence of objects in a realistic environment. 228 MultiON generalizes the ObjectGoal navigation task [<a href="https://arxiv.org/abs/1807.06757">1</a>, <a href="https://arxiv.org/abs/1705.08080">2</a>] and explicitly tests the ability of navigation agents to locate previously observed goal objects. 229 We perform a set of multiON experiments to examine how a variety of agent models perform across a spectrum of navigation task complexities. 230 Our experiments show that: i) navigation performance degrades dramatically with escalating task complexity; ii) a simple semantic map agent performs surprisingly well relative to more complex neural image feature map agents; and iii) even oracle map agents achieve relatively low performance, indicating the potential for future work in training embodied navigation agents using maps. 231 </div> 232 <br> 233 234 <table align=center width=300px> 235 <tr><td align=center width=300px> 236 <a href="resources/main_visualization1.gif"><img src = "resources/main_visualization1.gif" height="280px"></img></a> 237 </td></tr> 238 </table> 239 <br> 240 241 <hr> 242 243 <center><h1>MultiON</h1></center> 244 <div style="width:800px; margin:0 auto; text-align=center"> 245 In an episode of multiON, the agent must navigate to an ordered set of objects placed within the environment. 246 The number of objects m determines the overall complexity of the navigation episode. 247 We use m-ON to refer to an episode with m ordered goal objects. 248 The m objects are selected from a set of k available objects where k ≥ m. 249 </div><br/><hr> 250 <center><h1>Agent Architecture</h1></center> 251 <table align=center width=1000px> 252 <p style="margin-top:4px;"></p> 253 <tr><td width=2000> 254 <center><a href="resources/model_diagram_webpage.png"><img src = "resources/model_diagram_webpage.png" height="370px"></img></a><br></center> 255 </td></tr> 256 </table><hr> 257 258 <center><h1>Agents</h1></center> 259 <div style="width:860px; margin:0 auto; text-align=center"> 260 261 <span style="font-family:'Lucida Console', monospace">NoMap(RNN)</span>: Baseline agent that does not use any map information.<br> 262 <span style="font-family:'Lucida Console', monospace">OracleMap</span>: Uses the entire ground truth map which contains occupancy information and object goal information.<br> 263 <span style="font-family:'Lucida Console', monospace">OracleEgoMap</span>: Uses the ground truth map that is progressively revealed as the agent explores the environment.<br> 264 <span style="font-family:'Lucida Console', monospace">ObjRecogMap</span>: Does not use oracle information and progressively constructs the global map by predicting object categories of goals visible from the sensory inputs. Trained using auxiliary classification loss.<br> 265 <span style="font-family:'Lucida Console', monospace">ProjNeuralMap</span>: Neurally projects the image features into a global map using the projection module. <br> 266 </div><br/><hr> 267 268 <p style="margin-top:4px;"></p> 269 <center><h1>Results</h1></center> 270 <div style="width:860px; margin:0 auto; text-align=center"> 271 Here we show performance of different models across different tasks. We report metric PPL which is a version of SPL based on progress (fraction of objects goals successfully FOUND) 272 </div> 273 <table align=center width=1000px> 274 <p style="margin-top:40px;"></p> 275 <tr><td width=1200px> 276 <center><a href="resources/results.png"><img src = "resources/results.png" height="290px"></img></a><br></center> 277 </td></tr> 278 </table> 279 280 <br/><hr> 281 282 <!-- <center id="sourceCode"><h1>Code and Dataset</h1></center> 283 <div style="width:400px; margin:0 auto; text-align=center"> 284 Our code and dataset is publicly available on GitHub. 285 </div> 286 <table align=center width=900px> 287 <tr> 288 <td width=300px align=center>
289 <span style="font-size:28px"><a href=''>[GitHub]</a></span> 290 </td> 291 </tr> 292 </table> 293 <br><hr> --> 294 295 <center id="sourceCode"><h1>OracleMap Visualizations</h1></center> 296 <div style="width:860px; margin:0 auto; text-align=center"> 297 Here are some sample visualizations of OracleMap. White color represents non-navigable locations, gray color represents navigable locations, and colored squares represent correspondingly colored goal. First, second and third maps are for 2-ON, 3-ON, and 4-ON task respectively. 298 </div> 299 <table align=center width=1000px> 300 <p style="margin-top:40px;"></p> 301 <tr> 302 <td width=600px> 303 <center><a href="visualizations/10.jpg"><img src = "visualizations/10.jpg" height="290px"></img></a><br></center> 304 </td> 305 <td width=600px> 306 <center><a href="visualizations/1.jpg"><img src = "visualizations/1.jpg" height="290px"></img></a><br></center> 307 </td> 308 <td width=600px> 309 <center><a href="visualizations/13.jpg"><img src = "visualizations/13.jpg" height="290px"></img></a><br></center> 310 </td> 311 </tr> 312 </table> 313 <br><hr> 314 315 316 <center id="sourceCode"><h1>Code and Dataset</h1></center> 317 <!-- <div style="width:130px; margin:0 auto; text-align=center"> 318 Coming soon. 319 </div> --> 320 <table align=center width=900px> 321 <tr> 322 <td width=300px align=center> 323 <span style="font-size:28px"><a href="https://github.com/saimwani/multiON">[GitHub]</a></span> 324 <!-- <span style="font-size:28px">Coming soon</span> --> 325 </td> 326 </tr> 327 </table> 328 <br><hr> 329 330 <table align=center width=850px> 331 <center><h1>Paper and Bibtex</h1></center> 332 <tr> 333 <td width=250px align=left> 334 <!-- <p style="margin-top:4px;"></p> --> 335 <center> 336 <a href="resources/stacked_pdf.png"><img style="height:220px" src="resources/stacked_pdf.png"/></a> 337 </center> 338 <p style="margin-top:4px;"></p> 339 <center> 340 <!-- <span style="font-size:20pt"><a href="resources/MultiON.pdf">[Paper]</a> --> 341 <span style="font-size:20pt"><a href="https://arxiv.org/abs/2012.03912">[ArXiv]</a> 342 <!-- <span style="font-size:20pt"><a href="">[Slides]</a></span> --> 343 <span style="font-size:20pt"><a href="resources/Poster.pdf">[Poster]</a></span> 344 </center> 345 </td> 346 <td width=50px align=center> 347 </td> 348 <td width=550px align=left> 349 <!-- <p style="margin-top:4px;"></p> --> 350 <p style="text-align:left;"><b><span style="font-size:20pt">Citation</span></b><br/><span style="font-size:6px;"> <br/></span> <span style="font-size:15pt">Saim Wani*, Shivansh Patel*, Unnat Jain*, Angel X. Chang, Manolis Savva. <b>MultiON: Benchmarking Semantic Map Memory using Multi-Object Navigation</b> In <i>NeurIPS</i> 2020.</span></p> 351 <!-- <p style="margin-top:20px;"></p> --> 352 353 <span style="font-size:20pt"><a shape="rect" href="javascript:togglebib('MultiON_bib')" class="togglebib">[Bibtex]</a></span> 354 </td> 355 </tr> 356 <tr> 357 <td width="30%" align=left> 358 </td> 359 <td width="6%" align=center> 360 </td> 361 <td width="64%" align=left> 362 <div class="paper" id="MultiON_bib"> 363<pre xml:space="preserve"> 364@inproceedings{wani2020multion, 365 Author = {Saim Wani and Shivansh Patel and 366 Unnat Jain and Angel X. Chang and Manolis Savva}, 367 Title = {MultiON: Benchmarking Semantic 368 Map Memory using Multi-Object Navigation}, 369 Booktitle = {NeurIPS}, 370 Year = {2020} 371 } 372</pre> 373 374 375 <!-- <span style="font-size:20pt"><a shape="rect" href="javascript:togglebib('MultiON_bib')" class="togglebib">[Bibtex]</a></span> 376 </td> 377 </tr> 378 <tr> 379 <td width=250px align=left> 380 </td> 381 <td width=50px align=center> 382 </td> 383 <td width=550px align=left>
384 <div class="paper" id="MultiON_bib"> 385 <td width=550px align=left> 386 387 <pre xml:space="preserve"> 388 @inproceedings{wani2020multion, 389 title={{MultiON: Benchmarking Semantic Map Memory using Multi-Object Navigation}}, 390 author={Saim Wani and Shivansh Patel and Unnat Jain and Angel X. Chang and Manolis Savva}, 391 booktitle={NeurIPS}, 392 year={2020}, 393 } 394 }</pre> --> 395 396 397 </div> 398 </td> 399 </tr> 400 </table> 401 <br><hr> 402 403 <table align=center width=800px> 404 <tr><td width=800px><left> 405 <center><h1>Acknowledgements</h1></center> 406 We thank the anonymous reviewers for their helpful suggestions. Unnat Jain thanks Alexander Schwing and Svetlana Lazebnik for their support. Angel X. Chang is supported by the Canada CIFAR AI Chair program. Manolis Savva is supported by an NSERC Discovery Grant. This research was enabled in part by support provided by <a href='https://www.westgrid.ca/'>WestGrid</a> and <a href="https://www.computecanada.ca">Compute Canada</a><br> 407 </left></td></tr> 408 </table> 409 410 411<table style="font-size:14px" align="center"> 412 <tr> 413 <td> 414 Template credits: <a href="https://pathak22.github.io/noreward-rl/">Deepak</a> and <a href='https://richzhang.github.io/colorization/'>Richard</a>. 415 </td> 416 </tr> 417 </table>
418<script xml:space="preserve" language="JavaScript"> 419 420hideallbibs(); 421</script>
421 422</body> 423</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.