1<!DOCTYPE html> 2<html> 3 4<head> 5 <meta charset="utf-8"> 6 <meta name="description" 7 content="DeXtreme is a system to perform highly dextrous manipulation with robotic hands using scaled simulation data."> 8 <meta name="keywords" content="Dexterous, Robotic, Manipulation, Reinforcement Learning"> 9 <meta name="viewport" content="width=device-width, initial-scale=1"> 10 <title>DeXtreme: Transfer of Agile In-Hand Manipulation from Simulation to Reality</title> 11 12 <!-- Global site tag (gtag.js) - Google Analytics -->
vendor: 66 bytes, lines 12-13
12 13 <script async src="https://www.googletagmanager.com/gtag/js?id=
13G-PYVRSFMDRL
vendor: 14 bytes, lines 13-14
13"></script> 14
14<script> 15
vendor: 155 bytes, lines 15-23
15window.dataLayer = window.dataLayer || []; 16 17 function gtag() { 18 dataLayer.push(arguments); 19 } 20 21 gtag('js', new Date()); 22 23 gtag('config', '
23G-PYVRSFMDRL
vendor: 6 bytes, lines 23-24
23'); 24
24</script>
24 25 26 <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro" rel="stylesheet"> 27 28 <link rel="stylesheet" href="./static/css/bulma.min.css"> 29 <link rel="stylesheet" href="./static/css/bulma-carousel.min.css"> 30 <link rel="stylesheet" href="./static/css/bulma-slider.min.css"> 31 <link rel="stylesheet" href="./static/css/fontawesome.all.min.css"> 32 <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css"> 33 <link rel="stylesheet" href="./static/css/index.css"> 34 <!-- <link rel="icon" href="./static/images/favicon.svg"> --> 35 <link rel="icon" href="./static/images/favicon_cube.ico"> 36 37
37<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
37 38
38<script defer src="./static/js/fontawesome.all.min.js"></script>
38 39
39<script src="./static/js/bulma-carousel.min.js"></script>
39 40
40<script src="./static/js/bulma-slider.min.js"></script>
40 41
41<script src="./static/js/index.js"></script>
41 42</head> 43 44<body> 45 46 <nav class="navbar" role="navigation" aria-label="main navigation"> 47 <div class="navbar-brand"> 48 <a role="button" class="navbar-burger" aria-label="menu" aria-expanded="false"> 49 <span aria-hidden="true"></span> 50 <span aria-hidden="true"></span> 51 <span aria-hidden="true"></span> 52 </a> 53 </div> 54 <div class="navbar-menu"> 55 <div class="navbar-start" style="flex-grow: 1; justify-content: center;"> 56 <a class="navbar-item" href="https://developer.nvidia.com/isaac-gym"> 57 <span class="icon"> 58 <i class="fas fa-home"></i> 59 </span> 60 </a> 61 62 <div class="navbar-item has-dropdown is-hoverable"> 63 <a class="navbar-link"> 64 More Research 65 </a> 66 <div class="navbar-dropdown"> 67 <a class="navbar-item" href="https://sites.google.com/view/dexpbt"> 68 DexPBT 69 </a> 70 <a class="navbar-item" href="https://s2r2-ig.github.io/"> 71 Learning Dexterity for Trifinger 72 </a> 73 <a class="navbar-item" href="https://arxiv.org/abs/2108.10470"> 74 Isaac Gym 75 </a> 76 </div> 77 </div> 78 </div> 79 80 </div> 81 </nav> 82 83 84 <section class="hero"> 85 <div class="hero-body" style="padding-top: 1em"> 86 <div class="container is-max-desktop"> 87 <div class="columns is-centered"> 88 <div class="column has-text-centered"> 89 <h1 class="title is-1 publication-title">De<span style="color:#86B737">X</span>treme: Transfer of Agile 90 In-Hand Manipulation from Simulation to Reality</h1> 91 <div class="is-size-5 publication-authors"> 92 <span class="author-block"> 93 <a href="https://ankurhanda.github.io/">Ankur Handa</a>, 94 </span> 95 <span class="author-block"> 96 <a href="https://allshire.org">Arthur Allshire</a>, 97 </span> 98 <span class="author-block"> 99 <a href="https://twitter.com/viktor_m81?lang=en">Viktor Makoviychuk</a>, 100 </span> 101 <span class="author-block"> 102 <a href="https://alex-petrenko.github.io/">Aleksei Petrenko</a>, 103 </span> 104 105 <br /> 106 107 <span class="author-block"> 108 <a href="">Ritvik Singh</a>, 109 </span> 110 <span class="author-block"> 111 <a href="https://www.linkedin.com/in/jasonjzliu/">Jingzhou Liu</a>, 112 </span> 113 <span class="author-block"> 114 <a href="">Denys Makoviichuk</a>, 115 </span> 116 <span class="author-block"> 117 <a href="">Karl Van Wyk</a>, 118 </span> 119 <span class="author-block"> 120 <a href="">Alexander Zhurkevich</a>, 121 </span> 122 <br /> 123 <span class="author-block"> 124 <a href="">Balakumar Sundaralingam</a>, 125 </span> 126 <span class="author-block"> 127 <a href="">Yashraj Narang</a>, 128 </span> 129 <span class="author-block"> 130 <a href="">Jean-Francois Lafleche</a>, 131 </span> 132 <span class="author-block"> 133 <a href="">Dieter Fox</a>, 134 </span> 135 <span class="author-block"> 136 <a href="">Gavriel State</a> 137 </span> 138 </div> 139 140 <div class="is-size-5 publication-authors"> 141 <span class="author-block"><a href="https://research.nvidia.com/research-area/robotics"><b><img 142 src="static/images/nvidia.svg" style="height:40px"></b></a></span> 143 </div> 144 145 <div class="column has-text-centered"> 146 <div class="publication-links"> 147 <!-- PDF Link. --> 148 <span class="link-block"> 149 <a href="https://arxiv.org/pdf/2210.13702.pdf" class="external-link button is-normal is-rounded is-dark"> 150 <span class="icon"> 151 <i class="fas fa-file-pdf"></i> 152 </span>
153 <span>Paper</span> 154 </a> 155 </span> 156 <!-- <span class="link-block"> 157 <a href="" 158 class="external-link button is-normal is-rounded is-dark"> 159 <span class="icon"> 160 <i class="ai ai-arxiv"></i> 161 </span> 162 <span>arXiv</span> 163 </a> 164 </span> --> 165 <!-- Video Link. --> 166 <span class="link-block"> 167 <!-- <a href="https://www.youtube.com/watch?v=DDsRWCFqmI0" --> 168 <a href="#overview_video" class="external-link button is-normal is-rounded is-dark"> 169 <span class="icon"> 170 <i class="fab fa-youtube"></i> 171 </span> 172 <span>Video</span> 173 </a> 174 </span> 175 <!-- Code Link. --> 176 <span class="link-block"> 177 <a href="https://github.com/NVIDIA-Omniverse/IsaacGymEnvs/tree/main/isaacgymenvs/tasks/dextreme" class="external-link button is-normal is-rounded is-dark"> 178 <span class="icon"> 179 <i class="fab fa-github"></i> 180 </span> 181 <span>Code</span> 182 </a> 183 </span> 184 </div> 185 186 </div> 187 </div> 188 </div> 189 </div> 190 </div> 191 </section> 192 193 <section class="hero teaser"> 194 <div class="container is-max-desktop"> 195 <div class="hero-body"> 196 <video id="teaser" autoplay muted loop playsinline width="100%"> 197 <source src="static/images/teaser.mp4" type="video/mp4"> 198 </video> 199 <!--<h2 class="subtitle has-text-centered">--> 200 <h2 class="subtitle" align="justify"> 201 <span class="dnerf">DeXtreme</span> achives human-like in-hand manipulation while training entirely in 202 simulation. In the inset video, we show the live visualisation (no physics involved) of the states of hand and cube in Omniverse i.e. 203 the joint values of the hand and cube pose are teleported in simulation. The Omniverse video is recorded at 10Hz while the real-world video is captured at 30Hz so there might be some repetitive frames and lag in the inset video. The live visualisation is only possible because of the explicit representation used for tracking the cube and we found this to be very helpful for diagnosing the performance of the pose estimator as well as the policy. 204 </h2> 205 </div> 206 </div> 207 </section> 208 209 210 <section class="section"> 211 <div class="container is-max-desktop"> 212 <!-- Abstract. --> 213 <div class="columns is-centered has-text-centered"> 214 <div class="column is-four-fifths"> 215 <h2 class="title is-3">Abstract</h2> 216 <div class="content has-text-justified"> 217 218 <p>We present our techniques to train a) a policy that can perform robust dexterous manipulation on an 219 anthropomorphic robot hand and b) a robust pose estimator suitable 220 for providing reliable real-time information on the state of the object being manipulated.</p> 221 222 <p><span class="dnerf">DeXtreme</span> policies are trained to adapt to a wide range of conditions in 223 simulation. 224 Consequently, our vision-based policies significantly outperform the best vision policies 225 in the literature on the same reorientation task and are competitive with policies that 226 are given privileged state information via motion capture systems.</p> 227 228 <p>Recent work has demonstrated the ability of deep reinforcement learning (RL) algorithms 229 to learn complex robotic behaviours in simulation, including in the domain of multi-fingered manipulation. 230 However, such models can be challenging to transfer to the real world due to the gap between simulation 231 and reality.</p> 232 233 <p>Our work reaffirms the possibilities of sim-to-real transfer for dexterous manipulation in diverse kinds 234 of hardware and simulator setups, and in our case, with the Allegro Hand and Isaac Gym GPU-based 235 simulation. 236 Furthermore, it opens up possibilities for researchers to achieve such results with commonly-available, 237 affordable robot hands and cameras.</p> 238 </div> 239 </div> 240 </div> 241 <!--/ Abstract. --> 242 243 <!-- Paper video. --> 244 <div class="columns is-centered has-text-centered"> 245 <div class="column is-four-fifths"> 246 <h2 class="title is-3" id="overview_video">Video</h2>
247 <div class="publication-video"> 248 <!-- <iframe src="https://www.youtube.com/embed/MrKrnHhk8IA?rel=0&showinfo=0" 249 frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe> --> 250 <video id="teaser" controls width="100%"> 251 <!-- <source src="https://homes.cs.washington.edu/~kpar/nerfies/videos/teaser.mp4" 252 type="video/mp4"> --> 253 <source src="static/images/ICRA23_suppl.mp4" type="video/mp4"> 254 </video> 255 </div> 256 </div> 257 </div> 258 259 <div class="columns is-centered has-text-centered"> 260 <h2>All the video below are recorded at <b>1x real time</b>. 261 </h2> 262 </div> 263 264 <!--/ Paper video. --> 265 <div class="columns is-centered has-text-centered"> 266 <div class="column is-four-fifths"> 267 <h2 class="title is-3" id="overview_video">A video of a longer rollout: 86 CS</h2> 268 <div class="publication-video"> 269 <video id="teaser" controls width="100%"> 270 <source src="static/images/85CS_NoHold_2.mp4" type="video/mp4"> 271 </video> 272 </div> 273 <h2 class="subtitle has-text-centered"> 274 An example of a longer rollout where the hand achieves 86 sucesses in a row. 275 </h2> 276 </div> 277 </div> 278 279 <!--/ Paper video. --> 280 <div class="columns is-centered has-text-centered"> 281 <div class="column is-four-fifths"> 282 <h2 class="title is-3" id="overview_video">Goal frame hold rollout: 43CS</h2> 283 <div class="publication-video"> 284 <video id="goal_frame_43CS" controls width="100%"> 285 <source src="static/images/44CS_noOV_goal_frame_hold_with_audio_handbrake.mp4" type="video/mp4"> 286 </video> 287 </div> 288 <h2 class="subtitle has-text-centered"> 289 An example of a rollout achieving 43 CS where the cube is held at the goal pose for 10 frames in a row. 290 </h2> 291 </div> 292 </div> 293 294 <!--/ Paper video. --> 295 <div class="columns is-centered has-text-centered"> 296 <div class="column is-four-fifths"> 297 <h2 class="title is-3" id="overview_video">Goal frame hold rollout: 29CS</h2> 298 <div class="publication-video"> 299 <video id="teaser" controls width="100%"> 300 <source src="static/images/29CS_goal_frame_hold_with_audio_handbrake.mp4" type="video/mp4"> 301 </video> 302 </div> 303 <h2 class="subtitle has-text-centered"> 304 An example of a rollout achieving 29 CS where the cube is held at the goal pose for 10 frames in a row. 305 </h2> 306 </div> 307 </div> 308 309 <!--/ Paper video. --> 310 <div class="columns is-centered has-text-centered"> 311 <div class="column is-four-fifths"> 312 <h2 class="title is-3" id="overview_video">Goal frame hold rollout: 13CS</h2> 313 <div class="publication-video"> 314 <video id="teaser" controls width="100%"> 315 <source src="static/images/13CS_goal_frame_hold_with_audio_handbrake.mp4" type="video/mp4"> 316 </video> 317 </div> 318 <h2 class="subtitle has-text-centered"> 319 An example of a rollout achieving 13 CS where the cube is held at the goal pose for 10 frames in a row. 320 </h2> 321 </div> 322 </div> 323 324 325 <!-- Long rollout video. --> 326 <!-- <div class="columns is-centered has-text-centered"> 327 <div class="column is-four-fifths"> 328 <h2 class="title is-3" id="overview_video">Longer Rollout Video</h2> 329 <div class="publication-video"> 330 <video id="teaser" controls width="100%"> 331 <source src="static/images/long_video.mp4" 332 type="video/mp4"> 333 </video> 334 </div> 335 </div> 336 </div> --> 337 <!--/ Long rollout video. --> 338 339 </div> 340 </section> 341 342 343 <section class="section" id="BibTeX"> 344 <div class="container is-max-desktop content"> 345 <h2 class="title">BibTeX</h2> 346 <pre><code>@article{nvidia2022dextreme, 347 author = {Handa, Ankur and Allshire, Arthur and Makoviychuk, Viktor and Petrenko, Aleksei and Singh, Ritvik and Liu, Jingzhou and Makoviichuk, Denys and Van Wyk, Karl and Zhurkevich, Alexander and Sundaralingam, Balakumar and Narang, Yashraj and Lafleche, Jean-Francois and Fox, Dieter and State, Gavriel}, 348 title = {DeXtreme: Transfer of Agile In-Hand Manipulation from Simulation to Reality}, 349 journal = {arXiv}, 350 year = {2022}, 351}</code></pre> 352 </div> 353 </section> 354 355 356 <footer class="footer"> 357 <div class="container"> 358 <div class="content has-text-centered"> 359 <a class="icon-link" href="https://arxiv.org"> 360 <i class="fas fa-file-pdf"></i> 361 </a> 362 <a class="icon-link" href="https://github.com/arthurallshire" class="external-link" disabled> 363 <i class="fab fa-github"></i> 364 </a> 365 </div> 366 <div class="columns is-centered"> 367 <div class="column is-8"> 368 <div class="content"> 369 <p> 370 This website is licensed under a <a rel="license" 371 href="http://creativecommons.org/licenses/by-sa/4.0/">Creative 372 Commons Attribution-ShareAlike 4.0 International License</a>. 373 </p> 374 <p> 375 Template from <a href="https://github.com/nerfies/nerfies.github.io">Nerfies</a>. 376 </p> 377 </div> 378 </div> 379 </div> 380 </div> 381 </footer> 382 383</body> 384 385</html>
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.