https://aiot-mlsys-lab.github.io/static/js/346.bc724a30.chunk.js

1"use strict";(self.webpackChunklabwebsite=self.webpackChunklabwebsite||[]).push([[346],{876:function(e,i,s){var n=s(8316),t=s(184),r={height:"300px",color:"#fff",lineHeight:"160px",textAlign:"center",background:"#364d79"};i.Z=function(){return(0,t.jsxs)(n.Z,{autoplay:!0,children:[(0,t.jsx)("div",{children:(0,t.jsx)("h3",{style:r,children:"1"})}),(0,t.jsx)("div",{children:(0,t.jsx)("h3",{style:r,children:"2"})}),(0,t.jsx)("div",{children:(0,t.jsx)("h3",{style:r,children:"3"})}),(0,t.jsx)("div",{children:(0,t.jsx)("h3",{style:r,children:"4"})})]})}},8346:function(e,i,s){s.r(i),s.d(i,{default:function(){return t}});s(2791),s(876),s(4996);var n=s(184);var t=function(){return(0,n.jsxs)("div",{className:"project_container",children:[(0,n.jsx)("div",{className:"project_title",children:"D2O: Dynamic Discriminative Operations for Efficient Generative Inference of Large Language Models"}),(0,n.jsx)("div",{className:"project_conference",children:"arXiv:2406.13035"}),(0,n.jsx)("div",{className:"project_members",children:"Zhongwei Wan, Xinjian Wu, Yu Zhang, Yi Xin, Chaofan Tao, Zhihong Zhu, Xin Wang, Siqi Luo, Jing Xiong, Mi Zhang."}),(0,n.jsx)("div",{className:"project_buttons",children:(0,n.jsxs)("a",{href:"https://arxiv.org/abs/2406.13035",className:"project_button",children:[(0,n.jsx)("svg",{t:"1687728426228",class:"icon",viewBox:"0 0 1024 1024",version:"1.1",xmlns:"http://www.w3.org/2000/svg","p-id":"2399",width:"16",height:"16",children:(0,n.jsx)("path",{d:"M213.34016 0l597.34016 0q53.00224 0 90.50112 37.49888t37.49888 90.50112l0 768q0 53.00224-37.49888 90.50112t-90.50112 37.49888l-597.34016 0q-53.00224 0-90.50112-37.49888t-37.49888-90.50112l0-768q0-53.00224 37.49888-90.50112t90.50112-37.49888zM341.34016 725.34016l341.34016 0q17.67424 0 30.16704 12.4928t12.4928 30.16704-12.4928 30.16704-30.16704 12.4928l-341.34016 0q-17.67424 0-30.16704-12.4928t-12.4928-30.16704 12.4928-30.16704 30.16704-12.4928zM341.34016 554.65984l341.34016 0q17.67424 0 30.16704 12.4928t12.4928 30.16704-12.4928 30.16704-30.16704 12.4928l-341.34016 0q-17.67424 0-30.16704-12.4928t-12.4928-30.16704 12.4928-30.16704 30.16704-12.4928zM810.65984 85.34016l-597.34016 0q-17.67424 0-30.16704 12.4928t-12.4928 30.16704l0 768q0 17.67424 12.4928 30.16704t30.16704 12.4928l597.34016 0q17.67424 0 30.16704-12.4928t12.4928-30.16704l0-768q0-17.67424-12.4928-30.16704t-30.16704-12.4928zM341.34016 384l341.34016 0q17.67424 0 30.16704 12.4928t12.4928 30.16704-12.4928 30.16704-30.16704 12.4928l-341.34016 0q-17.67424 0-30.16704-12.4928t-12.4928-30.16704 12.4928-30.16704 30.16704-12.4928zM341.34016 213.34016l170.65984 0q17.67424 0 30.16704 12.4928t12.4928 30.16704-12.4928 30.16704-30.16704 12.4928l-170.65984 0q-17.67424 0-30.16704-12.4928t-12.4928-30.16704 12.4928-30.16704 30.16704-12.4928z",fill:"#ffffff","p-id":"2400"})}),"\xa0Paper"]})}),(0,n.jsx)("div",{className:"project_collection",children:(0,n.jsx)("div",{children:(0,n.jsx)("img",{src:"./images/projects/D2O_in.png",alt:"",className:"project_collection_img"})})}),(0,n.jsxs)("div",{className:"project_abstract",children:[(0,n.jsx)("div",{className:"project_abstract_title",children:"Abstract"}),(0,n.jsx)("div",{className:"project_abstract_content",children:"Efficient inference in Large Language Models (LLMs) is impeded by the growing memory demands of key-value (KV) caching, especially for longer sequences. Traditional KV cache eviction strategies, which prioritize less critical KV-pairs based on attention scores, often degrade generation quality, leading to issues such as context loss or hallucinations. To address this, we introduce Dynamic Discriminative Operations (D2O), a novel method that utilizes two-level discriminative strategies to optimize KV cache size without fine-tuning, while preserving essential context. Initially, by observing varying densities of attention weights between shallow and deep layers, we use this insight to determine which layers should avoid excessive eviction to minimize information loss. Subsequently, for the eviction strategy in each layer, D2O innovatively incorporates a compensation mechanism that maintains a similarity threshold to re-discriminate the importance of previously discarded tokens, determining whether they should be recalled and merged with similar tokens. Our approach not only achieves significant memory savings and enhances inference throughput by more than 3 times but also maintains high-quality long-text generation. Extensive experiments across various benchmarks and LLM architectures have demonstrated that D2O significantly enhances performance with a constrained KV cache budget."})]})]})}},4996:function(e,i,s){s.d(i,{Z:function(){return l}});var n=s(9439),t=s(2791),r=s(6913),a=s(7309),c=s(184),l=function(e){var i=(0,t.useState)(!1),s=(0,n.Z)(i,2),l=s[0],o=s[1],h=function(){o(!1)};return(0,c.jsxs)(c.Fragment,{children:[(0,c.jsxs)("button",{href:"",className:"project_button",onClick:function(){o(!0)},children:[(0,c.jsx)("svg",{t:"1687730915571",class:"icon",viewBox:"0 0 1024 1024",version:"1.1",xmlns:"http://www.w3.org/2000/svg","p-id":"2419",width:"16",height:"16",children:(0,c.jsx)("path",{d:"M313.6 170.666667h469.333333c27.733333 0 42.666667 14.933333 42.666667 42.666666v512c0 8.533333-4.266667 17.066667-10.666667 23.466667l-36.266666 36.266667c-2.133333 2.133333-6.4 4.266667-6.4 2.133333-2.133333 0-2.133333-4.266667-2.133334-6.4v-539.733333c0-4.266667-2.133333-6.4-4.266666-10.666667s-6.4-4.266667-10.666667-4.266667h-398.933333c-8.533333 0-17.066667 4.266667-23.466667 10.666667l-36.266667 36.266667c-2.133333 2.133333-4.266667 6.4-2.133333 6.4 0 2.133333 4.266667 2.133333 6.4 2.133333h398.933333c4.266667 0 6.4 2.133333 10.666667 4.266667s4.266667 6.4 4.266667 10.666666v539.733334c0 4.266667-2.133333 6.4-4.266667 10.666666s-6.4 4.266667-10.666667 4.266667H213.333333c-4.266667 0-6.4-2.133333-10.666666-4.266667s-4.266667-6.4-4.266667-10.666666v-554.666667c0-8.533333 4.266667-17.066667 10.666667-23.466667l78.933333-78.933333c6.4-4.266667 14.933333-8.533333 25.6-8.533333z","p-id":"2420",fill:"#ffffff"})}),"\xa0BibTex"]}),(0,c.jsx)(r.Z,{title:"BibTex",open:l,onCancel:h,width:1e3,footer

1:[(0,c.jsx)(a.ZP,{onClick:h,children:"Return"},"back")],children:(0,c.jsxs)("div",{className:"popModalBG",children:[(0,c.jsx)("p",{children:e.inproceeding}),(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.title]}),(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.author]}),e.booktitle&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.booktitle]}),e.journal&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.journal]}),e.volume&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.volume]}),e.number&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.number]}),e.pages&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.pages]}),(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.year]}),e.eprint&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.eprint]}),e.archivePrefix&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.archivePrefix]}),e.primaryClass&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.primaryClass]}),e.numpages&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.numpages]}),e.address&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.address]}),e.location&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.location]}),e.publisher&&(0,c.jsxs)("p",{children:["\xa0\xa0\xa0\xa0",e.publisher]}),(0,c.jsx)("p",{children:"}"})]})})]})}}}]);
2//# sourceMappingURL=346.bc724a30.chunk.js.map