1"use strict";(globalThis.webpackChunkmy_website=globalThis.webpackChunkmy_website||[]).push([[12269],{96973(e,n,t){t.r(n),t.d(n,{assets:()=>h,contentTitle:()=>c,default:()=>g,frontMatter:()=>o,metadata:()=>i,toc:()=>d});var i=t(31386),a=t(74848),r=t(28453),s=t(49107),l=t(15768);const o={title:"Long-Context Retrieval Models with Monarch Mixer",slug:"paper16",authors:["zain"],date:new Date("2024-01-15T00:00:00.000Z"),tags:[],image:"./img/hero.jpeg",description:"32k context length retreival models with sub-quadratic attention mechanism."},c=void 0,h={image:t(53854).A,authorsImageUrls:[void 0]},d=[{value:"In Short\u23e9:",id:"in-short",level:3},{value:"\ud83c\udfd7\ufe0fArchitechtural Details:",id:"\ufe0farchitechtural-details",level:3},{value:"\ud83e\ude83Training Details:",id:"training-details",level:3},...l.RM];function p(e){const n={a:"a",blockquote:"blockquote",h3:"h3",img:"img",li:"li",ol:"ol",p:"p",...(0,r.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"A preview of the paper",src:t(82385).A+"",width:"1200",height:"874"})}),"\n","\n",(0,a.jsx)(n.p,{children:"A breakdown of the Long Context Retrieval Embedding Models from Stanford!\ud83d\udca5"}),"\n",(0,a.jsx)(n.h3,{id:"in-short",children:"In Short\u23e9:"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"They release 3 long context(2k/8k/32k) BERT-like encoder embedding models on HuggingFace"}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"The models are only 80M params and outperform MUCH larger models (4-85x larger)"}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"Accessible via @togethercompute endpoints and integrated into @llama_index and @LangChainAI"}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"They also release LoCo a long context retrieval benchmark."}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"\ufe0farchitechtural-details",children:"\ud83c\udfd7\ufe0fArchitechtural Details:"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"They replace the Attention and MLP blocks in the transformer architecture with diagonal block matrix (Monarch Matrices -M2) operations which are hardware optimized and subquadratic in the sequence length - O(N^(1.5))"}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"This enables scaling sequence length and model parameters better."}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.h3,{id:"training-details",children:"\ud83e\ude83Training Details:"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"These M2 models are trained for long context retrieval on a mixture of long and short context tasks data - surprisingly only training on long context doesn't work."}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:["\n",(0,a.jsx)(n.p,{children:"Use a cosine similarity loss instead of the trusty supervised contrastive training loss."}),"\n",(0,a.jsxs)(n.blockquote,{children:["\n",(0,a.jsx)(n.p,{children:"This loss function. can be computed independently per datapoint in a batch instead of needing to sum over all negative examples in a batch."}),"\n"]}),"\n",(0,a.jsxs)(n.blockquote,{children:["\n",(0,a.jsx)(n.p,{children:"Thus training can be scaled for large batch sizes of long context inputs without OOM'ing"}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.a,{href:"https://hazyresearch.stanford.edu/blog/2024-01-11-m2-bert-retrieval",children:"\ud83d\udcdcBlog"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.a,{href:"https://github.com/HazyResearch/m2",children:"\ud83e\uddd1\u200d\ud83d\udcbbCode"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.a,{href:"https://huggingface.co/togethercomputer/m2-bert-80M-32k-retrieval",children:"\ud83d\udd37Models"})}),"\n",(0,a.jsx)("p",{children:(0,a.jsx)(s.fd,{link:"https://arxiv.org/abs/2310.12109",children:"\ud83d\udd17 arXiv Link"})}),"\n",(0,a.jsx)("p",{children:(0,a.jsx)(s.fd,{link:"https://arxiv.org/pdf/2310.12109",children:"\ud83d\udcdc Download paper"})}),"\n","\n",(0,a.jsx)(l.Ay,{})]})}function g(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(p,{...e})}):p(e)}},49107(e,n,t){t.d(n,{Ef:()=>u,fd:()=>g,z9:()=>p});var i=t(28774);t(96540);const a="buttons_NqrD",r="left_ej0y",s="right_kFGn",l="btn_VbJ1",o="btnMain_ywTD",c="btnAccent_uqV6",h="btnOutlined_BYrT";var d=t(74848);function p({children:e,link:n,color:t="main",newTab:a=!0,className:r,style:s}){let p=l;switch(t){case"accent":p+=" "+c;break;case"outlined":p+=" "+h;break;default:p+=" "+o}return r&&(p+=" "+r),(0,d.jsx)(i.A,{className:p,style:s,to:n,target:a?"_blank":"_self",children:e})}function g({children:e,link:n,color:t="main",newTab:i=!0}){let a=l;if("accent"===t)a+=" "+c;else a+=" "+o;return(0,d.jsx)("a",{className:a,href:n,download:!0,children:e})}function u({children:e,position:n="center"}){let t=a;switch(n){case"left":t+=" "+r;break;case"right":t+=" "+s}return(0,d.jsx)("div",{className:t,children:e})}},53854(e,n,t){t.d(n,{A:()=>i});const i=t.p+"assets/images/hero-0b709c8abd18ca664ec3b01009b53e50.jpeg"},82385(e,n,t){t.d(n,{A:()=>i});const i=t.p+"assets/images/hero-0b709c8abd18ca664ec3b01009b53e50.jpeg"},31386(e){e.exports=JSON.parse('{"permalink":"/papers/paper16","source":"@site/papers/2024-01-15-paper16/index.mdx","title":"Long-Context Retrieval Models with Monarch Mixer","description":"32k context length retreival models with sub-quadratic attention mechanism.","date":"2024-01-15T00:00:00.000Z","tags":[],"readingTime":1.35,"hasTruncateMarker":true,"authors":[{"name":"Zain Hasan","title":"Developer Advocate","url":"https://github.com/zainhas","imageURL":"/img/people/icon/zain.jpg","key":"zain","page":null}],"frontMatter":{"title":"Long-Context Retrieval Models with Monarch Mixer","slug":"paper16","authors":["zain"],"date":"2024-01-15T00:00:00.000Z","tags":[],"image":"./img/hero.jpeg","description":"32k context length retreival models with sub-quadratic attention mechanism."},"unlisted":false,"prevItem":{"title":"Fine-grained Hallucination Detection and Editing for Language Models","permalink":"/papers/paper18"},"nextItem":{"title":"How Johnny Can Persuade LLMs to Jailbreak Them: Rethinking Persuasion to Challenge AI Safety by Humanizing LLMs","permalink":"/papers/paper15"}}')}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.