1"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([["7103"],{58490(e,n,s){s.r(n),s.d(n,{metadata:()=>i,default:()=>h,frontMatter:()=>o,contentTitle:()=>l,toc:()=>a,assets:()=>c});var i=JSON.parse('{"id":"examples/memory-tiers/prefix-caching","title":"Prefix caching","description":"What this demonstrates: reusing pre-computed KV cache across","source":"@site/docs/examples/memory-tiers/prefix-caching.mdx","sourceDirName":"examples/memory-tiers","slug":"/examples/memory-tiers/prefix-caching","permalink":"/docs/examples/memory-tiers/prefix-caching","draft":false,"unlisted":false,"editUrl":"https://github.com/casys-kaist/LLMServingSim/tree/main/website/docs/examples/memory-tiers/prefix-caching.mdx","tags":[],"version":"current","sidebarPosition":1,"frontMatter":{"title":"Prefix caching","sidebar_position":1},"sidebar":"userSidebar","previous":{"title":"PIM attention offload","permalink":"/docs/examples/disaggregated/pim-attention-offload"},"next":{"title":"CXL extended memory","permalink":"/docs/examples/memory-tiers/cxl-memory"}}'),r=s(74934),t=s(31137);let o={title:"Prefix caching",sidebar_position:1},l="Prefix caching",c={},a=[{value:"Prerequisites",id:"prerequisites",level:2},{value:"Cluster config",id:"cluster-config",level:2},{value:"Run",id:"run",level:2},{value:"Per-instance prefix caching (default)",id:"per-instance-prefix-caching-default",level:3},{value:"Shared CPU prefix pool",id:"shared-cpu-prefix-pool",level:3},{value:"Expected output",id:"expected-output",level:2},{value:"What's interesting",id:"whats-interesting",level:2},{value:"Related examples",id:"related-examples",level:2}];function d(e){let n={a:"a",blockquote:"blockquote",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,t.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(n.header,{children:(0,r.jsx)(n.h1,{id:"prefix-caching",children:"Prefix caching"})}),"\n",(0,r.jsxs)(n.blockquote,{children:["\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.strong,{children:"What this demonstrates:"})," reusing pre-computed KV cache across\nrequests with shared prompt prefixes, including a second-tier CPU\npool shared across instances."]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:"For workloads where many requests share a system prompt, RAG context,\nor a long instruction (e.g., agent traces), recomputing prefill for\neach request is wasted work. Prefix caching keeps prefix KV blocks\naround (in NPU memory by default; optionally also in CPU or CXL) and\nreuses them on hits."}),"\n",(0,r.jsx)(n.p,{children:"LLMServingSim ships block-hash prefix caching (ported from vLLM\nv0.19.0's block pool) with three flavors:"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Per-instance NPU pool"})," (default, always on)."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Cross-instance shared CPU pool"}),": second-tier prefix cache, which\nmodels attaching LMCache or vLLM's ",(0,r.jsx)(n.code,{children:"OffloadingConnector"}),"."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"CXL-backed pool"}),": same as above but in CXL memory."]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Simulator container set up"}),"\n",(0,r.jsxs)(n.li,{children:["A workload with shared prefixes (the bundled ",(0,r.jsx)(n.code,{children:"example_trace.jsonl"}),"\nhas some, real ShareGPT or agentic traces have a lot)"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"cluster-config",children:"Cluster config"}),"\n",(0,r.jsxs)(n.p,{children:["The simplest setup uses\n",(0,r.jsx)(n.code,{children:"configs/cluster/single_node_multi_instance.json"})," (two instances on\none node, no special memory config). The shared CPU pool is enabled\nat runtime via CLI flags, not the config:"]}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-json",metastring:'title="configs/cluster/single_node_multi_instance.json"',children:'{\n "num_nodes": 1,\n "link_bw": 16,\n "link_latency": 20000,\n "nodes": [\n {\n "num_instances": 2,\n "cpu_mem": {"mem_size": 512, "mem_bw": 256, "mem_latency": 0},\n "instances": [\n {\n "model_name": "meta-llama/Llama-3.1-8B",\n "hardware": "RTXPRO6000",\n "npu_mem": {"mem_size": 96, "mem_bw": 1597, "mem_latency": 0},\n "pd_type": null,\n "tp_size": 1\n },\n { "...": "second instance, identical" }\n ]\n }\n ]\n}\n'})}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"cpu_mem.mem_size"})," (512 GB here) caps how big the CPU prefix pool\ncan grow."]}),"\n",(0,r.jsx)(n.h2,{id:"run",children:"Run"}),"\n",(0,r.jsx)(n.h3,{id:"per-instance-prefix-caching-default",children:"Per-instance prefix caching (default)"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"python -m serving \\\n --cluster-config 'configs/cluster/single_node_multi_instance.json' \\\n --dtype bfloat16 --block-size 16 \\\n --dataset 'workloads/example_trace.jsonl' \\\n --output 'outputs/prefix_default_run.csv'\n"})}),"\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"--enable-prefix-caching"})," is on by default. Prefix blocks are kept\nin each instance's own NPU memory; if a request lands on instance A\nthat prefixes-into a cached block on instance B, no reuse happens."]}),"\n",(0,r.jsx)(n.h3,{id:"shared-cpu-prefix-pool",children:"Shared CPU prefix pool"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-bash",children:"python -m serving \\\n --cluster-config 'configs/cluster/single_node_multi_instance.json' \\\n --dtype bfloat16 --block-size 16 \\\n --enable-prefix-caching --enable-prefix-sharing --prefix-storage CPU \\\n --dataset 'workloads/example_trace.jsonl' \\\n --output 'outputs/prefix_cpu_pool_run.csv'\n"})}),"\n",(0,r.jsx)(n.p,{children:"The two extra flags:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"--enable-prefix-sharing"}),": turn on the second-tier pool."]}
1),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.code,{children:"--prefix-storage CPU"}),": pool lives in ",(0,r.jsx)(n.code,{children:"cpu_mem"}),". Other options:\n",(0,r.jsx)(n.code,{children:"CXL"})," (requires a ",(0,r.jsx)(n.code,{children:"cxl_mem"})," config block), ",(0,r.jsx)(n.code,{children:"None"})," (NPU-only)."]}),"\n"]}),"\n",(0,r.jsxs)(n.p,{children:["When an NPU prefix is evicted, it spills to the CPU pool instead of\ndisappearing. Requests on ",(0,r.jsx)(n.strong,{children:"any"})," instance can now hit the CPU pool\non lookup."]}),"\n",(0,r.jsx)(n.h2,{id:"expected-output",children:"Expected output"}),"\n",(0,r.jsx)(n.p,{children:"With the shared CPU pool enabled, the throughput log gains prefix\nhit-rate counters:"}),"\n",(0,r.jsx)(n.pre,{children:(0,r.jsx)(n.code,{className:"language-text",children:"[20.0s] Avg prompt throughput: 2412.0 tokens/s, Avg generation throughput: 820.0 tokens/s\n \u251C\u2500Running Instance[0]: 6 reqs, Waiting: 0 reqs, Total # 1 NPUs, Each NPU Memory Usage 63218.51 MB (64.301 % Used), Prefix Cache Hit ratio 41.82 %, (31024 / 74208)\n \u251C\u2500Running Instance[1]: 4 reqs, Waiting: 0 reqs, Total # 1 NPUs, Each NPU Memory Usage 63204.51 MB (64.287 % Used), Prefix Cache Hit ratio 42.11 %, (30298 / 71950)\n \u2514\u2500Node[0]: Total CPU Memory Usage 8192.00 MB, 1.600 % Used, Prefix Cache Hit ratio 36.04 %, (52664 / 146158)\n"})}),"\n",(0,r.jsxs)(n.p,{children:["The per-instance ratios are NPU-tier hits; the ratio on the ",(0,r.jsx)(n.code,{children:"Node"}),"\nline is the shared CPU pool's, and it replaces the per-instance CPU\nsplit because ",(0,r.jsx)(n.code,{children:"--enable-prefix-sharing --prefix-storage CPU"})," makes the\npool node-wide."]}),"\n",(0,r.jsxs)(n.p,{children:["The ",(0,r.jsx)(n.code,{children:"prompt_t"})," (prompt throughput) counts ",(0,r.jsx)(n.strong,{children:"all"})," input tokens,\nincluding those served from cache, matching vLLM's reporting\nconvention."]}),"\n",(0,r.jsx)(n.h2,{id:"whats-interesting",children:"What's interesting"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"NPU memory pressure stays bounded"})," even on workloads with huge\nshared prefixes. The CPU pool absorbs eviction."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Cross-instance reuse"})," is the killer feature for multi-replica\ndeployments. Without prefix sharing, a 90% prefix-overlap workload\neffectively sees prefix caching as 1/N as effective with N\ninstances."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"CXL pool is an option when CPU memory is the bottleneck."})," Set\n",(0,r.jsx)(n.code,{children:"--prefix-storage CXL"})," and add a ",(0,r.jsx)(n.code,{children:"cxl_mem"})," block to the cluster\nconfig (see ",(0,r.jsx)(n.a,{href:"./cxl-memory",children:"CXL memory tier"}),"). The pool then lives\nin CXL memory at CXL latency."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:"Block-aware tracking."})," The simulator's\n",(0,r.jsx)(n.code,{children:"prompt_t"})," accumulator includes prefix-cache-hit tokens, so its\nreported prompt throughput matches vLLM's (which also counts cached\ntokens)."]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"related-examples",children:"Related examples"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:(0,r.jsx)(n.a,{href:"./cxl-memory",children:"CXL memory tier"})}),": backing the prefix pool with\nCXL memory."]}),"\n",(0,r.jsxs)(n.li,{children:[(0,r.jsx)(n.strong,{children:(0,r.jsx)(n.a,{href:"../disaggregated/multi-instance",children:"Multi-instance LOAD routing"})})," -\nthe multi-instance baseline this builds on."]}),"\n"]})]})}function h(e={}){let{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(d,{...e})}):d(e)}},31137(e,n,s){s.d(n,{R:()=>o,x:()=>l});var i=s(2086);let r={},t=i.createContext(r);function o(e){let n=i.useContext(t);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function l(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:o(e.components),i.createElement(t.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.