1"use strict";(self.webpackChunksemantic_router_docs=self.webpackChunksemantic_router_docs||[]).push([[10194],{78473(e,n,t){t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>a,default:()=>h,frontMatter:()=>s,metadata:()=>i,toc:()=>l});const i=JSON.parse('{"id":"proposals/production-stack-integration","title":"Semantic Intelligence Layer for the vLLM Production Stack","description":"Proposes an integration between semantic request routing and the vLLM Production Stack.","source":"@site/docs/proposals/production-stack-integration.md","sourceDirName":"proposals","slug":"/proposals/production-stack-integration","permalink":"/docs/proposals/production-stack-integration","draft":false,"unlisted":false,"editUrl":"https://github.com/vllm-project/semantic-router/edit/main/website/docs/proposals/production-stack-integration.md","tags":[],"version":"current","frontMatter":{"title":"Semantic Intelligence Layer for the vLLM Production Stack","description":"Proposes an integration between semantic request routing and the vLLM Production Stack.","created":"2025-10-13T00:00:00.000Z","status":"Proposal"},"sidebar":"tutorialSidebar","previous":{"title":"Protocol-Neutral Codec Matrix","permalink":"/docs/proposals/multi-protocol-adaptor"},"next":{"title":"Semantic Intelligence Layer for NVIDIA Dynamo","permalink":"/docs/proposals/nvidia-dynamo-integration"}}');var r=t(74848),o=t(28453);const s={title:"Semantic Intelligence Layer for the vLLM Production Stack",description:"Proposes an integration between semantic request routing and the vLLM Production Stack.",created:new Date("2025-10-13T00:00:00.000Z"),status:"Proposal"},a=void 0,c={},l=[{value:"Problem",id:"problem",level:2},{value:"Proposal",id:"proposal",level:2},{value:"Responsibilities",id:"responsibilities",level:2},{value:"Integration contract",id:"integration-contract",level:2},{value:"Cache and scheduling boundary",id:"cache-and-scheduling-boundary",level:2},{value:"Security and failure behavior",id:"security-and-failure-behavior",level:2},{value:"Scope",id:"scope",level:2},{value:"Validation",id:"validation",level:2},{value:"Open questions",id:"open-questions",level:2},{value:"References",id:"references",level:2}];function d(e){const n={a:"a",blockquote:"blockquote",h2:"h2",li:"li",mermaid:"mermaid",ol:"ol",p:"p",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,o.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsxs)(n.blockquote,{children:["\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.strong,{children:"Status:"})," Proposal \xb7 ",(0,r.jsx)(n.strong,{children:"Created:"})," 2025-10-13"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"problem",children:"Problem"}),"\n",(0,r.jsx)(n.p,{children:"The vLLM Production Stack manages model serving, service discovery, replica\nscheduling, and inference lifecycle. Those capabilities do not decide which model pool\nbest fits the meaning or policy requirements of an incoming request."}),"\n",(0,r.jsx)(n.p,{children:"Semantic Router can make that logical model choice, but it should not duplicate the\nProduction Stack's replica scheduling or infrastru
1cture control plane."}),"\n",(0,r.jsx)(n.h2,{id:"proposal",children:"Proposal"}),"\n",(0,r.jsx)(n.p,{children:"Run Semantic Router before the Production Stack request router:"}),"\n",(0,r.jsx)(n.mermaid,{value:'flowchart LR\n Client --\x3e Gateway\n Gateway --\x3e|"ExtProc"| Semantic["Semantic Router"]\n Semantic --\x3e|"provider model"| Router["Production Stack router"]\n Router --\x3e Replica["Eligible vLLM replica"]'}),"\n",(0,r.jsx)(n.p,{children:"Semantic Router selects a provider model from the recipe. The Production Stack then\nselects a replica that serves that model."}),"\n",(0,r.jsx)(n.h2,{id:"responsibilities",children:"Responsibilities"}),"\n",(0,r.jsxs)(n.table,{children:[(0,r.jsx)(n.thead,{children:(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.th,{children:"Component"}),(0,r.jsx)(n.th,{children:"Owns"})]})}),(0,r.jsxs)(n.tbody,{children:[(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Semantic Router"}),(0,r.jsx)(n.td,{children:"Entrypoints, signals, decisions, logical model selection, and recipe-scoped plugins."})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"vLLM Production Stack"}),(0,r.jsx)(n.td,{children:"Model deployment, service discovery, replica scheduling, and inference lifecycle."})]}),(0,r.jsxs)(n.tr,{children:[(0,r.jsx)(n.td,{children:"Gateway"}),(0,r.jsx)(n.td,{children:"Client traffic, ExtProc attachment, and forwarding to the selected backend."})]})]})]}),"\n",(0,r.jsx)(n.p,{children:"This separation keeps semantic policy independent from changing replica topology."}),"\n",(0,r.jsx)(n.h2,{id:"integration-contract",children:"Integration contract"}),"\n",(0,r.jsx)(n.p,{children:"Each selectable provider model must map to a stable OpenAI-compatible Production Stack\nendpoint and served model identifier. Kubernetes DNS names, credentials, and\nprovider-specific identifiers belong in provider bindings. Decisions should reference\nlogical model names rather than cluster addresses."}),"\n",(0,r.jsx)(n.p,{children:"The end-to-end path is ready only when:"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsx)(n.li,{children:"every backend works directly;"}),"\n",(0,r.jsx)(n.li,{children:"the full Semantic Router configuration validates;"}),"\n",(0,r.jsx)(n.li,{children:"the gateway invokes ExtProc before backend selection;"}),"\n",(0,r.jsx)(n.li,{children:"the selected logical model maps to the intended Production Stack pool; and"}),"\n",(0,r.jsx)(n.li,{children:"logs or traces show which replica ultimately executed the request."}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"cache-and-scheduling-boundary",children:"Cache and scheduling boundary"}),"\n",(0,r.jsx)(n.p,{children:"Semantic response caching and inference scheduling solve different problems. A\nresponse cache may complete a request before it reaches the Production Stack.\nPrefix-aware or KV-aware scheduling only affects requests that continue to inference.\nTheir hit rates and latency effects should be measured separately."}),"\n",(0,r.jsx)(n.p,{children:"Semantic signals may narrow the eligible model pool, but they should not directly\nchoose a replica. Live queue depth, prefix locality, and replica health remain\nProduction Stack concerns."}),"\n",(0,r.jsx)(n.h2,{id:"security-and-failure-behavior",children:"Security and failure behavior"}),"\n",(0,r.jsx)(n.p,{children:"Detection is not enforcement. PII, jailbreak, or other signals affect traffic only\nwhen a decision or plugin applies an action."}),"\n",(0,r.jsx)(n.p,{children:"The gateway must declare ExtProc failure behavior. Fail-open preserves a path to the\nbackend but may bypass semantic and security policy. Fail-closed protects policy at the\ncost of availability. The correct choice is route-specific and should be tested."}),"\n",(0,r.jsx)(n.p,{children:"Secrets, tenant identity, and data-retention settings must remain consistent across\nthe gateway, Semantic Router, Production Stack, and any external stores."}),"\n",(0,r.jsx)(n.h2,{id:"scope",children:"Scope"}),"\n",(0,r.jsx)(n.p,{children:"This proposal defines the layering and model-binding contract. It does not:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"replace Production Stack deployment or scheduling APIs;"}),"\n",(0,r.jsx)(n.li,{children:"prescribe a particular replica-routing algorithm;"}),"\n",(0,r.jsx)(n.li,{children:"make quality, cost, or latency claims;"}),"\n",(0,r.jsx)(n.li,{children:"couple Semantic Router releases to Production Stack releases; or"}),"\n",(0,r.jsx)(n.li,{children:"require shared cache state between the two systems."}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"validation",children:"Validation"}),"\n",(0,r.jsx)(n.p,{children:"Use a small set of requests with known expected model pools. Confirm the semantic\nselection and the physical replica independently. Record configuration versions,\nserved model identifiers, cache state, and traffic shape before publishing any\nperformance comparison."}),"\n",(0,r.jsx)(n.h2,{id:"open-questions",children:"Open questions"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:"Should a deployment expose one multi-model endpoint or one endpoint per model pool?"}),"\n",(0,r.jsx)(n.li,{children:"Which request identifier should correlate gateway, semantic, and replica traces?"}),"\n",(0,r.jsx)(n.li,{children:"Which routes must fail closed when Semantic Router is unavailable?"}),"\n",(0,r.jsx)(n.li,{children:"How should model aliases be rolled out without breaking in-flight requests?"}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"references",children:"References"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsx)(n.li,{children:(0,r.jsx)(n.a,{href:"../installation/k8s/production-stack",children:"Current Production Stack integration guide"})}),"\n",(0,r.jsx)(n.li,{children:(0,r.jsx)(n.a,{href:"https://docs.vllm.ai/projects/production-stack/en/latest/",children:"vLLM Production Stack documentation"})}),"\n",(0,r.jsx)(n.li,{children:(0,r.jsx)(n.a,{href:"https://docs.vllm.ai/projects/production-stack/en/latest/use_cases/semantic-router-integration.html",children:"Production Stack semantic-routing use case"})}),"\n",(0,r.jsx)(n.li,{children:(0,r.jsx)(n.a,{href:"../overview/semantic-router-overview",children:"Semantic Router system overview"})}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(d,{...e})}):d(e)}},28453(e,n,t){t.d(n,{R:()=>s,x:()=>a});var i=t(96540);const r={},o=i.createContext(r);function s(e){const n=i.useContext(o);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function a(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(r):e.components||r:s(e.components),i.createElement(o.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.