1"use strict";(self.webpackChunksemantic_router_docs=self.webpackChunksemantic_router_docs||[]).push([[8004],{9695(e,n,t){t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>o,default:()=>h,frontMatter:()=>d,metadata:()=>i,toc:()=>c});const i=JSON.parse('{"id":"tutorials/global/model-runtime","title":"Built-in Model Runtime","description":"Overview","source":"@site/docs/tutorials/global/model-runtime.md","sourceDirName":"tutorials/global","slug":"/tutorials/global/model-runtime","permalink":"/docs/tutorials/global/model-runtime","draft":false,"unlisted":false,"editUrl":"https://github.com/vllm-project/semantic-router/edit/main/website/docs/tutorials/global/model-runtime.md","tags":[],"version":"current","frontMatter":{},"sidebar":"tutorialSidebar","previous":{"title":"Vela Router Models","permalink":"/docs/tutorials/global/vela-models"},"next":{"title":"Configuration","permalink":"/docs/installation/configuration"}}');var s=t(74848),r=t(28453);const d={},o="Built-in Model Runtime",l={},c=[{value:"Overview",id:"overview",level:2},{value:"What Problem Does It Solve?",id:"what-problem-does-it-solve",level:2},{value:"When to Use",id:"when-to-use",level:2},{value:"Configuration",id:"configuration",level:2}];function a(e){const n={code:"code",h1:"h1",h2:"h2",header:"header",p:"p",pre:"pre",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",...(0,r.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"built-in-model-runtime",children:"Built-in Model Runtime"})}),"\n",(0,s.jsx)(n.h2,{id:"overview",children:"Overview"}),"\n",(0,s.jsxs)(n.p,{children:["The model runtime serves decision models for the Router. It is a separate\nprocess with one HTTP contract: ",(0,s.jsx)(n.code,{children:"POST /v1/decisions"})," (a strict superset of\nSystem One, also available as ",(0,s.jsx)(n.code,{children:"POST /v1/systemone"}),"), ",(0,s.jsx)(n.code,{children:"GET /v1/models"}),",\n",(0,s.jsx)(n.code,{children:"GET /health"})," and ",(0,s.jsx)(n.code,{children:"GET /metrics"}),". The Router talks to it over a Unix socket\nwhen it manages the runtime, or over HTTP when it attaches to an engine\nstarted elsewhere."]}),"\n",(0,s.jsx)(n.p,{children:"Phase 1 serves the six Decision 2.0 models out of the box:"}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Model"}),(0,s.jsx)(n.th,{children:"Revision"}),(0,s.jsx)(n.th,{children:"Backbone"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"vllm-sr/Decision-2.0-Kai-0.6B"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"881bee41"})}),(0,s.jsx)(n.td,{children:"Qwen3 dense"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"vllm-sr/Decision-2.0-Eos-0.8B"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"ad0aa724"})}),(0,s.jsx)(n.td,{children:"Qwen3.5 Gated DeltaNet hybrid"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"vllm-sr/Decision-2.0-Sol-2B"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"4b75b521"})}),(0,s.jsx)(n.td,{children:"Qwen3.5 Gated DeltaNet hybrid"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"vllm-sr/Decision-2.0-Nox-4B"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"ce1bdc9d"})}),(0,s.jsx)(n.td,{children:"Qwen3.5 Gated DeltaNet hybrid"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"vllm-sr/Decision-2.0-Lux-9B"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"214ffa43"})}),(0,s.jsx)(n.td,{children:"Qwen3.5 Gated DeltaNet hybrid"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"vllm-sr/Decision-2.0-Vega-27B"})}),(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"9b067a95"})}),(0,s.jsx)(n.td,{children:"Qwen3.8-27B with a LoRA adapter"})]})]})]}),"\n",(0,s.jsx)(n.p,{children:"Every package is pinned by revision and verified file by file against its\nmanifest before load. Model code is built into the runtime; code shipped\ninside a model package is never executed."}),"\n",(0,s.jsx)(n.h2,{id:"what-problem-does-it-solve",children:"What Problem Does It Solve?"}),"\n",(0,s.jsx)(n.p,{children:"Decision models are decoders with custom readouts, so they do not fit the\nRouter's in-
1process encoder bindings. The runtime gives them one supervised,\npluggable home: model families, execution engines and hardware accelerators\nare plugins, and faster numerics are opt-in profiles measured against the\ndefault."}),"\n",(0,s.jsx)(n.h2,{id:"when-to-use",children:"When to Use"}),"\n",(0,s.jsxs)(n.p,{children:["Use it whenever a ",(0,s.jsx)(n.code,{children:"decision"})," signal or the ",(0,s.jsx)(n.code,{children:"decision"})," selection algorithm is\nconfigured. Run it on its own with ",(0,s.jsx)(n.code,{children:"vllm-sr serve <hf-model>"})," to serve a\ndecision model to other clients or to share one engine between Routers."]}),"\n",(0,s.jsx)(n.h2,{id:"configuration",children:"Configuration"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-yaml",children:"global:\n model_catalog:\n deployments:\n decision-kai:\n provider: model_runtime\n artifact: vllm-sr/Decision-2.0-Kai-0.6B # Hub repository or an absolute package path\n revision: 881bee413681d80ebeac86afcda8b4138dae516e\n device: auto # auto, cpu, cuda[:N] or rocm[:N]\n profile: exact # exact, shared_context, batching or max_speed\n decision-shared:\n provider: model_runtime\n endpoint: http://decision-runtime:8100 # attach; unix:///path also works\n"})}),"\n",(0,s.jsxs)(n.p,{children:["Without ",(0,s.jsx)(n.code,{children:"endpoint"}),", the Router starts ",(0,s.jsx)(n.code,{children:"vllm-sr-runtime serve"})," for each\ndeployment that a signal or selector uses, on a private Unix socket, restarts\nit with back-off when it exits, and stops it on shutdown. Until the runtime\nreports ready, decision signals are unknown and decision selectors fall back.\n",(0,s.jsx)(n.code,{children:"VLLM_SR_RUNTIME_COMMAND"})," overrides the command, ",(0,s.jsx)(n.code,{children:"VLLM_SR_RUNTIME_CACHE_DIR"}),"\nsets the model cache and ",(0,s.jsx)(n.code,{children:"VLLM_SR_RUNTIME_DIR"})," the socket directory."]}),"\n",(0,s.jsxs)(n.p,{children:["The ",(0,s.jsx)(n.code,{children:"vllm-sr"})," image ships the runtime with CPU PyTorch, so managed CPU\ndeployments work out of the box, and downloaded models persist under\n",(0,s.jsx)(n.code,{children:"/app/models/model-runtime"}),". For a GPU, start the runtime with a GPU build of\nPyTorch (",(0,s.jsx)(n.code,{children:"vllm-sr serve <model> --device rocm:0"}),", or the runtime image built\nfrom ",(0,s.jsx)(n.code,{children:"src/model-runtime/Dockerfile"})," with a GPU wheel) and attach to it with\n",(0,s.jsx)(n.code,{children:"endpoint"}),"."]}),"\n",(0,s.jsx)(n.p,{children:"Run an engine on its own:"}),"\n",(0,s.jsx)(n.pre,{children:(0,s.jsx)(n.code,{className:"language-bash",children:"pip install -e src/model-runtime\nvllm-sr serve vllm-sr/Decision-2.0-Kai-0.6B --device cpu --port 8100\nvllm-sr serve vllm-sr/Decision-2.0-Lux-9B --device rocm:0 --profile shared_context\n"})}),"\n",(0,s.jsxs)(n.table,{children:[(0,s.jsx)(n.thead,{children:(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.th,{children:"Profile"}),(0,s.jsx)(n.th,{children:"Numerics"}),(0,s.jsx)(n.th,{children:"Purpose"})]})}),(0,s.jsxs)(n.tbody,{children:[(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"exact"})}),(0,s.jsx)(n.td,{children:"identical to the released packages"}),(0,s.jsx)(n.td,{children:"the default"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"shared_context"})}),(0,s.jsx)(n.td,{children:"approximate"}),(0,s.jsx)(n.td,{children:"run the shared state of a multi-question request once"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"batching"})}),(0,s.jsx)(n.td,{children:"approximate"}),(0,s.jsx)(n.td,{children:"share forwards between concurrent requests"})]}),(0,s.jsxs)(n.tr,{children:[(0,s.jsx)(n.td,{children:(0,s.jsx)(n.code,{children:"max_speed"})}),(0,s.jsx)(n.td,{children:"approximate"}),(0,s.jsx)(n.td,{children:"faster kernels that may change rounding"})]})]})]}),"\n",(0,s.jsxs)(n.p,{children:["CPU and ROCm (MI300X, MI325X) are validated. CUDA is implemented and unit\ntested but not yet validated on NVIDIA hardware. The runtime exposes\nPrometheus metrics at ",(0,s.jsx)(n.code,{children:"/metrics"}),", and the Router exports\n",(0,s.jsx)(n.code,{children:"vsr_model_runtime_*"})," metrics per deployment."]})]})}function h(e={}){const{wrapper:n}={...(0,r.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(a,{...e})}):a(e)}},28453(e,n,t){t.d(n,{R:()=>d,x:()=>o});var i=t(96540);const s={},r=i.createContext(s);function d(e){const n=i.useContext(r);return i.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:d(e.components),i.createElement(r.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.