1"use strict";(self.webpackChunksemantic_router_docs=self.webpackChunksemantic_router_docs||[]).push([[21100],{92582(e,t,n){n.r(t),n.d(t,{assets:()=>r,contentTitle:()=>l,default:()=>u,frontMatter:()=>o,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"installation/k8s/istio","title":"Deploy with Istio Gateway","description":"Run Semantic Router as an ExtProc service behind an Istio Gateway and two model backends.","source":"@site/docs/installation/k8s/istio.md","sourceDirName":"installation/k8s","slug":"/installation/k8s/istio","permalink":"/docs/installation/k8s/istio","draft":false,"unlisted":false,"editUrl":"https://github.com/vllm-project/semantic-router/edit/main/website/docs/installation/k8s/istio.md","tags":[],"version":"current","frontMatter":{"title":"Deploy with Istio Gateway","description":"Run Semantic Router as an ExtProc service behind an Istio Gateway and two model backends."},"sidebar":"tutorialSidebar","previous":{"title":"Streamed ExtProc and immediate responses","permalink":"/docs/installation/k8s/streamed-extproc"},"next":{"title":"Gateway API Inference Extension","permalink":"/docs/installation/k8s/gateway-api-inference-extension"}}');var i=n(74848),a=n(28453);const o={title:"Deploy with Istio Gateway",description:"Run Semantic Router as an ExtProc service behind an Istio Gateway and two model backends."},l="Deploy with Istio Gateway",r={},c=[{value:"Responsibility split",id:"responsibility-split",level:2},{value:"Prerequisites",id:"prerequisites",level:2},{value:"Step 1: Verify the cluster",id:"step-1-verify-the-cluster",level:2},{value:"Step 2: Deploy LLM models",id:"step-2-deploy-llm-models",level:2},{value:"Step 3: Install Gateway API and Istio",id:"step-3-install-gateway-api-and-istio",level:2},{value:"Step 4: Update vsr config (Optional)",id:"step-4-update-vsr-config-optional",level:2},{value:"Step 5: Deploy vLLM Semantic Router",id:"step-5-deploy-vllm-semantic-router",level:2},{value:"Step 6: Install additional Istio configuration",id:"step-6-install-additional-istio-configuration",level:2},{value:"Step 7: Install gateway routes",id:"step-7-install-gateway-routes",level:2},{value:"Step 8: Testing the Deployment",id:"step-8-testing-the-deployment",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"Common Issues",id:"common-issues",level:3},{value:"Cleanup",id:"cleanup",level:2},{value:"Next Steps",id:"next-steps",level:2}];function d(e){const t={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,a.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(t.header,{children:(0,i.jsx)(t.h1,{id:"deploy-with-istio-gateway",children:"Deploy with Istio Gateway"})}),"\n",(0,i.jsxs)(t.p,{children:["This guide shows a reference topology for running Semantic Router as an\nExtProc service behind an Istio Gateway. Istio owns ingress and ",(0,i.jsx)(t.code,{children:"HTTPRoute"}),"\nprocessing; Semantic Router owns prompt-aware model selection. The supplied\n",(0,i.jsx)(t.code,{children:"EnvoyFilter"})," and ",(0,i.jsx)(t.code,{children:"DestinationRule"})," are specific to this example, so do not\nreuse manifests from a different gateway integration without comparing its\nExtProc mode."]}),"\n",(0,i.jsx)(t.h2,{id:"responsibility-split",children:"Responsibility split"}),"\n",(0,i.jsx)(t.p,{children:"The deployment consists of:"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"Semantic Router"})," evaluates routing policy and selects a model."]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"Istio Gateway"})," accepts client traffic and calls Semantic Router through\nExtProc."]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"Gateway API resources"})," map the selected model to a Kubernetes backend."]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"The two vLLM deployments"})," are example inference backends and require\nsuitable compute and model access."]}),"\n"]}),"\n",(0,i.jsx)(t.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,i.jsx)(t.p,{children:"You need:"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:["Kubernetes ",(0,i.jsx)(t.code,{children:"1.31"}),"\u2013",(0,i.jsx)(t.code,{children:"1.35"}),", the supported range for the pinned Istio ",(0,i.jsx)(t.code,{children:"1.29"}),"\nrelease, with at least two schedulable NVIDIA GPUs for the supplied model\nmanifests or equivalent capacity for replacement backends;"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://kubernetes.io/docs/tasks/tools/",children:"kubectl"}),";"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://helm.sh/docs/intro/install/",children:"Helm"}),"; and"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://istio.io/latest/docs/ops/diagnostic-tools/istioctl/",children:"istioctl"}),"."]}),"\n"]}),"\n",(0,i.jsxs)(t.p,{children:["The supplied manifests request one ",(0,i.jsx)(t.code,{children:"nvidia.com/gpu"})," device for each of two\nvLLM Deployments. They pin the vLLM image used by this example. You can replace\nthem with other OpenAI-compatible backends, but the Router providers, Service\nnames, and ",(0,i.jsx)(t.code,{children:"HTTPRoute"})," matches must change together."]}
1),"\n",(0,i.jsx)(t.h2,{id:"step-1-verify-the-cluster",children:"Step 1: Verify the cluster"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl wait --for=condition=Ready nodes --all --timeout=300s\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-2-deploy-llm-models",children:"Step 2: Deploy LLM models"}),"\n",(0,i.jsxs)(t.p,{children:["The example deploys ",(0,i.jsx)(t.code,{children:"meta-llama/Llama-3.1-8B-Instruct"})," and\n",(0,i.jsx)(t.code,{children:"microsoft/Phi-4-mini-instruct"})," with separate vLLM servers. Export a Hugging\nFace token before creating the Kubernetes Secret. To use different\nOpenAI-compatible backends, update the model names and endpoint references in\nboth the Router values and the route manifests."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl create secret generic hf-token-secret --from-literal=token=$HF_TOKEN\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Create vLLM service running llama3-8b\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vLlama3.yaml\n"})}),"\n",(0,i.jsx)(t.p,{children:"The first start downloads model weights and can take several minutes. Deploy\nthe second backend, then wait for both Deployments."}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Create vLLM service running phi4-mini\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vPhi4.yaml\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl wait --for=condition=Available deployment/llama-8b --timeout=900s\nkubectl wait --for=condition=Available deployment/phi4-mini --timeout=900s\nkubectl get pods,services\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-3-install-gateway-api-and-istio",children:"Step 3: Install Gateway API and Istio"}),"\n",(0,i.jsx)(t.p,{children:"This direct-Service topology needs Kubernetes Gateway API and Istio; it does\nnot need Gateway API Inference Extension CRDs. Install a compatible pair and\npin the versions in your deployment automation:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:'export GATEWAY_API_VERSION=v1.5.1\nexport ISTIO_VERSION=1.29.6\n\nkubectl apply --server-side \\\n -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/standard-install.yaml"\n\ncurl -L https://istio.io/downloadIstio | ISTIO_VERSION="${ISTIO_VERSION}" sh -\nexport PATH="$PWD/istio-${ISTIO_VERSION}/bin:$PATH"\nistioctl install -y --set profile=minimal\n\nkubectl wait --for=condition=Available deployment/istiod \\\n -n istio-system --timeout=300s\n'})}),"\n",(0,i.jsx)(t.h2,{id:"step-4-update-vsr-config-optional",children:"Step 4: Update vsr config (Optional)"}),"\n",(0,i.jsx)(t.p,{children:"The semantic router configuration is provided via a Helm values file. If you need to customize the configuration (e.g., to match different model names or endpoints), download the values file and modify it:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Download the values file for customization\ncurl -O https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/semantic-router-values/values.yaml\n"})}),"\n",(0,i.jsx)(t.p,{children:"Ensure that the models in the config file match the models you are using. It is usually good to start with basic features of vsr such as prompt classification and model routing before experimenting with other features such as PromptGuard or ToolCalling."}),"\n",(0,i.jsx)(t.h2,{id:"step-5-deploy-vllm-semantic-router",children:"Step 5: Deploy vLLM Semantic Router"}),"\n",(0,i.jsx)(t.p,{children:"Deploy Semantic Router with the integration values:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Install semantic router using Helm from GHCR OCI registry\nhelm install semantic-router oci://ghcr.io/vllm-project/charts/semantic-router \\\n --version 0.0.0-latest \\\n --namespace vllm-semantic-router-system \\\n --create-namespace \\\n -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/semantic-router-values/values.yaml\n\n# Wait for deployment to be ready (this may take several minutes for model downloads)\nkubectl wait --for=condition=Available deployment/semantic-router -n vllm-semantic-router-system --timeout=600s\n\n# Verify deployment status\nkubectl get pods -n vllm-semantic-router-system\n"})}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Note"}),": The values file contains provider bindings, signals, decisions, and\nrouting rules. Download and review\n",(0,i.jsx)(t.a,{href:"https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/semantic-router-values/values.yaml",children:"values.yaml"}),"\nbefore adapting it to a real provider pool. Keep\n",(0,i.jsx)(t.code,{children:"global.router.clear_route_cache: true"}),": after ExtProc writes\n",(0,i.jsx)(t.code,{children:"x-selected-model"}),", Envoy must discard its earlier route and evaluate the\nheader-based ",(0,i.jsx)(t.code,{children:"HTTPRoute"})," again."]}),"\n",(0,i.jsx)(t.h2,{id:"step-6-install-additional-istio-configuration",children:"Step 6: Install additional Istio configuration"}),"\n",(0,i.jsxs)(t.p,{children:["Install the ",(0,i.jsx)(t.code,{children:"DestinationRule"})," and gateway-scoped ",(0,i.jsx)(t.code,{children:"EnvoyFilter"}
1)," that connect the\nIstio gateway to Semantic Router over ExtProc:"]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/destinationrule.yaml\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/envoyfilter.yaml\n"})}),"\n",(0,i.jsxs)(t.p,{children:["The example filter sets ",(0,i.jsx)(t.code,{children:"failure_mode_allow: true"}),", which lets Envoy continue\nits HTTP filter chain when ExtProc is unavailable. It does not guarantee that\nthe request reaches a backend: the supplied routes require ",(0,i.jsx)(t.code,{children:"x-selected-model"}),",\nso route selection can still fail when the Router did not add that header. A\ntrue bypass needs a deliberate catch-all route, which also changes the security\nboundary. Prefer fail-close behavior when semantic routing is an authorization\nor data-boundary control, and test the exact outage path before production use."]}),"\n",(0,i.jsx)(t.h2,{id:"step-7-install-gateway-routes",children:"Step 7: Install gateway routes"}),"\n",(0,i.jsxs)(t.p,{children:["Create the Istio-managed Gateway, then install the two ",(0,i.jsx)(t.code,{children:"HTTPRoute"})," resources."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/gateway.yaml\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-llama3-8b.yaml\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-phi4-mini.yaml\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-8-testing-the-deployment",children:"Step 8: Testing the Deployment"}),"\n",(0,i.jsxs)(t.p,{children:["Follow ",(0,i.jsx)(t.a,{href:"gateway-testing",children:"Test a Kubernetes Gateway Deployment"})," to resolve the\nactual gateway URL, compare direct and routed model requests, inspect routing\nheaders, and verify the selected backend. Do not copy the example IP or port;\nthey are assigned by your cluster."]}),"\n",(0,i.jsx)(t.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,i.jsx)(t.h3,{id:"common-issues",children:"Common Issues"}),"\n",(0,i.jsx)(t.p,{children:(0,i.jsx)(t.strong,{children:"Gateway/ Front end not working:"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Check istio gateway status\nkubectl get gateway\n\n# Check istio gw service status\nkubectl get svc inference-gateway-istio\n\n# Check Istio's Envoy logs\nkubectl logs deploy/inference-gateway-istio -c istio-proxy\n"})}),"\n",(0,i.jsx)(t.p,{children:(0,i.jsx)(t.strong,{children:"Semantic router not responding:"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Check semantic router pod\nkubectl get pods -n vllm-semantic-router-system\n\n# Check semantic router service\nkubectl get svc -n vllm-semantic-router-system\n\n# Check semantic router logs\nkubectl logs -n vllm-semantic-router-system deployment/semantic-router\n"})}),"\n",(0,i.jsx)(t.h2,{id:"cleanup",children:"Cleanup"}),"\n",(0,i.jsx)(t.p,{children:"To remove the entire deployment:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Remove gateway routes\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-llama3-8b.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-phi4-mini.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/gateway.yaml\n\n# Remove Istio configuration\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/envoyfilter.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/destinationrule.yaml\n\n# Remove semantic router\nhelm uninstall semantic-router -n vllm-semantic-router-system\n\n# Remove Istio\nistioctl uninstall --purge\n\n# Remove LLMs\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vLlama3.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vPhi4.y
1aml\n\n"})}),"\n",(0,i.jsx)(t.h2,{id:"next-steps",children:"Next Steps"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsx)(t.li,{children:"Replace the example models with pinned, production-operated backends."}),"\n",(0,i.jsx)(t.li,{children:"Add authentication, network policy, observability, and capacity controls."}),"\n",(0,i.jsxs)(t.li,{children:["Use the ",(0,i.jsx)(t.a,{href:"gateway-api-inference-extension",children:"Gateway API Inference Extension guide"}),"\nwhen each selected model needs an endpoint picker rather than a direct\nService backend."]}),"\n"]})]})}function u(e={}){const{wrapper:t}={...(0,a.R)(),...e.components};return t?(0,i.jsx)(t,{...e,children:(0,i.jsx)(d,{...e})}):d(e)}},28453(e,t,n){n.d(t,{R:()=>o,x:()=>l});var s=n(96540);const i={},a=s.createContext(i);function o(e){const t=s.useContext(a);return s.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function l(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:o(e.components),s.createElement(a.Provider,{value:t},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.