1"use strict";(self.webpackChunksemantic_router_docs=self.webpackChunksemantic_router_docs||[]).push([[11829],{44599(e,t,n){n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>u,frontMatter:()=>r,metadata:()=>s,toc:()=>c});const s=JSON.parse('{"id":"installation/k8s/istio","title":"Install with Istio Gateway","description":"This guide provides step-by-step instructions for deploying the vLLM Semantic Router (vsr) with Istio Gateway on Kubernetes. Istio Gateway uses Envoy under the covers so it is possible to use vsr with it. However there are differences between how different Envoy based Gateways process the ExtProc protocol, hence the deployment described here is different from the deployment of vsr alongwith other types of Envoy based Gateways as described in the other guides in this repo. There are multiple architecture options possible to combine Istio Gateway with vsr. This document describes one of the options.","source":"@site/versioned_docs/version-v0.3/installation/k8s/istio.md","sourceDirName":"installation/k8s","slug":"/installation/k8s/istio","permalink":"/docs/v0.3/installation/k8s/istio","draft":false,"unlisted":false,"editUrl":"https://github.com/vllm-project/semantic-router/edit/main/website/docs/installation/k8s/istio.md","tags":[],"version":"v0.3","frontMatter":{},"sidebar":"tutorialSidebar","previous":{"title":"Install with agentgateway","permalink":"/docs/v0.3/installation/k8s/agentgateway"},"next":{"title":"Install with Gateway API Inference Extension","permalink":"/docs/v0.3/installation/k8s/gateway-api-inference-extension"}}');var i=n(74848),o=n(28453);const r={},a="Install with Istio Gateway",l={},c=[{value:"Architecture Overview",id:"architecture-overview",level:2},{value:"Prerequisites",id:"prerequisites",level:2},{value:"Step 1: Create Minikube Cluster",id:"step-1-create-minikube-cluster",level:2},{value:"Step 2: Deploy LLM models",id:"step-2-deploy-llm-models",level:2},{value:"Step 3: Install Istio Gateway, Gateway API, Inference Extension CRDs",id:"step-3-install-istio-gateway-gateway-api-inference-extension-crds",level:2},{value:"Step 4: Update vsr config (Optional)",id:"step-4-update-vsr-config-optional",level:2},{value:"Step 5: Deploy vLLM Semantic Router",id:"step-5-deploy-vllm-semantic-router",level:2},{value:"Step 6: Install additional Istio configuration",id:"step-6-install-additional-istio-configuration",level:2},{value:"Step 7: Install gateway routes",id:"step-7-install-gateway-routes",level:2},{value:"Step 8: Testing the Deployment",id:"step-8-testing-the-deployment",level:2},{value:"Send Test Requests",id:"send-test-requests",level:3},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"Common Issues",id:"common-issues",level:3},{value:"Cleanup",id:"cleanup",level:2},{value:"Next Steps",id:"next-steps",level:2}];function d(e){const t={a:"a",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,o.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(t.header,{children:(0,i.jsx)(t.h1,{id:"install-with-istio-gateway",children:"Install with Istio Gateway"})}),"\n",(0,i.jsx)(t.p,{children:"This guide provides step-by-step instructions for deploying the vLLM Semantic Router (vsr) with Istio Gateway on Kubernetes. Istio Gateway uses Envoy under the covers so it is possible to use vsr with it. However there are differences between how different Envoy based Gateways process the ExtProc protocol, hence the deployment described here is different from the deployment of vsr alongwith other types of Envoy based Gateways as described in the other guides in this repo. There are multiple architecture options possible to combine Istio Gateway with vsr. This document describes one of the options."}),"\n",(0,i.jsx)(t.h2,{id:"architecture-overview",children:"Architecture Overview"}),"\n",(0,i.jsx)(t.p,{children:"The deployment consists of:"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"vLLM Semantic Router"}),": Provides intelligent request routing and processing decisions to Envoy based Gateways"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"Istio Gateway"}),": Istio's implementation of Kubernetes Gateway API that uses an Envoy proxy under the covers"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"Gateway API Inference Extension"}),": Additional APIs to extend the Gateway API for Inference via ExtProc servers"]}
1),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.strong,{children:"Two instances of vLLM serving 1 model each"}),": Example backend LLMs for illustrating semantic routing in this topology"]}),"\n"]}),"\n",(0,i.jsx)(t.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,i.jsx)(t.p,{children:"Before starting, ensure you have the following tools installed:"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://docs.docker.com/get-docker/",children:"Docker"})," - Container runtime"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://minikube.sigs.k8s.io/docs/start/",children:"minikube"})," - Local Kubernetes"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://kind.sigs.k8s.io/docs/user/quick-start/#installation",children:"kind"})," - Kubernetes in Docker"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://kubernetes.io/docs/tasks/tools/",children:"kubectl"})," - Kubernetes CLI"]}),"\n",(0,i.jsxs)(t.li,{children:[(0,i.jsx)(t.a,{href:"https://helm.sh/docs/intro/install/",children:"Helm"})," - Package manager for Kubernetes"]}),"\n"]}),"\n",(0,i.jsx)(t.p,{children:"Either minikube or kind works to deploy a local kubernetes cluster needed for this exercise so you only need one of these two. We use minikube in the description below but the same steps should work with a Kind cluster once the cluster is created in Step 1."}),"\n",(0,i.jsx)(t.p,{children:"We will also deploy two different LLMs in this exercise to illustrate the semantic routing and model routing function more clearly so you ideally you should run this on a machine that has GPU support to run the two models used in this exercise and adequate memory and storage for these models. You can also use equivalent steps on a smaller server that runs smaller LLMs on a CPU based server without GPUs."}),"\n",(0,i.jsx)(t.h2,{id:"step-1-create-minikube-cluster",children:"Step 1: Create Minikube Cluster"}),"\n",(0,i.jsx)(t.p,{children:"Create a local Kubernetes cluster via minikube (or equivalently via Kind)."}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Create cluster\n$ minikube start \\\n --driver docker \\\n --container-runtime docker \\\n --gpus all \\\n --memory no-limit \\\n --cpus no-limit\n\n# Verify cluster is ready\n$ kubectl wait --for=condition=Ready nodes --all --timeout=300s\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-2-deploy-llm-models",children:"Step 2: Deploy LLM models"}),"\n",(0,i.jsxs)(t.p,{children:["In this exercise we deploy two LLMs viz. a llama3-8b model (meta-llama/Llama-3.1-8B-Instruct) and a phi4-mini model (microsoft/Phi-4-mini-instruct). We serve these models using two separate instances of the ",(0,i.jsx)(t.a,{href:"https://docs.vllm.ai/en/latest/",children:"vLLM inference server"})," running in the default namespace of the kubernetes cluster. You may choose any other inference engines as long as they expose OpenAI API endpoints. First install a secret for your HuggingFace token previously stored in env variable HF_TOKEN and then deploy the models as shown below. Note that the file path names used in the example kubectl clis in this guide are expected to be executed from the top folder of this repo."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl create secret generic hf-token-secret --from-literal=token=$HF_TOKEN\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Create vLLM service running llama3-8b\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vLlama3.yaml\n"})}),"\n",(0,i.jsx)(t.p,{children:"This may take several (10+) minutes the first time this is run to download the model up until the vLLM pod running this model is in READY state. Similarly also deploy the second LLM (phi4-mini) and wait for several minutes until the pod is in READY state."}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Create vLLM service running phi4-mini\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vPhi4.y
1aml\n"})}),"\n",(0,i.jsx)(t.p,{children:"At the end of this you should be able to see both your vLLM pods are READY and serving these LLMs using the command below. You should also see Kubernetes services exposing the IP/ port on which these models are being served. In the example below the llama3-8b model is being served via a kubernetes service with service IP of 10.108.250.109 and port 80."}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Verify that vLLM pods running the two LLMs are READY and serving\n\nkubectl get pods\nNAME READY STATUS RESTARTS AGE\nllama-8b-57b95475bd-ph7s4 1/1 Running 0 9d\nphi4-mini-887476b56-74twv 1/1 Running 0 9d\n\n# View the IP/port of the Kubernetes services on which these models are being served\n\nkubectl get service\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\nkubernetes ClusterIP 10.96.0.1 <none> 443/TCP 36d\nllama-8b ClusterIP 10.108.250.109 <none> 80/TCP 18d\nphi4-mini ClusterIP 10.97.252.33 <none> 80/TCP 9d\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-3-install-istio-gateway-gateway-api-inference-extension-crds",children:"Step 3: Install Istio Gateway, Gateway API, Inference Extension CRDs"}),"\n",(0,i.jsx)(t.p,{children:"We will use a recent build of Istio for this exercise so that we have the option of also using the v1.0.0 GA version of the Gateway API Inference Extension CRDs and EPP functionality."}),"\n",(0,i.jsxs)(t.p,{children:["Follow the procedures described in the Gateway API ",(0,i.jsx)(t.a,{href:"https://gateway-api-inference-extension.sigs.k8s.io/guides/",children:"Inference Extensions documentation"})," to deploy the 1.28 (or newer) version of Istio control plane, Istio Gateway, the Kubernetes Gateway API CRDs and the Gateway API Inference Extension v1.0.0. Do not install any of the HTTPRoute resources nor the EndPointPicker from that guide however, just use it to deploy the Istio gateway and CRDs. If installed correctly you should see the api CRDs for gateway api and inference extension as well as pods running for the Istio gateway and Istiod using the commands shown below."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl get crds | grep gateway\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl get crds | grep inference\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl get pods | grep istio\n"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl get pods -n istio-system\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-4-update-vsr-config-optional",children:"Step 4: Update vsr config (Optional)"}),"\n",(0,i.jsx)(t.p,{children:"The semantic router configuration is provided via a Helm values file. If you need to customize the configuration (e.g., to match different model names or endpoints), download the values file and modify it:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Download the values file for customization\ncurl -O https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/semantic-router-values/values.yaml\n"})}),"\n",(0,i.jsx)(t.p,{children:"Ensure that the models in the config file match the models you are using. It is usually good to start with basic features of vsr such as prompt classification and model routing before experimenting with other features such as PromptGuard or ToolCalling."}),"\n",(0,i.jsx)(t.h2,{id:"step-5-deploy-vllm-semantic-router",children:"Step 5: Deploy vLLM Semantic Router"}),"\n",(0,i.jsx)(t.p,{children:"Deploy the semantic router service with all required components using Helm:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Install semantic router using Helm from GHCR OCI registry\nhelm install semantic-router oci://ghcr.io/vllm-project/charts/semantic-router \\\n --version v0.0.0-latest \\\n --namespace vllm-semantic-router-system \\\n --create-namespace \\\n -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/semantic-router-values/values.yaml\n\n# Wait for deployment to be ready (this may take several minutes for model downloads)\nkubectl wait --for=condition=Available deployment/semantic-router -n vllm-semantic-router-system --timeout=600s\n\n# Verify deployment status\nkubectl get pods -n vllm-semantic-router-system\n"})}),"\n",(0,i.jsxs)(t.p,{children:[(0,i.jsx)(t.strong,{children:"Note"}),": The values file contains the configuration for the semantic router, including model settings, c
1ategories, and routing rules. You can download and customize it from ",(0,i.jsx)(t.a,{href:"https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/semantic-router-values/values.yaml",children:"values.yaml"}),"."]}),"\n",(0,i.jsx)(t.h2,{id:"step-6-install-additional-istio-configuration",children:"Step 6: Install additional Istio configuration"}),"\n",(0,i.jsx)(t.p,{children:"Install the destinationrule and envoy filter needed for Istio gateway to use ExtProc based interface with vLLM Semantic router"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/destinationrule.yaml\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/envoyfilter.yaml\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-7-install-gateway-routes",children:"Step 7: Install gateway routes"}),"\n",(0,i.jsx)(t.p,{children:"Install HTTPRoutes in the Istio gateway."}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"kubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-llama3-8b.yaml\nkubectl apply -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-phi4-mini.yaml\n"})}),"\n",(0,i.jsx)(t.h2,{id:"step-8-testing-the-deployment",children:"Step 8: Testing the Deployment"}),"\n",(0,i.jsxs)(t.p,{children:["To expose the IP on which the Istio gateway listens to client requests from outside the cluster, you can choose any standard kubernetes option for external load balancing. We tested our feature by ",(0,i.jsx)(t.a,{href:"https://metallb.universe.tf/installation/",children:"deploying and configuring metallb"})," into the cluster to be the LoadBalancer provider. Please refer to metallb documentation for installation procedures if needed. Finally, for the minikube case, we get the external url as shown below."]}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"minikube service inference-gateway-istio --url\nhttp://192.168.49.2:30913\n"})}),"\n",(0,i.jsxs)(t.p,{children:["Now we can send LLM prompts via curl to ",(0,i.jsx)(t.a,{href:"http://192.168.49.2:30913",children:"http://192.168.49.2:30913"})," to access the Istio gateway which will then use information from vLLM semantic router to dynamically route to one of the two LLMs we are using as backends in this case."]}),"\n",(0,i.jsx)(t.h3,{id:"send-test-requests",children:"Send Test Requests"}),"\n",(0,i.jsx)(t.p,{children:'Try the following cases with and without model "auto" selection to confirm that Istio + vsr together are able to route queries to the appropriate model. The query responses will include information about which model was used to serve that request.'}),"\n",(0,i.jsx)(t.p,{children:"Example queries to try include the following"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:'# Model name llama3-8b provided explicitly, should route to this backend\ncurl http://192.168.49.2:30913/v1/chat/completions -H "Content-Type: application/json" -d \'{\n "model": "llama3-8b",\n "messages": [\n {"role": "user", "content": "Linux is said to be an open source kernel because "}\n ],\n "max_tokens": 100,\n "temperature": 0\n }\'\n'})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:'# Model name set to "auto", should categorize to "computer science" & route to llama3-8b\ncurl http://192.168.49.2:30913/v1/chat/completions -H "Content-Type: application/json" -d \'{\n "model": "auto",\n "messages": [\n {"role": "user", "content": "Linux is said to be an open source kernel because "}\n ],\n "max_tokens": 100,\n "temperature": 0\n }\'\n'})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:'# Model name phi4-mini provided explicitly, should route to this backend\ncurl http://192.168.49.2:30913/v1/c
1hat/completions -H "Content-Type: application/json" -d \'{\n "model": "phi4-mini",\n "messages": [\n {"role": "user", "content": "2+2 is "}\n ],\n "max_tokens": 100,\n "temperature": 0\n }\'\n'})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:'# Model name set to "auto", should categorize to "math" & route to phi4-mini\ncurl http://192.168.49.2:30913/v1/chat/completions -H "Content-Type: application/json" -d \'{\n "model": "auto",\n "messages": [\n {"role": "user", "content": "2+2 is "}\n ],\n "max_tokens": 100,\n "temperature": 0\n }\'\n'})}),"\n",(0,i.jsx)(t.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,i.jsx)(t.h3,{id:"common-issues",children:"Common Issues"}),"\n",(0,i.jsx)(t.p,{children:(0,i.jsx)(t.strong,{children:"Gateway/ Front end not working:"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Check istio gateway status\nkubectl get gateway\n\n# Check istio gw service status\nkubectl get svc inference-gateway-istio\n\n# Check Istio's Envoy logs\nkubectl logs deploy/inference-gateway-istio -c istio-proxy\n"})}),"\n",(0,i.jsx)(t.p,{children:(0,i.jsx)(t.strong,{children:"Semantic router not responding:"})}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Check semantic router pod\nkubectl get pods -n vllm-semantic-router-system\n\n# Check semantic router service\nkubectl get svc -n vllm-semantic-router-system\n\n# Check semantic router logs\nkubectl logs -n vllm-semantic-router-system deployment/semantic-router\n"})}),"\n",(0,i.jsx)(t.h2,{id:"cleanup",children:"Cleanup"}),"\n",(0,i.jsx)(t.p,{children:"To remove the entire deployment:"}),"\n",(0,i.jsx)(t.pre,{children:(0,i.jsx)(t.code,{className:"language-bash",children:"# Remove gateway routes\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-llama3-8b.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/httproute-phi4-mini.yaml\n\n# Remove Istio configuration\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/envoyfilter.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/destinationrule.yaml\n\n# Remove semantic router\nhelm uninstall semantic-router -n vllm-semantic-router-system\n\n# Remove Istio\nistioctl uninstall --purge\n\n# Remove LLMs\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vLlama3.yaml\nkubectl delete -f https://raw.githubusercontent.com/vllm-project/semantic-router/refs/heads/main/deploy/kubernetes/istio/vPhi4.yaml\n\n# Stop minikube cluster\nminikube stop\n\n# Delete minikube cluster (optional)\nminikube delete\n"})}),"\n",(0,i.jsx)(t.h2,{id:"next-steps",children:"Next Steps"}),"\n",(0,i.jsxs)(t.ul,{children:["\n",(0,i.jsx)(t.li,{children:"Test/ experiment with different features of vLLM Semantic Router"}),"\n",(0,i.jsx)(t.li,{children:"Additional use cases/ topologies with Istio Gateway (including with EPP and LLM-D)"}),"\n",(0,i.jsx)(t.li,{children:"Set up monitoring and observability"}),"\n",(0,i.jsx)(t.li,{children:"Implement authentication and authorization"}),"\n",(0,i.jsx)(t.li,{children:"Scale the semantic router deployment for production workloads"}),"\n"]})]})}function u(e={}){const{wrapper:t}={...(0,o.R)(),...e.components};return t?(0,i.jsx)(t,{...e,children:(0,i.jsx)(d,{...e})}):d(e)}},28453(e,t,n){n.d(t,{R:()=>r,x:()=>a});var s=n(96540);const i={},o=s.createContext(i);function r(e){const t=s.useContext(o);return s.useMemo(function(){return"function"==typeof e?e(t):{...t,...e}},[t,e])}function a(e){let t;return t=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:r(e.components),s.createElement(o.Provider,{value:t},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.